feat: clean up staging objects after processing and during reconciliation
This commit is contained in:
+10
-5
@@ -59,12 +59,13 @@ func run() error {
|
||||
return err
|
||||
}
|
||||
|
||||
reconciler := reconcile.New(s3Client, db.New(pool), log)
|
||||
reconciler := reconcile.New(pool, s3Client, db.New(pool), log)
|
||||
|
||||
report, err := reconciler.Run(ctx, reconcile.Options{
|
||||
DryRun: *dryRun,
|
||||
Grace: *grace,
|
||||
MaxDeletes: *maxDeletes,
|
||||
DryRun: *dryRun,
|
||||
Grace: *grace,
|
||||
MaxDeletes: *maxDeletes,
|
||||
StagingRetention: cfg.S3.StagingRetention,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -96,7 +97,8 @@ func printReport(report reconcile.Report) {
|
||||
report.ListPages,
|
||||
)
|
||||
fmt.Printf(
|
||||
"found orphan_objects=%d missing_attachments=%d corrupt_objects=%d\n",
|
||||
"found staging_objects=%d orphan_objects=%d missing_attachments=%d corrupt_objects=%d\n",
|
||||
len(report.StagingObjects),
|
||||
len(report.OrphanObjects),
|
||||
len(report.MissingAttachments),
|
||||
len(report.CorruptObjects),
|
||||
@@ -112,6 +114,9 @@ func printReport(report reconcile.Report) {
|
||||
report.MarkedFailed,
|
||||
)
|
||||
|
||||
for _, key := range report.StagingObjects {
|
||||
fmt.Printf("staging_object %s\n", key)
|
||||
}
|
||||
for _, key := range report.OrphanObjects {
|
||||
fmt.Printf("orphan_object %s\n", key)
|
||||
}
|
||||
|
||||
@@ -50,6 +50,11 @@ INSERT INTO message_attachments (
|
||||
SELECT * FROM attachments
|
||||
WHERE id = $1::uuid;
|
||||
|
||||
-- name: LockAttachmentByStagingKey :one
|
||||
SELECT * FROM attachments
|
||||
WHERE staging_key = sqlc.arg(staging_key)::text
|
||||
FOR UPDATE;
|
||||
|
||||
-- name: ListAttachmentsByIDs :many
|
||||
SELECT status
|
||||
FROM attachments
|
||||
@@ -69,7 +74,7 @@ WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[])
|
||||
ORDER BY message_id, position;
|
||||
|
||||
-- name: ListAttachmentsForReconcile :many
|
||||
SELECT id, s3_key, status, size_bytes
|
||||
SELECT id, s3_key, staging_key, status, size_bytes, updated_at
|
||||
FROM attachments;
|
||||
|
||||
-- name: LockAttachmentJob :one
|
||||
@@ -90,11 +95,10 @@ ORDER BY lease_until, id
|
||||
LIMIT sqlc.arg(batch_size)::int
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
-- name: MarkAttachmentReady :one
|
||||
-- name: ClearAttachmentStagingKey :exec
|
||||
UPDATE attachments
|
||||
SET status = 'ready', updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
RETURNING *;
|
||||
SET staging_key = NULL
|
||||
WHERE id = sqlc.arg(id)::uuid;
|
||||
|
||||
-- name: MarkAttachmentFailedIfReady :execrows
|
||||
UPDATE attachments
|
||||
|
||||
@@ -76,6 +76,7 @@ type S3Config struct {
|
||||
BootstrapSecretKeyFile string
|
||||
MaxUploadBytes int
|
||||
MaxImagePixels int
|
||||
StagingRetention time.Duration
|
||||
}
|
||||
|
||||
func (c S3Config) APIAccessKey() (string, error) {
|
||||
@@ -138,6 +139,7 @@ func LoadS3() S3Config {
|
||||
BootstrapSecretKeyFile: getEnv("S3_BOOTSTRAP_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_bootstrap_secret_access_key"),
|
||||
MaxUploadBytes: getEnvPositiveInt("MAX_UPLOAD_BYTES", 10485760),
|
||||
MaxImagePixels: getEnvPositiveInt("MAX_IMAGE_PIXELS", 40000000),
|
||||
StagingRetention: time.Duration(getEnvPositiveInt("STAGING_RETENTION_SECONDS", 86400)) * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +67,17 @@ func (q *Queries) ClaimAttachmentJob(ctx context.Context, arg ClaimAttachmentJob
|
||||
return i, err
|
||||
}
|
||||
|
||||
const clearAttachmentStagingKey = `-- name: ClearAttachmentStagingKey :exec
|
||||
UPDATE attachments
|
||||
SET staging_key = NULL
|
||||
WHERE id = $1::uuid
|
||||
`
|
||||
|
||||
func (q *Queries) ClearAttachmentStagingKey(ctx context.Context, id uuid.UUID) error {
|
||||
_, err := q.db.Exec(ctx, clearAttachmentStagingKey, id)
|
||||
return err
|
||||
}
|
||||
|
||||
const completeAttachmentJob = `-- name: CompleteAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'ready',
|
||||
@@ -320,15 +331,17 @@ func (q *Queries) ListAttachmentsByIDs(ctx context.Context, arg ListAttachmentsB
|
||||
}
|
||||
|
||||
const listAttachmentsForReconcile = `-- name: ListAttachmentsForReconcile :many
|
||||
SELECT id, s3_key, status, size_bytes
|
||||
SELECT id, s3_key, staging_key, status, size_bytes, updated_at
|
||||
FROM attachments
|
||||
`
|
||||
|
||||
type ListAttachmentsForReconcileRow struct {
|
||||
ID uuid.UUID
|
||||
S3Key string
|
||||
Status AttachmentStatus
|
||||
SizeBytes int
|
||||
ID uuid.UUID
|
||||
S3Key string
|
||||
StagingKey *string
|
||||
Status AttachmentStatus
|
||||
SizeBytes int
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttachmentsForReconcileRow, error) {
|
||||
@@ -343,8 +356,10 @@ func (q *Queries) ListAttachmentsForReconcile(ctx context.Context) ([]ListAttach
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.S3Key,
|
||||
&i.StagingKey,
|
||||
&i.Status,
|
||||
&i.SizeBytes,
|
||||
&i.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -415,6 +430,39 @@ func (q *Queries) ListMessageAttachmentsByMessageIDs(ctx context.Context, messag
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockAttachmentByStagingKey = `-- name: LockAttachmentByStagingKey :one
|
||||
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments
|
||||
WHERE staging_key = $1::text
|
||||
FOR UPDATE
|
||||
`
|
||||
|
||||
func (q *Queries) LockAttachmentByStagingKey(ctx context.Context, stagingKey string) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, lockAttachmentByStagingKey, stagingKey)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.S3Key,
|
||||
&i.Filename,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.StagingKey,
|
||||
&i.JobID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempts,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.LeaseUntil,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const lockAttachmentJob = `-- name: LockAttachmentJob :one
|
||||
SELECT id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until FROM attachments
|
||||
WHERE id = $1::uuid
|
||||
@@ -535,40 +583,6 @@ func (q *Queries) MarkAttachmentFailedIfReady(ctx context.Context, arg MarkAttac
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const markAttachmentReady = `-- name: MarkAttachmentReady :one
|
||||
UPDATE attachments
|
||||
SET status = 'ready', updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
RETURNING id, chat_id, owner_id, s3_key, filename, content_type, size_bytes, sha256, status, error_code, created_at, updated_at, staging_key, job_id, pipeline_version, attempts, width, height, lease_until
|
||||
`
|
||||
|
||||
func (q *Queries) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (Attachment, error) {
|
||||
row := q.db.QueryRow(ctx, markAttachmentReady, id)
|
||||
var i Attachment
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.ChatID,
|
||||
&i.OwnerID,
|
||||
&i.S3Key,
|
||||
&i.Filename,
|
||||
&i.ContentType,
|
||||
&i.SizeBytes,
|
||||
&i.Sha256,
|
||||
&i.Status,
|
||||
&i.ErrorCode,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.StagingKey,
|
||||
&i.JobID,
|
||||
&i.PipelineVersion,
|
||||
&i.Attempts,
|
||||
&i.Width,
|
||||
&i.Height,
|
||||
&i.LeaseUntil,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const recoverExpiredAttachmentJob = `-- name: RecoverExpiredAttachmentJob :execrows
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
|
||||
@@ -8,28 +8,34 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
)
|
||||
|
||||
const (
|
||||
Prefix = "chats/"
|
||||
DefaultGrace = time.Hour
|
||||
DefaultMaxDeletes = 100
|
||||
ErrorCodeObjectMissing = "object_missing"
|
||||
Prefix = "chats/"
|
||||
StagingPrefix = "staging/"
|
||||
DefaultStagingRetention = 24 * time.Hour
|
||||
DefaultGrace = time.Hour
|
||||
DefaultMaxDeletes = 100
|
||||
ErrorCodeObjectMissing = "object_missing"
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
DryRun bool
|
||||
Grace time.Duration
|
||||
MaxDeletes int
|
||||
DryRun bool
|
||||
Grace time.Duration
|
||||
MaxDeletes int
|
||||
StagingRetention time.Duration
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
ScannedObjects int `json:"scanned_objects"`
|
||||
ScannedRows int `json:"scanned_rows"`
|
||||
ListPages int `json:"list_pages"`
|
||||
StagingObjects []string `json:"staging_objects"`
|
||||
OrphanObjects []string `json:"orphan_objects"`
|
||||
MissingAttachments []uuid.UUID `json:"missing_attachments"`
|
||||
CorruptObjects []string `json:"corrupt_objects"`
|
||||
@@ -47,32 +53,50 @@ type ObjectClient interface {
|
||||
DeleteAdmin(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
type txBeginner interface {
|
||||
Begin(ctx context.Context) (pgx.Tx, error)
|
||||
}
|
||||
|
||||
type AttachmentQueries interface {
|
||||
LockAttachmentByStagingKey(ctx context.Context, stagingKey string) (db.Attachment, error)
|
||||
ClearAttachmentStagingKey(ctx context.Context, id uuid.UUID) error
|
||||
ListAttachmentsForReconcile(ctx context.Context) ([]db.ListAttachmentsForReconcileRow, error)
|
||||
MarkAttachmentFailedIfReady(ctx context.Context, arg db.MarkAttachmentFailedIfReadyParams) (int64, error)
|
||||
}
|
||||
|
||||
type Reconciler struct {
|
||||
q AttachmentQueries
|
||||
store ObjectClient
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
pool txBeginner
|
||||
q AttachmentQueries
|
||||
withTx func(pgx.Tx) AttachmentQueries
|
||||
store ObjectClient
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type plan struct {
|
||||
orphans []s3store.ObjectInfo
|
||||
staging []s3store.ObjectInfo
|
||||
missing []db.ListAttachmentsForReconcileRow
|
||||
corrupt []s3store.ObjectInfo
|
||||
skippedFresh int
|
||||
skippedInactive int
|
||||
}
|
||||
|
||||
func New(store ObjectClient, queries AttachmentQueries, log *slog.Logger) *Reconciler {
|
||||
func New(
|
||||
pool *pgxpool.Pool,
|
||||
store ObjectClient,
|
||||
queries *db.Queries,
|
||||
log *slog.Logger,
|
||||
) *Reconciler {
|
||||
if log == nil {
|
||||
log = slog.Default()
|
||||
}
|
||||
|
||||
return &Reconciler{
|
||||
pool: pool,
|
||||
withTx: func(tx pgx.Tx) AttachmentQueries {
|
||||
return queries.WithTx(tx)
|
||||
},
|
||||
store: store,
|
||||
q: queries,
|
||||
log: log,
|
||||
@@ -86,6 +110,7 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
|
||||
|
||||
report := Report{
|
||||
OrphanObjects: []string{},
|
||||
StagingObjects: []string{},
|
||||
MissingAttachments: []uuid.UUID{},
|
||||
CorruptObjects: []string{},
|
||||
DryRun: opts.DryRun,
|
||||
@@ -97,20 +122,28 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
|
||||
return report, err
|
||||
}
|
||||
|
||||
staging, err := r.store.ListAll(ctx, StagingPrefix)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
rows, err := r.q.ListAttachmentsForReconcile(ctx)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
report.ScannedObjects = len(listed.Objects)
|
||||
report.ScannedObjects = len(listed.Objects) + len(staging.Objects)
|
||||
report.ScannedRows = len(rows)
|
||||
report.ListPages = listed.Pages
|
||||
report.ListPages = listed.Pages + staging.Pages
|
||||
|
||||
p := buildPlan(listed.Objects, rows, started, opts.Grace)
|
||||
p := buildPlan(listed.Objects, staging.Objects, rows, started, opts)
|
||||
|
||||
for _, obj := range p.orphans {
|
||||
report.OrphanObjects = append(report.OrphanObjects, obj.Key)
|
||||
}
|
||||
for _, obj := range p.staging {
|
||||
report.StagingObjects = append(report.StagingObjects, obj.Key)
|
||||
}
|
||||
for _, obj := range p.corrupt {
|
||||
report.CorruptObjects = append(report.CorruptObjects, obj.Key)
|
||||
}
|
||||
@@ -134,6 +167,7 @@ func (r *Reconciler) Run(ctx context.Context, opts Options) (Report, error) {
|
||||
"scanned_rows", report.ScannedRows,
|
||||
"list_pages", report.ListPages,
|
||||
"orphan_objects", len(report.OrphanObjects),
|
||||
"staging_objects", len(report.StagingObjects),
|
||||
"missing_attachments", len(report.MissingAttachments),
|
||||
"corrupt_objects", len(report.CorruptObjects),
|
||||
"deleted_objects", report.DeletedObjects,
|
||||
@@ -152,14 +186,19 @@ func (o Options) withDefaults() Options {
|
||||
o.MaxDeletes = DefaultMaxDeletes
|
||||
}
|
||||
|
||||
if o.StagingRetention <= 0 {
|
||||
o.StagingRetention = DefaultStagingRetention
|
||||
}
|
||||
|
||||
return o
|
||||
}
|
||||
|
||||
func buildPlan(
|
||||
objects []s3store.ObjectInfo,
|
||||
staging []s3store.ObjectInfo,
|
||||
rows []db.ListAttachmentsForReconcileRow,
|
||||
now time.Time,
|
||||
grace time.Duration,
|
||||
opts Options,
|
||||
) plan {
|
||||
var p plan
|
||||
|
||||
@@ -168,7 +207,12 @@ func buildPlan(
|
||||
byKey[obj.Key] = obj
|
||||
}
|
||||
|
||||
byStagingKey := make(map[string]db.ListAttachmentsForReconcileRow, len(rows))
|
||||
for _, row := range rows {
|
||||
if row.StagingKey != nil {
|
||||
byStagingKey[*row.StagingKey] = row
|
||||
}
|
||||
|
||||
obj, exists := byKey[row.S3Key]
|
||||
if exists {
|
||||
delete(byKey, row.S3Key)
|
||||
@@ -193,13 +237,35 @@ func buildPlan(
|
||||
}
|
||||
|
||||
for _, obj := range byKey {
|
||||
if now.Sub(obj.LastModified) < grace {
|
||||
if now.Sub(obj.LastModified) < opts.Grace {
|
||||
p.skippedFresh++
|
||||
continue
|
||||
}
|
||||
p.orphans = append(p.orphans, obj)
|
||||
}
|
||||
|
||||
for _, obj := range staging {
|
||||
retainedSince := obj.LastModified
|
||||
if row, exists := byStagingKey[obj.Key]; exists {
|
||||
if row.Status != db.AttachmentStatusReady && row.Status != db.AttachmentStatusFailed {
|
||||
p.skippedInactive++
|
||||
continue
|
||||
}
|
||||
|
||||
retainedSince = row.UpdatedAt
|
||||
}
|
||||
|
||||
if now.Sub(retainedSince) < opts.StagingRetention {
|
||||
p.skippedFresh++
|
||||
continue
|
||||
}
|
||||
|
||||
p.staging = append(p.staging, obj)
|
||||
}
|
||||
|
||||
sort.Slice(p.staging, func(i, j int) bool {
|
||||
return p.staging[i].Key < p.staging[j].Key
|
||||
})
|
||||
sort.Slice(p.orphans, func(i, j int) bool {
|
||||
return p.orphans[i].Key < p.orphans[j].Key
|
||||
})
|
||||
@@ -243,4 +309,15 @@ func (r *Reconciler) apply(ctx context.Context, opts Options, p plan, report *Re
|
||||
|
||||
report.MarkedFailed += int(marked)
|
||||
}
|
||||
|
||||
for _, obj := range p.staging {
|
||||
if report.DeletedObjects >= opts.MaxDeletes {
|
||||
report.Errors = append(report.Errors, "max deletes reached")
|
||||
break
|
||||
}
|
||||
|
||||
if err := r.cleanupStagingObject(ctx, obj, opts, report); err != nil {
|
||||
report.Errors = append(report.Errors, fmt.Sprintf("cleanup staging %s: %v", obj.Key, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
)
|
||||
|
||||
func (r *Reconciler) cleanupStagingObject(
|
||||
ctx context.Context,
|
||||
obj s3store.ObjectInfo,
|
||||
opts Options,
|
||||
report *Report,
|
||||
) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := r.withTx(tx)
|
||||
row, err := q.LockAttachmentByStagingKey(ctx, obj.Key)
|
||||
orphan := errors.Is(err, pgx.ErrNoRows)
|
||||
if err != nil && !orphan {
|
||||
return err
|
||||
}
|
||||
|
||||
if !orphan {
|
||||
if row.Status != db.AttachmentStatusReady && row.Status != db.AttachmentStatusFailed {
|
||||
report.SkippedInactive++
|
||||
return nil
|
||||
}
|
||||
|
||||
if r.now().Sub(row.UpdatedAt) < opts.StagingRetention {
|
||||
report.SkippedFresh++
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := r.store.DeleteAdmin(ctx, obj.Key); err != nil {
|
||||
return err
|
||||
}
|
||||
report.DeletedObjects++
|
||||
|
||||
if !orphan {
|
||||
if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
@@ -238,6 +238,13 @@ func (w *Worker) handle(ctx context.Context, job model.AttachmentJob) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if row.StagingKey != nil {
|
||||
err := w.deleteStagingObject(ctx, *row.StagingKey)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
w.log.Warn("delete ready attachment staging", "attachment_id", row.ID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
w.log.Info("attachment ready", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
|
||||
return nil
|
||||
}
|
||||
@@ -391,6 +398,41 @@ func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cau
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) deleteStagingObject(ctx context.Context, key string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
row, err := q.LockAttachmentByStagingKey(ctx, key)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if row.Status != db.AttachmentStatusReady {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := w.s3.DeleteAdmin(ctx, key); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.ClearAttachmentStagingKey(ctx, row.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (w *Worker) recoverLoop(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
recovered, err := w.recoverOne(ctx)
|
||||
|
||||
Reference in New Issue
Block a user