2026-11-20T04:41:27+03:00
failure between S3 write and database commit: an object must not stay unaccounted

$ scripts/reconcile.sh   # baseline
{
  "prefix": "chats/",
  "grace_seconds": 3600,
  "dry_run": true,
  "scanned_objects": 17,
  "scanned_rows": 17,
  "truncated": false,
  "orphans": [],
  "missing": [],
  "deleted_objects": 0,
  "deleted_metadata": 0,
  "retained_missing": 0,
  "delete_failures": 0,
  "s3_requests": 1,
  "db_queries": 2,
  "duration_ms": 7,
  "objects_per_second": 2428.5714285714284
}

$ podman exec -i postgres psql < scripts/sql/inject-db-failure.sql
CREATE FUNCTION
NOTICE:  trigger "inject_attachment_insert_failure" for relation "attachments" does not exist, skipping
DROP TRIGGER
CREATE TRIGGER

$ scripts/upload-once.py --expect 503   # insert fails after a successful S3 PUT
status=503

$ scripts/reconcile.sh   # same object count: compensating DELETE removed the object
{
  "prefix": "chats/",
  "grace_seconds": 3600,
  "dry_run": true,
  "scanned_objects": 17,
  "scanned_rows": 17,
  "truncated": false,
  "orphans": [],
  "missing": [],
  "deleted_objects": 0,
  "deleted_metadata": 0,
  "retained_missing": 0,
  "delete_failures": 0,
  "s3_requests": 1,
  "db_queries": 2,
  "duration_ms": 7,
  "objects_per_second": 2428.5714285714284
}

$ podman exec -i postgres psql < scripts/sql/reset-db-failure.sql
DROP TRIGGER
DROP FUNCTION

$ scripts/upload-once.py --expect 201   # service healthy again
status=201
attachment=7dd89013-32f1-4732-a2fd-529fe2081bb8 size=653

$ scripts/reconcile.sh   # one more accounted object
{
  "prefix": "chats/",
  "grace_seconds": 3600,
  "dry_run": true,
  "scanned_objects": 18,
  "scanned_rows": 18,
  "truncated": false,
  "orphans": [],
  "missing": [],
  "deleted_objects": 0,
  "deleted_metadata": 0,
  "retained_missing": 0,
  "delete_failures": 0,
  "s3_requests": 1,
  "db_queries": 2,
  "duration_ms": 7,
  "objects_per_second": 2571.4285714285716
}
