245 lines
8.3 KiB
JSON
245 lines
8.3 KiB
JSON
{
|
|
"command": "faults",
|
|
"date_utc": "2026-10-29T10:31:36Z",
|
|
"commit": "ec540a1",
|
|
"result": {
|
|
"grace_used": "1m",
|
|
"faults": {
|
|
"F1_orphan_object_no_row": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69",
|
|
"F2_metadata_row_lost(U3)": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed",
|
|
"F3_object_lost_of_used(U2)": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad",
|
|
"F4_crash_after_put_before_ready": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a",
|
|
"F5_crash_before_put": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
|
"F6_F7_fresh_upload_in_progress": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/693c91e3-43f6-46f6-9bef-2b0de987546a",
|
|
"F8_fresh_orphan_object": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc"
|
|
},
|
|
"steps": [
|
|
{
|
|
"step": "dry-run",
|
|
"report": {
|
|
"mode": "dry-run",
|
|
"objects_scanned": 233,
|
|
"rows_scanned": 235,
|
|
"orphan_objects": 3,
|
|
"fresh_skipped": 1,
|
|
"stale_pending": 3,
|
|
"missing_objects": 1,
|
|
"size_mismatch": 0,
|
|
"recovered": 0,
|
|
"fixed": 0,
|
|
"errors": 0,
|
|
"total_ms": 34
|
|
},
|
|
"items": [
|
|
{
|
|
"kind": "stale_pending",
|
|
"key": "attachments/22b62cf4-b6c8-4cfb-bd41-0a248cef291e/34bf2792-f4dd-4fc8-b242-2f2ed7b49209",
|
|
"attachment_id": "34bf2792-f4dd-4fc8-b242-2f2ed7b49209",
|
|
"action": "would delete row and object"
|
|
},
|
|
{
|
|
"kind": "stale_pending",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a",
|
|
"attachment_id": "761f488e-6a97-417b-a96b-140f3f99660a",
|
|
"action": "would delete row and object"
|
|
},
|
|
{
|
|
"kind": "stale_pending",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
|
"attachment_id": "63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
|
"action": "would delete row and object"
|
|
},
|
|
{
|
|
"kind": "missing_object",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad",
|
|
"attachment_id": "421fa744-895f-4f52-96cf-ea62ffec57ad",
|
|
"action": "would mark missing (metadata kept)"
|
|
},
|
|
{
|
|
"kind": "orphan_object",
|
|
"key": "attachments/22b62cf4-b6c8-4cfb-bd41-0a248cef291e/5585fa4c-c149-49f6-b5d1-1196038eba43",
|
|
"action": "would delete object"
|
|
},
|
|
{
|
|
"kind": "orphan_object",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc",
|
|
"action": "skip: younger than grace"
|
|
},
|
|
{
|
|
"kind": "orphan_object",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed",
|
|
"action": "would delete object"
|
|
},
|
|
{
|
|
"kind": "orphan_object",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69",
|
|
"action": "would delete object"
|
|
}
|
|
],
|
|
"state_before": {
|
|
"s3_objects": 233,
|
|
"db_rows": 235,
|
|
"db_pending": 4,
|
|
"db_missing_marked": 4
|
|
},
|
|
"state_after": {
|
|
"s3_objects": 233,
|
|
"db_rows": 235,
|
|
"db_pending": 4,
|
|
"db_missing_marked": 4
|
|
},
|
|
"dry_run_changed_nothing": true,
|
|
"this_experiment": {
|
|
"stale_pending": 2,
|
|
"missing_object": 1,
|
|
"orphan_object/skip": 1,
|
|
"orphan_object": 2
|
|
},
|
|
"other_findings_outside_experiment": {
|
|
"stale_pending": 1,
|
|
"orphan_object": 1
|
|
}
|
|
},
|
|
{
|
|
"step": "apply #1",
|
|
"report": {
|
|
"mode": "apply",
|
|
"objects_scanned": 233,
|
|
"rows_scanned": 235,
|
|
"orphan_objects": 3,
|
|
"fresh_skipped": 1,
|
|
"stale_pending": 3,
|
|
"missing_objects": 1,
|
|
"size_mismatch": 0,
|
|
"recovered": 0,
|
|
"fixed": 7,
|
|
"errors": 0,
|
|
"total_ms": 46
|
|
},
|
|
"this_experiment": {
|
|
"stale_pending": 2,
|
|
"missing_object": 1,
|
|
"orphan_object/skip": 1,
|
|
"orphan_object": 2
|
|
},
|
|
"items": [
|
|
{
|
|
"kind": "stale_pending",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a",
|
|
"attachment_id": "761f488e-6a97-417b-a96b-140f3f99660a",
|
|
"action": "deleted row and object"
|
|
},
|
|
{
|
|
"kind": "stale_pending",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
|
"attachment_id": "63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
|
"action": "deleted row and object"
|
|
},
|
|
{
|
|
"kind": "missing_object",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad",
|
|
"attachment_id": "421fa744-895f-4f52-96cf-ea62ffec57ad",
|
|
"action": "marked missing (metadata kept)"
|
|
},
|
|
{
|
|
"kind": "orphan_object",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc",
|
|
"action": "skip: younger than grace"
|
|
},
|
|
{
|
|
"kind": "orphan_object",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed",
|
|
"action": "deleted object"
|
|
},
|
|
{
|
|
"kind": "orphan_object",
|
|
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69",
|
|
"action": "deleted object"
|
|
}
|
|
],
|
|
"state_after": {
|
|
"s3_objects": 228,
|
|
"db_rows": 232,
|
|
"db_pending": 1,
|
|
"db_missing_marked": 5
|
|
}
|
|
},
|
|
{
|
|
"step": "apply #2 (repeat)",
|
|
"report": {
|
|
"mode": "apply",
|
|
"objects_scanned": 228,
|
|
"rows_scanned": 232,
|
|
"orphan_objects": 0,
|
|
"fresh_skipped": 1,
|
|
"stale_pending": 0,
|
|
"missing_objects": 0,
|
|
"size_mismatch": 0,
|
|
"recovered": 0,
|
|
"fixed": 0,
|
|
"errors": 0,
|
|
"total_ms": 25
|
|
},
|
|
"this_experiment": {
|
|
"orphan_object/skip": 1
|
|
},
|
|
"state_after": {
|
|
"s3_objects": 228,
|
|
"db_rows": 232,
|
|
"db_pending": 1,
|
|
"db_missing_marked": 5
|
|
}
|
|
},
|
|
{
|
|
"step": "apply after restoring U2 object",
|
|
"report": {
|
|
"mode": "apply",
|
|
"objects_scanned": 229,
|
|
"rows_scanned": 232,
|
|
"orphan_objects": 0,
|
|
"fresh_skipped": 1,
|
|
"stale_pending": 0,
|
|
"missing_objects": 0,
|
|
"size_mismatch": 0,
|
|
"recovered": 1,
|
|
"fixed": 1,
|
|
"errors": 0,
|
|
"total_ms": 27
|
|
}
|
|
},
|
|
{
|
|
"step": "dry-run with production grace 15m",
|
|
"report": {
|
|
"mode": "dry-run",
|
|
"objects_scanned": 229,
|
|
"rows_scanned": 232,
|
|
"orphan_objects": 0,
|
|
"fresh_skipped": 1,
|
|
"stale_pending": 0,
|
|
"missing_objects": 0,
|
|
"size_mismatch": 0,
|
|
"recovered": 0,
|
|
"fixed": 0,
|
|
"errors": 0,
|
|
"total_ms": 31
|
|
}
|
|
}
|
|
],
|
|
"verify": {
|
|
"U1_used_healthy_download_sha_ok": true,
|
|
"U2_metadata_kept": 200,
|
|
"U2_content_status": 404,
|
|
"U2_marked_missing": true,
|
|
"message_still_references_U1_U2": true,
|
|
"F1_orphan_deleted": true,
|
|
"F2_orphan_of_lost_row_deleted": true,
|
|
"F4_row_and_object_deleted": true,
|
|
"F5_row_deleted": true,
|
|
"F7_fresh_pending_row_kept": true,
|
|
"F7_fresh_object_kept": true,
|
|
"F8_fresh_orphan_kept": true,
|
|
"U2_restored_and_unmarked": true
|
|
}
|
|
}
|
|
}
|