forked from mtusi2026/mtusi2026lab5
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> collect_evidence на новом clone (новые secrets, пустые volumes): make check/validate/test, test-persistence, test-failover (обе реплики), сценарий защиты, свои тесты 377 passed, матрица трека 2 12/12 без нарушений, аудит ok (522 объекта, SHA-256). Исправлен порядок шагов сценария «reorder версий» (не запускать сервис с обычной конфигурацией между шагами).
277 lines
5.5 KiB
JSON
277 lines
5.5 KiB
JSON
{
|
|
"experiment": "track 2: idempotency of the versioned pipeline under injected faults",
|
|
"started": "2026-10-31T14:48:37Z",
|
|
"finished": "2026-10-31T14:57:10Z",
|
|
"host": {
|
|
"os": "Windows-10-10.0.19045-SP0",
|
|
"python": "3.14.6"
|
|
},
|
|
"uploads_per_plain_scenario": 5,
|
|
"scenarios": [
|
|
{
|
|
"scenario": "baseline_no_faults",
|
|
"faults": {
|
|
"worker": {}
|
|
},
|
|
"uploads": 5,
|
|
"ready": 5,
|
|
"claims_total": 5,
|
|
"max_claims": 1,
|
|
"time_to_ready_s": {
|
|
"p50": 2.21,
|
|
"max": 3.15
|
|
},
|
|
"event_sequences": [
|
|
"created > claimed > succeeded"
|
|
],
|
|
"problems": {}
|
|
},
|
|
{
|
|
"scenario": "duplicate_publish",
|
|
"faults": {
|
|
"relay": {
|
|
"RELAY_FAULT_POINTS": "duplicate_publish"
|
|
}
|
|
},
|
|
"uploads": 5,
|
|
"ready": 5,
|
|
"claims_total": 5,
|
|
"max_claims": 1,
|
|
"time_to_ready_s": {
|
|
"p50": 1.89,
|
|
"max": 2.52
|
|
},
|
|
"event_sequences": [
|
|
"created > claimed > succeeded"
|
|
],
|
|
"problems": {}
|
|
},
|
|
{
|
|
"scenario": "duplicate_and_reverse_order",
|
|
"faults": {
|
|
"relay": {
|
|
"RELAY_FAULT_POINTS": "duplicate_publish,reverse_order"
|
|
}
|
|
},
|
|
"uploads": 5,
|
|
"ready": 5,
|
|
"claims_total": 5,
|
|
"max_claims": 1,
|
|
"time_to_ready_s": {
|
|
"p50": 1.89,
|
|
"max": 2.52
|
|
},
|
|
"event_sequences": [
|
|
"created > claimed > succeeded"
|
|
],
|
|
"problems": {}
|
|
},
|
|
{
|
|
"scenario": "relay_crash_after_confirm_before_mark",
|
|
"faults": {
|
|
"relay": {
|
|
"RELAY_FAULT_POINTS": "crash_after_publish"
|
|
}
|
|
},
|
|
"uploads": 5,
|
|
"ready": 5,
|
|
"claims_total": 5,
|
|
"max_claims": 1,
|
|
"time_to_ready_s": {
|
|
"p50": 1.89,
|
|
"max": 2.83
|
|
},
|
|
"event_sequences": [
|
|
"created > claimed > fault_injected > succeeded",
|
|
"created > claimed > succeeded"
|
|
],
|
|
"problems": {}
|
|
},
|
|
{
|
|
"scenario": "worker_crash_before_side_effect",
|
|
"faults": {
|
|
"worker": {
|
|
"WORKER_FAULT_POINTS": "crash_after_claim",
|
|
"JOB_LEASE_SECONDS": "6",
|
|
"ORPHAN_GRACE_SECONDS": "3",
|
|
"CLEANUP_INTERVAL_SECONDS": "1"
|
|
}
|
|
},
|
|
"uploads": 5,
|
|
"ready": 5,
|
|
"claims_total": 14,
|
|
"max_claims": 3,
|
|
"time_to_ready_s": {
|
|
"p50": 21.66,
|
|
"max": 21.68
|
|
},
|
|
"event_sequences": [
|
|
"created > claimed > fault_injected > lease_expired > claimed > lease_expired > claimed > succeeded",
|
|
"created > claimed > fault_injected > lease_expired > claimed > succeeded",
|
|
"created > claimed > lease_expired > claimed > fault_injected > lease_expired > claimed > succeeded"
|
|
],
|
|
"problems": {}
|
|
},
|
|
{
|
|
"scenario": "worker_crash_after_s3_before_commit",
|
|
"faults": {
|
|
"worker": {
|
|
"WORKER_FAULT_POINTS": "crash_after_s3_write",
|
|
"JOB_LEASE_SECONDS": "6",
|
|
"ORPHAN_GRACE_SECONDS": "3",
|
|
"CLEANUP_INTERVAL_SECONDS": "1"
|
|
}
|
|
},
|
|
"uploads": 5,
|
|
"ready": 5,
|
|
"claims_total": 10,
|
|
"max_claims": 2,
|
|
"time_to_ready_s": {
|
|
"p50": 15.49,
|
|
"max": 16.13
|
|
},
|
|
"event_sequences": [
|
|
"created > claimed > fault_injected > lease_expired > claimed > succeeded"
|
|
],
|
|
"problems": {}
|
|
},
|
|
{
|
|
"scenario": "worker_crash_after_commit_before_ack",
|
|
"faults": {
|
|
"worker": {
|
|
"WORKER_FAULT_POINTS": "crash_after_commit",
|
|
"JOB_LEASE_SECONDS": "6",
|
|
"ORPHAN_GRACE_SECONDS": "3",
|
|
"CLEANUP_INTERVAL_SECONDS": "1"
|
|
}
|
|
},
|
|
"uploads": 5,
|
|
"ready": 5,
|
|
"claims_total": 5,
|
|
"max_claims": 1,
|
|
"time_to_ready_s": {
|
|
"p50": 5.3,
|
|
"max": 8.4
|
|
},
|
|
"event_sequences": [
|
|
"created > claimed > succeeded > fault_injected"
|
|
],
|
|
"problems": {}
|
|
},
|
|
{
|
|
"scenario": "transient_errors_then_success",
|
|
"faults": {
|
|
"worker": {
|
|
"WORKER_FAULT_POINTS": "transient_error=2",
|
|
"JOB_RETRY_BASE_SECONDS": "0.5",
|
|
"JOB_LEASE_SECONDS": "6",
|
|
"ORPHAN_GRACE_SECONDS": "3",
|
|
"CLEANUP_INTERVAL_SECONDS": "1"
|
|
}
|
|
},
|
|
"uploads": 5,
|
|
"ready": 5,
|
|
"claims_total": 15,
|
|
"max_claims": 3,
|
|
"time_to_ready_s": {
|
|
"p50": 4.11,
|
|
"max": 4.13
|
|
},
|
|
"event_sequences": [
|
|
"created > claimed > retry_scheduled > claimed > retry_scheduled > claimed > succeeded"
|
|
],
|
|
"problems": {}
|
|
},
|
|
{
|
|
"scenario": "two_workers_duplicate_publish",
|
|
"faults": {
|
|
"worker": {}
|
|
},
|
|
"uploads": 5,
|
|
"ready": 5,
|
|
"claims_total": 5,
|
|
"max_claims": 1,
|
|
"time_to_ready_s": {
|
|
"p50": 1.58,
|
|
"max": 2.21
|
|
},
|
|
"event_sequences": [
|
|
"created > claimed > succeeded"
|
|
],
|
|
"problems": {}
|
|
},
|
|
{
|
|
"scenario": "stale_version_and_reorder",
|
|
"uploads": 1,
|
|
"statuses_seen_during_reprocess": [
|
|
"ready"
|
|
],
|
|
"job_states": [
|
|
"succeeded:v1",
|
|
"superseded:v2",
|
|
"succeeded:v3"
|
|
],
|
|
"published_version": 3,
|
|
"variant_rows_by_version": 6,
|
|
"objects": 10,
|
|
"events": [
|
|
"created",
|
|
"claimed",
|
|
"succeeded",
|
|
"created",
|
|
"claimed",
|
|
"created",
|
|
"claimed",
|
|
"succeeded",
|
|
"superseded"
|
|
],
|
|
"problems": []
|
|
},
|
|
{
|
|
"scenario": "reordered_versions_in_queue",
|
|
"job_states": [
|
|
"succeeded:v1",
|
|
"superseded:v2",
|
|
"succeeded:v3"
|
|
],
|
|
"claims": 2,
|
|
"published_version": 3,
|
|
"events": [
|
|
"created",
|
|
"claimed",
|
|
"succeeded",
|
|
"created",
|
|
"created",
|
|
"claimed",
|
|
"succeeded",
|
|
"superseded"
|
|
],
|
|
"problems": []
|
|
},
|
|
{
|
|
"scenario": "hung_worker_docker_pause",
|
|
"frozen": "bcd68c4d813f",
|
|
"claims": 2,
|
|
"events": [
|
|
"created",
|
|
"claimed",
|
|
"lease_expired",
|
|
"claimed",
|
|
"succeeded"
|
|
],
|
|
"problems": []
|
|
}
|
|
],
|
|
"audit_after_all": {
|
|
"ok": true,
|
|
"violations": [],
|
|
"notes": 0,
|
|
"objects": {
|
|
"final": 523,
|
|
"staging": 2,
|
|
"published": 522,
|
|
"checksums_verified": true
|
|
}
|
|
},
|
|
"violations": 0
|
|
} |