diff --git a/evidence/failover/baseline/meta.json b/evidence/failover/baseline/meta.json new file mode 100644 index 00000000..fa289b0a --- /dev/null +++ b/evidence/failover/baseline/meta.json @@ -0,0 +1,207 @@ +{ + "kind": "failover", + "label": "baseline", + "started": "2026-10-15T10:55:52Z", + "profile": { + "name": "failover", + "limits": { + "api_replicas": 2, + "api_cpus": 1.0, + "api_memory": "512m", + "db_cpus": 2.0, + "db_memory": "1g", + "db_max_connections": 100, + "cache_cpus": 0.5, + "cache_memory": "128m", + "lb_cpus": 1.0, + "lb_memory": "256m", + "gen_cpus": 2.0, + "gen_memory": "1g", + "sampler_cpus": 0.25, + "sampler_memory": "256m" + }, + "users": 4, + "read_messages": 1000, + "write_rate": 100.0, + "read_rate": 100.0, + "gen_procs": 2, + "session_ttl": 21600, + "seed": 20261014, + "modes": [ + "stop", + "kill", + "pause" + ], + "targets": 2, + "before_s": 10.0, + "fail_s": 15.0, + "after_s": 10.0, + "restore_s": 10.0, + "rollout_before_s": 0.0, + "rollout_budget_s": 0.0, + "rollout_after_s": 0.0, + "bounds": [ + [ + "stop", + 2.5 + ], + [ + "kill", + 2.5 + ], + [ + "pause", + 6.5 + ] + ] + }, + "host": { + "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", + "machine": "arm64", + "cpu_model": "Apple M4", + "cpu_count": 10, + "ram_gb": 16.0, + "swap_used_mib": 0.0, + "power": "Battery Power", + "memory_pressure": null, + "memory_free_percent": 79, + "swapouts": 0, + "loadavg_1m": 1.34 + }, + "docker": { + "ServerVersion": "28.0.4", + "OperatingSystem": "Docker Desktop", + "KernelVersion": "6.10.14-linuxkit", + "Architecture": "aarch64", + "NCPU": 10, + "MemTotal": 8218034176, + "CgroupVersion": "2", + "CgroupDriver": "cgroupfs", + "compose_version": "2.34.0-desktop.1", + "other_running_containers": [] + }, + "budget": { + "limits_cpus": 7.75, + "limits_memory_gib": 3.62, + "vm_cpus": 10, + "vm_memory_gib": 7.65 + }, + "host_warnings": [ + "host on battery power: CPU may be throttled" + ], + "image": { + "tag": "mtusi-messenger-vladimirceva:bench-baseline", + "id": "sha256:ac17d94cda500d4a65418c184deec9428e706ff40a68a7789b068c973c7d4e05", + "created": "2026-10-15T10:39:12.700675009Z", + "src_sha256": "0349e9a2701784673391c0ba4bc2eedba7013098d6ff7f871261aab36f78354a", + "versions": { + "python": "3.13.15", + "fastapi": "0.142.1", + "starlette": "1.7.0", + "uvicorn": "0.54.0", + "pydantic": "2.13.5", + "sqlalchemy": "2.1.1", + "asyncpg": "0.31.0", + "alembic": "1.20.0", + "redis": "8.1.0", + "argon2-cffi": "25.1.0" + }, + "app_defaults": { + "pool_size": 10, + "pool_overflow": 5, + "pool_timeout": 5.0, + "operation_timeout": 6.0, + "statement_timeout_ms": 5000, + "kdf_concurrency": 4, + "keep_alive": 60, + "shutdown_timeout": 5 + } + }, + "code": { + "src_sha256": "0349e9a2701784673391c0ba4bc2eedba7013098d6ff7f871261aab36f78354a", + "git_commit": "006ec24d208670f6405bc1bd02f2aafe207e2b30", + "git_dirty": false + }, + "setup": { + "users": 4, + "read_chat": "691d7f26-cb3d-4fbf-8037-95c2b0ff2f98", + "read_messages": 1000, + "cursors": 20, + "fill_s": 0.94, + "fill_rps": 1069.0 + }, + "stand": { + "postgres": { + "image": "postgres:18.6-trixie@sha256:5a5a84b19854a9ffaa54082c166ff4ec27473a361e496e5ea167f298f2da9722", + "version": "18.6 (Debian 18.6-1.pgdg13+2)", + "settings": { + "checkpoint_timeout": "5min", + "fsync": "on", + "jit": "on", + "max_connections": "100", + "max_wal_size": "1GB", + "password_encryption": "scram-sha-256", + "shared_buffers": "128MB", + "synchronous_commit": "on", + "wal_level": "replica" + } + }, + "valkey": { + "image": "valkey/valkey:9.1.2-trixie@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd", + "version": "9.1.2" + }, + "haproxy": { + "image": "haproxy:3.4.6-trixie@sha256:e273915691a8eab3f105bf3e39fd4c29a4ecf334e33421903b5589d25e14db26" + }, + "limits": { + "api": { + "nano_cpus": 1000000000, + "memory": 536870912, + "memory_swap": 536870912, + "published_ports": {} + }, + "postgres": { + "nano_cpus": 2000000000, + "memory": 1073741824, + "memory_swap": 1073741824, + "published_ports": {} + }, + "valkey": { + "nano_cpus": 500000000, + "memory": 134217728, + "memory_swap": 134217728, + "published_ports": {} + }, + "haproxy": { + "nano_cpus": 1000000000, + "memory": 268435456, + "memory_swap": 268435456, + "published_ports": {} + } + }, + "api_replicas": 2, + "app_env": { + "APP_HOST": "0.0.0.0", + "APP_PORT": "8080", + "CACHE_HOST": "valkey", + "CACHE_PORT": "6379", + "DB_HOST": "postgres", + "DB_NAME": "bench", + "DB_PORT": "5432", + "DB_USER": "bench", + "IDEMPOTENCY_TTL_SECONDS": "86400", + "LOG_LEVEL": "INFO", + "SESSION_TTL_SECONDS": "21600", + "SHUTDOWN_TIMEOUT": "5" + } + }, + "sampler_ticks": 281, + "finished": "2026-10-15T11:00:43Z", + "host_state_end": { + "slept_seconds": 0.0, + "swapouts_start": 0, + "swapouts_end": 0, + "power_end": "Battery Power", + "swapouts_delta": 0 + } +} diff --git a/evidence/failover/baseline/phases.csv b/evidence/failover/baseline/phases.csv new file mode 100644 index 00000000..b0cfb9d0 --- /dev/null +++ b/evidence/failover/baseline/phases.csv @@ -0,0 +1,7 @@ +phase,mode,replica,target,slot,action_at_ms,restore_at_ms,detection_s,bound_s,over_bound,lb_return_s,errors_before,errors_fail,errors_after,error_kinds,first_error_s,last_error_s,recovery_s,ok_rps_before,ok_rps_fail,ok_rps_after,p95_before_ms,p95_fail_ms,p95_after_ms,max_fail_ms,target_after_down,writes,retried,recovered,rows,distinct_texts,verified,lb_errors +p1-stop-r1,stop,r1,cce5762ff431,api-2,1792061771104.8,1792061786580.0,1.79,2.5,False,2.03,0,1,0,IncompleteReadError:1,0.09,0.09,1.0,200.0,199.9,199.9,6.5,13.3,6.9,512.6,0,4500,0,0,4500,4500,True, +p2-kill-r1,kill,r1,cce5762ff431,api-2,1792061817739.5,1792061832912.1,1.21,2.5,False,1.8,1,0,0,IncompleteReadError:1,-0.0,-0.0,0.0,199.8,200.0,199.9,6.4,9.0,6.7,536.1,0,4500,1,1,4500,4500,True, +p3-pause-r1,pause,r1,cce5762ff431,api-2,1792061864401.4,1792061879442.8,5.89,6.5,False,1.04,1,584,0,504:585,-0.01,5.83,6.0,199.7,161.2,199.9,6.9,7.8,6.7,9998.4,0,4500,12,12,4500,4500,True,504:585 +p4-stop-r2,stop,r2,dad5d68a3f49,api-1,1792061911082.5,1792061926559.0,1.27,2.5,False,0.81,0,0,0,,,,0.0,200.2,199.9,200.0,6.6,8.6,6.6,528.0,0,4500,0,0,4500,4500,True, +p5-kill-r2,kill,r2,dad5d68a3f49,api-1,1792061957859.1,1792061973051.8,0.26,2.5,False,1.92,0,0,0,,,,0.0,199.8,200.1,199.9,6.5,7.5,6.6,507.9,0,4500,0,0,4500,4500,True, +p6-pause-r2,pause,r2,dad5d68a3f49,api-1,1792062004497.9,1792062019540.0,6.06,6.5,False,0.98,1,553,0,504:554,-0.01,5.52,6.0,199.9,163.2,199.9,6.8,7.6,6.9,10000.6,0,4500,548,548,4500,4500,True,504:554 diff --git a/evidence/failover/baseline/raw/p1-stop-r1-requests.csv.gz b/evidence/failover/baseline/raw/p1-stop-r1-requests.csv.gz new file mode 100644 index 00000000..2be43f10 Binary files /dev/null and b/evidence/failover/baseline/raw/p1-stop-r1-requests.csv.gz differ diff --git a/evidence/failover/baseline/raw/p2-kill-r1-requests.csv.gz b/evidence/failover/baseline/raw/p2-kill-r1-requests.csv.gz new file mode 100644 index 00000000..3a2d2f37 Binary files /dev/null and b/evidence/failover/baseline/raw/p2-kill-r1-requests.csv.gz differ diff --git a/evidence/failover/baseline/raw/p3-pause-r1-requests.csv.gz b/evidence/failover/baseline/raw/p3-pause-r1-requests.csv.gz new file mode 100644 index 00000000..871df453 Binary files /dev/null and b/evidence/failover/baseline/raw/p3-pause-r1-requests.csv.gz differ diff --git a/evidence/failover/baseline/raw/p4-stop-r2-requests.csv.gz b/evidence/failover/baseline/raw/p4-stop-r2-requests.csv.gz new file mode 100644 index 00000000..f32e38c2 Binary files /dev/null and b/evidence/failover/baseline/raw/p4-stop-r2-requests.csv.gz differ diff --git a/evidence/failover/baseline/raw/p5-kill-r2-requests.csv.gz b/evidence/failover/baseline/raw/p5-kill-r2-requests.csv.gz new file mode 100644 index 00000000..bf9d802c Binary files /dev/null and b/evidence/failover/baseline/raw/p5-kill-r2-requests.csv.gz differ diff --git a/evidence/failover/baseline/raw/p6-pause-r2-requests.csv.gz b/evidence/failover/baseline/raw/p6-pause-r2-requests.csv.gz new file mode 100644 index 00000000..caef1efe Binary files /dev/null and b/evidence/failover/baseline/raw/p6-pause-r2-requests.csv.gz differ diff --git a/evidence/failover/baseline/resources.csv b/evidence/failover/baseline/resources.csv new file mode 100644 index 00000000..492fc17e --- /dev/null +++ b/evidence/failover/baseline/resources.csv @@ -0,0 +1,19 @@ +phase,window,api_cpu_cores,api_cpu_max,api_throttled_max,api_memory_max_mib,db_cpu_cores,db_memory_mib,lb_cpu_cores,cache_cpu_cores,gen_cpu_cores,vm_busy_share,api_cpu_us_per_req,db_cpu_us_per_req +p1-stop-r1,before,0.47,0.243,0.0,70.1,0.085,143.3,0.015,0.006,0.028,0.0814,2350.0,425.0 +p1-stop-r1,fail,0.343,0.343,0.0,71.3,0.071,143.3,0.013,0.005,0.024,0.0563,1715.9,355.2 +p1-stop-r1,after,0.484,0.243,0.0,71.3,0.091,137.2,0.016,0.006,0.026,0.0726,2421.2,455.2 +p2-kill-r1,before,0.51,0.259,0.0,71.3,0.089,138.5,0.017,0.006,0.03,0.074,2552.6,445.4 +p2-kill-r1,fail,0.351,0.351,0.0,72.1,0.069,138.3,0.014,0.006,0.024,0.0597,1755.0,345.0 +p2-kill-r1,after,0.489,0.252,0.0,71.8,0.097,147.8,0.015,0.006,0.029,0.0737,2446.2,485.2 +p3-pause-r1,before,0.509,0.267,0.0,72.3,0.099,149.4,0.016,0.007,0.03,0.0784,2548.8,495.7 +p3-pause-r1,fail,0.326,0.306,0.0,86.7,0.065,162.2,0.017,0.006,0.035,0.0563,2022.3,403.2 +p3-pause-r1,after,0.522,0.275,0.0,94.1,0.104,162.5,0.017,0.007,0.029,0.0812,2611.3,520.3 +p4-stop-r2,before,0.496,0.255,0.0,94.1,0.115,163.5,0.016,0.006,0.029,0.0801,2477.5,574.4 +p4-stop-r2,fail,0.35,0.35,0.0,95.1,0.072,163.4,0.014,0.006,0.023,0.0578,1750.9,360.2 +p4-stop-r2,after,0.521,0.275,0.005,94.8,0.094,168.3,0.016,0.006,0.027,0.0768,2605.0,470.0 +p5-kill-r2,before,0.529,0.266,0.0,94.6,0.096,169.1,0.017,0.006,0.03,0.0758,2647.6,480.5 +p5-kill-r2,fail,0.344,0.344,0.0,95.1,0.085,169.1,0.013,0.006,0.023,0.0622,1719.1,424.8 +p5-kill-r2,after,0.488,0.251,0.0,94.9,0.092,175.9,0.016,0.006,0.027,0.0746,2441.2,460.2 +p6-pause-r2,before,0.502,0.257,0.0,94.6,0.094,177.0,0.016,0.006,0.031,0.0756,2511.3,470.2 +p6-pause-r2,fail,0.328,0.311,0.0,94.9,0.065,177.7,0.018,0.006,0.039,0.0555,2009.8,398.3 +p6-pause-r2,after,0.505,0.266,0.0051,94.7,0.116,193.4,0.017,0.006,0.029,0.0808,2526.3,580.3 diff --git a/evidence/failover/baseline/summary.md b/evidence/failover/baseline/summary.md new file mode 100644 index 00000000..bd69c10e --- /dev/null +++ b/evidence/failover/baseline/summary.md @@ -0,0 +1,45 @@ +# Отказ реплики под нагрузкой: baseline + +Код: src_sha256 `0349e9a270178467`, commit 006ec24d2086. Хост: Apple M4, 16.0 ГиБ; ВМ Docker 10 CPU, 7.7 ГиБ, Docker 28.0.4, compose 2.34.0-desktop.1. Начало 2026-10-15T10:55:52Z, конец 2026-10-15T11:00:43Z. + +Нагрузка: открытая модель, 100 записей с `Idempotency-Key` и 100 чтений страницы истории в секунду через HAProxy стенда; две реплики с лимитами рабочего стенда. Фаза: 10 с до действия, действие, 15 с отказа, возврат той же реплики, ещё 10 с. + +Отметки — часы ВМ Docker: действие и возврат — события Docker, исключение и возврат в LB — журнал HAProxy, запросы — генератор (момент по расписанию). Обнаружение — от действия до `DOWN` слота; восстановление — от действия до начала секунды, после которой до возврата нет ошибок и успешных ≥ 90 % поданной нагрузки. + +| Фаза | Режим | Реплика | Обнаружение, с (граница) | Возврат в LB, с | Ошибки до / отказ / после | Виды ошибок | Восстановление, с | Успешных/с до / отказ / после | p95, мс до / отказ / после | max в отказе, мс | Повтор: не прошло → прошло | Сверка | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| p1-stop-r1 | stop | r1 (cce5762ff431) | 1.79 (2.5) | 2.03 | 0 / 1 / 0 | IncompleteReadError:1 | 1 | 200.0 / 199.9 / 199.9 | 6.5 / 13.3 / 6.9 | 512.6 | 0 → 0 | ok: 4500 ключей = 4500 строк | +| p2-kill-r1 | kill | r1 (cce5762ff431) | 1.21 (2.5) | 1.8 | 1 / 0 / 0 | IncompleteReadError:1 | 0 | 199.8 / 200.0 / 199.9 | 6.4 / 9 / 6.7 | 536.1 | 1 → 1 | ok: 4500 ключей = 4500 строк | +| p3-pause-r1 | pause | r1 (cce5762ff431) | 5.89 (6.5) | 1.04 | 1 / 584 / 0 | 504:585 | 6 | 199.7 / 161.2 / 199.9 | 6.9 / 7.8 / 6.7 | 9998.4 | 12 → 12 | ok: 4500 ключей = 4500 строк | +| p4-stop-r2 | stop | r2 (dad5d68a3f49) | 1.27 (2.5) | 0.81 | 0 / 0 / 0 | — | 0 | 200.2 / 199.9 / 200.0 | 6.6 / 8.6 / 6.6 | 528 | 0 → 0 | ok: 4500 ключей = 4500 строк | +| p5-kill-r2 | kill | r2 (dad5d68a3f49) | 0.26 (2.5) | 1.92 | 0 / 0 / 0 | — | 0 | 199.8 / 200.1 / 199.9 | 6.5 / 7.5 / 6.6 | 507.9 | 0 → 0 | ok: 4500 ключей = 4500 строк | +| p6-pause-r2 | pause | r2 (dad5d68a3f49) | 6.06 (6.5) | 0.98 | 1 / 553 / 0 | 504:554 | 6 | 199.9 / 163.2 / 199.9 | 6.8 / 7.6 / 6.9 | 10000.6 | 548 → 548 | ok: 4500 ключей = 4500 строк | + +## CPU и память по окнам фазы + +Сэмплер (cgroup каждого контейнера раз в секунду). Окна: до действия (от первого запроса + 2 с), отказ (действие → возврат), после (возврат в LB → конец нагрузки). CPU реплик — сумма; на успешный запрос — только по окнам с нагрузкой. + +| Фаза | Окно | API, ядер (макс. реплика) | PostgreSQL | HAProxy | Valkey | Генератор | ВМ занята | API, мкс/запрос | PostgreSQL, мкс/запрос | +|---|---|---|---|---|---|---|---|---|---| +| p1-stop-r1 | before | 0.47 (0.243) | 0.085 | 0.015 | 0.006 | 0.028 | 0.0814 | 2350 | 425 | +| p1-stop-r1 | fail | 0.343 (0.343) | 0.071 | 0.013 | 0.005 | 0.024 | 0.0563 | 1715.9 | 355.2 | +| p1-stop-r1 | after | 0.484 (0.243) | 0.091 | 0.016 | 0.006 | 0.026 | 0.0726 | 2421.2 | 455.2 | +| p2-kill-r1 | before | 0.51 (0.259) | 0.089 | 0.017 | 0.006 | 0.03 | 0.074 | 2552.6 | 445.4 | +| p2-kill-r1 | fail | 0.351 (0.351) | 0.069 | 0.014 | 0.006 | 0.024 | 0.0597 | 1755 | 345 | +| p2-kill-r1 | after | 0.489 (0.252) | 0.097 | 0.015 | 0.006 | 0.029 | 0.0737 | 2446.2 | 485.2 | +| p3-pause-r1 | before | 0.509 (0.267) | 0.099 | 0.016 | 0.007 | 0.03 | 0.0784 | 2548.8 | 495.7 | +| p3-pause-r1 | fail | 0.326 (0.306) | 0.065 | 0.017 | 0.006 | 0.035 | 0.0563 | 2022.3 | 403.2 | +| p3-pause-r1 | after | 0.522 (0.275) | 0.104 | 0.017 | 0.007 | 0.029 | 0.0812 | 2611.3 | 520.3 | +| p4-stop-r2 | before | 0.496 (0.255) | 0.115 | 0.016 | 0.006 | 0.029 | 0.0801 | 2477.5 | 574.4 | +| p4-stop-r2 | fail | 0.35 (0.35) | 0.072 | 0.014 | 0.006 | 0.023 | 0.0578 | 1750.9 | 360.2 | +| p4-stop-r2 | after | 0.521 (0.275) | 0.094 | 0.016 | 0.006 | 0.027 | 0.0768 | 2605 | 470 | +| p5-kill-r2 | before | 0.529 (0.266) | 0.096 | 0.017 | 0.006 | 0.03 | 0.0758 | 2647.6 | 480.5 | +| p5-kill-r2 | fail | 0.344 (0.344) | 0.085 | 0.013 | 0.006 | 0.023 | 0.0622 | 1719.1 | 424.8 | +| p5-kill-r2 | after | 0.488 (0.251) | 0.092 | 0.016 | 0.006 | 0.027 | 0.0746 | 2441.2 | 460.2 | +| p6-pause-r2 | before | 0.502 (0.257) | 0.094 | 0.016 | 0.006 | 0.031 | 0.0756 | 2511.3 | 470.2 | +| p6-pause-r2 | fail | 0.328 (0.311) | 0.065 | 0.018 | 0.006 | 0.039 | 0.0555 | 2009.8 | 398.3 | +| p6-pause-r2 | after | 0.505 (0.266) | 0.116 | 0.017 | 0.006 | 0.029 | 0.0808 | 2526.3 | 580.3 | + +## Предупреждения о хосте + +- host on battery power: CPU may be throttled diff --git a/evidence/failover/baseline/timeline.csv b/evidence/failover/baseline/timeline.csv new file mode 100644 index 00000000..49df49bd --- /dev/null +++ b/evidence/failover/baseline/timeline.csv @@ -0,0 +1,277 @@ +phase,second,sent,ok_write,ok_read,ok_target,ok_other,ok_old,ok_new,errors,error_kinds,p95_ms,max_ms +p1-stop-r1,-10,194,97,97,97,97,,,0,,5.6,11.6 +p1-stop-r1,-9,200,100,100,100,100,,,0,,6.8,10.8 +p1-stop-r1,-8,200,100,100,100,100,,,0,,6.6,35.7 +p1-stop-r1,-7,200,100,100,100,100,,,0,,6.7,9.3 +p1-stop-r1,-6,200,100,100,100,100,,,0,,6.4,7.5 +p1-stop-r1,-5,200,100,100,100,100,,,0,,6.0,7.3 +p1-stop-r1,-4,200,100,100,100,100,,,0,,6.1,7.3 +p1-stop-r1,-3,200,100,100,100,100,,,0,,6.6,10.3 +p1-stop-r1,-2,200,100,100,100,100,,,0,,6.6,9.0 +p1-stop-r1,-1,200,100,100,100,100,,,0,,6.6,7.7 +p1-stop-r1,0,200,100,99,8,191,,,1,IncompleteReadError:1,506.1,512.6 +p1-stop-r1,1,200,100,100,0,200,,,0,,506.6,507.1 +p1-stop-r1,2,200,100,100,0,200,,,0,,7.4,8.5 +p1-stop-r1,3,200,100,100,0,200,,,0,,7.4,10.1 +p1-stop-r1,4,200,100,100,0,200,,,0,,8.5,10.8 +p1-stop-r1,5,200,100,100,0,200,,,0,,8.6,10.6 +p1-stop-r1,6,200,100,100,0,200,,,0,,7.3,9.1 +p1-stop-r1,7,200,100,100,0,200,,,0,,7.1,9.0 +p1-stop-r1,8,200,100,100,0,200,,,0,,8.1,16.2 +p1-stop-r1,9,200,100,100,0,200,,,0,,7.4,11.2 +p1-stop-r1,10,200,100,100,0,200,,,0,,7.1,8.6 +p1-stop-r1,11,200,100,100,0,200,,,0,,8.1,16.0 +p1-stop-r1,12,200,100,100,0,200,,,0,,8.4,9.8 +p1-stop-r1,13,200,100,100,0,200,,,0,,7.6,12.0 +p1-stop-r1,14,200,100,100,0,200,,,0,,7.4,8.3 +p1-stop-r1,15,200,100,100,0,200,,,0,,7.7,10.5 +p1-stop-r1,16,200,100,100,0,200,,,0,,7.7,9.1 +p1-stop-r1,17,200,100,100,50,150,,,0,,8.0,31.9 +p1-stop-r1,18,200,100,100,100,100,,,0,,6.2,9.5 +p1-stop-r1,19,200,100,100,100,100,,,0,,6.7,11.9 +p1-stop-r1,20,200,100,100,100,100,,,0,,6.7,9.6 +p1-stop-r1,21,200,100,100,100,100,,,0,,6.8,10.2 +p1-stop-r1,22,200,100,100,100,100,,,0,,6.8,8.7 +p1-stop-r1,23,200,100,100,100,100,,,0,,7.1,10.2 +p1-stop-r1,24,200,100,100,100,100,,,0,,7.2,8.2 +p1-stop-r1,25,200,100,100,100,100,,,0,,6.7,39.2 +p1-stop-r1,26,200,100,100,100,100,,,0,,6.5,7.5 +p1-stop-r1,27,200,100,100,100,100,,,0,,6.7,8.4 +p1-stop-r1,28,200,100,100,100,100,,,0,,7.3,10.4 +p1-stop-r1,29,200,100,100,100,100,,,0,,8.7,15.9 +p1-stop-r1,30,200,100,100,100,100,,,0,,6.5,7.8 +p1-stop-r1,31,200,100,100,100,100,,,0,,6.1,9.6 +p1-stop-r1,32,200,100,100,100,100,,,0,,6.1,7.0 +p1-stop-r1,33,200,100,100,100,100,,,0,,6.6,8.7 +p1-stop-r1,34,200,100,100,100,100,,,0,,6.8,13.4 +p1-stop-r1,35,6,3,3,3,3,,,0,,5.4,5.4 +p2-kill-r1,-10,188,94,94,94,94,,,0,,7.2,30.6 +p2-kill-r1,-9,200,100,100,100,100,,,0,,6.4,7.5 +p2-kill-r1,-8,200,100,100,100,100,,,0,,6.4,7.3 +p2-kill-r1,-7,200,100,100,100,100,,,0,,6.5,9.2 +p2-kill-r1,-6,200,100,100,100,100,,,0,,6.8,13.3 +p2-kill-r1,-5,200,100,100,100,100,,,0,,6.2,6.7 +p2-kill-r1,-4,200,100,100,100,100,,,0,,6.4,7.6 +p2-kill-r1,-3,200,100,100,100,100,,,0,,6.6,9.6 +p2-kill-r1,-2,200,100,100,100,100,,,0,,6.2,6.7 +p2-kill-r1,-1,200,99,100,99,100,,,1,IncompleteReadError:1,6.2,7.4 +p2-kill-r1,0,200,100,100,0,200,,,0,,507.0,536.1 +p2-kill-r1,1,200,100,100,0,200,,,0,,505.6,507.4 +p2-kill-r1,2,200,100,100,0,200,,,0,,7.8,10.0 +p2-kill-r1,3,200,100,100,0,200,,,0,,6.8,9.0 +p2-kill-r1,4,200,100,100,0,200,,,0,,7.7,13.3 +p2-kill-r1,5,200,100,100,0,200,,,0,,6.9,9.0 +p2-kill-r1,6,200,100,100,0,200,,,0,,7.0,9.6 +p2-kill-r1,7,200,100,100,0,200,,,0,,7.4,9.2 +p2-kill-r1,8,200,100,100,0,200,,,0,,7.4,9.4 +p2-kill-r1,9,200,100,100,0,200,,,0,,7.3,9.7 +p2-kill-r1,10,200,100,100,0,200,,,0,,7.6,9.7 +p2-kill-r1,11,200,100,100,0,200,,,0,,7.5,9.1 +p2-kill-r1,12,200,100,100,0,200,,,0,,8.3,9.4 +p2-kill-r1,13,200,100,100,0,200,,,0,,8.4,9.5 +p2-kill-r1,14,200,100,100,0,200,,,0,,8.1,11.4 +p2-kill-r1,15,200,100,100,0,200,,,0,,7.9,16.9 +p2-kill-r1,16,200,100,100,3,197,,,0,,6.5,67.9 +p2-kill-r1,17,200,100,100,100,100,,,0,,6.3,52.7 +p2-kill-r1,18,200,100,100,100,100,,,0,,6.4,7.1 +p2-kill-r1,19,200,100,100,100,100,,,0,,8.5,10.0 +p2-kill-r1,20,200,100,100,100,100,,,0,,7.1,10.0 +p2-kill-r1,21,200,100,100,100,100,,,0,,6.6,8.9 +p2-kill-r1,22,200,100,100,100,100,,,0,,6.5,8.3 +p2-kill-r1,23,200,100,100,100,100,,,0,,6.1,11.9 +p2-kill-r1,24,200,100,100,100,100,,,0,,6.6,34.6 +p2-kill-r1,25,200,100,100,100,100,,,0,,6.7,8.9 +p2-kill-r1,26,200,100,100,100,100,,,0,,6.3,8.2 +p2-kill-r1,27,200,100,100,100,100,,,0,,7.1,16.5 +p2-kill-r1,28,200,100,100,100,100,,,0,,6.3,6.9 +p2-kill-r1,29,200,100,100,100,100,,,0,,6.3,6.9 +p2-kill-r1,30,200,100,100,100,100,,,0,,6.8,8.2 +p2-kill-r1,31,200,100,100,100,100,,,0,,6.8,9.0 +p2-kill-r1,32,200,100,100,100,100,,,0,,6.9,8.6 +p2-kill-r1,33,200,100,100,100,100,,,0,,6.1,6.8 +p2-kill-r1,34,200,100,100,100,100,,,0,,6.9,14.5 +p2-kill-r1,35,12,6,6,6,6,,,0,,6.5,6.5 +p3-pause-r1,-10,196,98,98,98,98,,,0,,7.4,17.6 +p3-pause-r1,-9,200,100,100,100,100,,,0,,6.9,9.4 +p3-pause-r1,-8,200,100,100,100,100,,,0,,6.5,8.0 +p3-pause-r1,-7,200,100,100,100,100,,,0,,6.7,7.9 +p3-pause-r1,-6,200,100,100,100,100,,,0,,6.7,8.3 +p3-pause-r1,-5,200,100,100,100,100,,,0,,6.7,8.8 +p3-pause-r1,-4,200,100,100,100,100,,,0,,6.7,8.2 +p3-pause-r1,-3,200,100,100,100,100,,,0,,7.5,13.7 +p3-pause-r1,-2,200,100,100,100,100,,,0,,7.4,21.2 +p3-pause-r1,-1,200,100,99,99,100,,,1,504:1,6.6,7.1 +p3-pause-r1,0,200,98,2,0,100,,,100,504:100,8.5,16.3 +p3-pause-r1,1,200,99,1,0,100,,,100,504:100,7.2,24.5 +p3-pause-r1,2,200,96,4,0,100,,,100,504:100,8.0,20.2 +p3-pause-r1,3,200,99,1,0,100,,,100,504:100,8.6,42.7 +p3-pause-r1,4,200,97,3,0,100,,,100,504:100,7.6,18.0 +p3-pause-r1,5,200,99,17,5,111,,,84,504:84,30.5,9998.4 +p3-pause-r1,6,200,100,100,0,200,,,0,,7.2,9.2 +p3-pause-r1,7,200,100,100,0,200,,,0,,8.7,13.1 +p3-pause-r1,8,200,100,100,0,200,,,0,,7.6,9.9 +p3-pause-r1,9,200,100,100,0,200,,,0,,8.7,10.5 +p3-pause-r1,10,200,100,100,0,200,,,0,,7.2,8.2 +p3-pause-r1,11,200,100,100,0,200,,,0,,7.4,8.0 +p3-pause-r1,12,200,100,100,0,200,,,0,,6.8,8.1 +p3-pause-r1,13,200,100,100,0,200,,,0,,7.5,9.1 +p3-pause-r1,14,200,100,100,0,200,,,0,,7.5,8.5 +p3-pause-r1,15,200,100,100,0,200,,,0,,6.2,9.8 +p3-pause-r1,16,200,100,100,91,109,,,0,,6.6,7.3 +p3-pause-r1,17,200,100,100,100,100,,,0,,6.4,14.6 +p3-pause-r1,18,200,100,100,100,100,,,0,,6.4,7.2 +p3-pause-r1,19,200,100,100,100,100,,,0,,6.6,9.7 +p3-pause-r1,20,200,100,100,100,100,,,0,,6.9,17.5 +p3-pause-r1,21,200,100,100,100,100,,,0,,6.3,7.0 +p3-pause-r1,22,200,100,100,100,100,,,0,,6.9,9.5 +p3-pause-r1,23,200,100,100,100,100,,,0,,6.5,7.6 +p3-pause-r1,24,200,100,100,100,100,,,0,,6.3,7.1 +p3-pause-r1,25,200,100,100,100,100,,,0,,7.0,8.5 +p3-pause-r1,26,200,100,100,100,100,,,0,,6.6,10.3 +p3-pause-r1,27,200,100,100,100,100,,,0,,7.1,8.3 +p3-pause-r1,28,200,100,100,100,100,,,0,,6.8,16.1 +p3-pause-r1,29,200,100,100,100,100,,,0,,6.5,10.3 +p3-pause-r1,30,200,100,100,100,100,,,0,,6.9,15.4 +p3-pause-r1,31,200,100,100,100,100,,,0,,6.7,7.5 +p3-pause-r1,32,200,100,100,100,100,,,0,,6.5,7.6 +p3-pause-r1,33,200,100,100,100,100,,,0,,7.2,8.9 +p3-pause-r1,34,200,100,100,100,100,,,0,,6.7,8.1 +p3-pause-r1,35,4,2,2,2,2,,,0,,4.9,4.9 +p4-stop-r2,-10,192,96,96,96,96,,,0,,7.8,32.4 +p4-stop-r2,-9,200,100,100,100,100,,,0,,7.1,11.2 +p4-stop-r2,-8,200,100,100,100,100,,,0,,6.1,7.8 +p4-stop-r2,-7,200,100,100,100,100,,,0,,6.5,8.6 +p4-stop-r2,-6,200,100,100,100,100,,,0,,6.1,10.3 +p4-stop-r2,-5,200,100,100,100,100,,,0,,6.2,7.3 +p4-stop-r2,-4,200,100,100,100,100,,,0,,7.3,8.7 +p4-stop-r2,-3,200,100,100,100,100,,,0,,6.8,7.7 +p4-stop-r2,-2,200,100,100,100,100,,,0,,6.0,7.0 +p4-stop-r2,-1,200,100,100,100,100,,,0,,6.7,7.4 +p4-stop-r2,0,200,100,100,7,193,,,0,,506.4,511.7 +p4-stop-r2,1,200,100,100,0,200,,,0,,505.7,528.0 +p4-stop-r2,2,200,100,100,0,200,,,0,,7.0,9.0 +p4-stop-r2,3,200,100,100,0,200,,,0,,7.5,10.8 +p4-stop-r2,4,200,100,100,0,200,,,0,,7.8,10.3 +p4-stop-r2,5,200,100,100,0,200,,,0,,8.5,9.8 +p4-stop-r2,6,200,100,100,0,200,,,0,,7.0,9.0 +p4-stop-r2,7,200,100,100,0,200,,,0,,6.7,8.2 +p4-stop-r2,8,200,100,100,0,200,,,0,,6.9,15.2 +p4-stop-r2,9,200,100,100,0,200,,,0,,7.8,10.4 +p4-stop-r2,10,200,100,100,0,200,,,0,,8.5,9.9 +p4-stop-r2,11,200,100,100,0,200,,,0,,8.6,10.7 +p4-stop-r2,12,200,100,100,0,200,,,0,,7.2,8.3 +p4-stop-r2,13,200,100,100,0,200,,,0,,6.8,8.6 +p4-stop-r2,14,200,100,100,0,200,,,0,,6.6,8.9 +p4-stop-r2,15,200,100,100,0,200,,,0,,7.4,18.4 +p4-stop-r2,16,200,100,100,72,128,,,0,,6.7,31.1 +p4-stop-r2,17,200,100,100,100,100,,,0,,6.3,7.2 +p4-stop-r2,18,200,100,100,100,100,,,0,,6.1,7.3 +p4-stop-r2,19,200,100,100,100,100,,,0,,6.3,7.3 +p4-stop-r2,20,200,100,100,100,100,,,0,,6.3,8.9 +p4-stop-r2,21,200,100,100,100,100,,,0,,7.3,15.6 +p4-stop-r2,22,200,100,100,100,100,,,0,,6.5,8.3 +p4-stop-r2,23,200,100,100,100,100,,,0,,6.4,11.1 +p4-stop-r2,24,200,100,100,100,100,,,0,,6.5,30.3 +p4-stop-r2,25,200,100,100,100,100,,,0,,7.4,9.5 +p4-stop-r2,26,200,100,100,100,100,,,0,,7.8,9.5 +p4-stop-r2,27,200,100,100,100,100,,,0,,6.3,7.0 +p4-stop-r2,28,200,100,100,100,100,,,0,,5.9,17.8 +p4-stop-r2,29,200,100,100,100,100,,,0,,5.9,6.8 +p4-stop-r2,30,200,100,100,100,100,,,0,,6.2,6.7 +p4-stop-r2,31,200,100,100,100,100,,,0,,7.6,12.3 +p4-stop-r2,32,200,100,100,100,100,,,0,,6.3,12.0 +p4-stop-r2,33,200,100,100,100,100,,,0,,7.4,11.7 +p4-stop-r2,34,200,100,100,100,100,,,0,,6.2,6.6 +p4-stop-r2,35,8,4,4,4,4,,,0,,6.5,6.5 +p5-kill-r2,-10,192,96,96,96,96,,,0,,7.1,26.8 +p5-kill-r2,-9,200,100,100,100,100,,,0,,6.3,7.3 +p5-kill-r2,-8,200,100,100,100,100,,,0,,6.5,8.9 +p5-kill-r2,-7,200,100,100,100,100,,,0,,6.3,6.9 +p5-kill-r2,-6,200,100,100,100,100,,,0,,7.5,13.3 +p5-kill-r2,-5,200,100,100,100,100,,,0,,6.2,6.8 +p5-kill-r2,-4,200,100,100,100,100,,,0,,6.4,9.3 +p5-kill-r2,-3,200,100,100,100,100,,,0,,8.0,20.5 +p5-kill-r2,-2,200,100,100,100,100,,,0,,6.3,7.4 +p5-kill-r2,-1,200,100,100,100,100,,,0,,6.5,7.9 +p5-kill-r2,0,200,100,100,0,200,,,0,,506.2,507.9 +p5-kill-r2,1,200,100,100,0,200,,,0,,6.2,8.6 +p5-kill-r2,2,200,100,100,0,200,,,0,,7.3,8.0 +p5-kill-r2,3,200,100,100,0,200,,,0,,6.5,7.1 +p5-kill-r2,4,200,100,100,0,200,,,0,,7.9,10.4 +p5-kill-r2,5,200,100,100,0,200,,,0,,6.6,9.3 +p5-kill-r2,6,200,100,100,0,200,,,0,,6.8,8.7 +p5-kill-r2,7,200,100,100,0,200,,,0,,7.1,8.4 +p5-kill-r2,8,200,100,100,0,200,,,0,,7.5,15.0 +p5-kill-r2,9,200,100,100,0,200,,,0,,6.5,7.6 +p5-kill-r2,10,200,100,100,0,200,,,0,,7.8,9.9 +p5-kill-r2,11,200,100,100,0,200,,,0,,7.0,8.7 +p5-kill-r2,12,200,100,100,0,200,,,0,,7.7,9.7 +p5-kill-r2,13,200,100,100,0,200,,,0,,7.5,9.8 +p5-kill-r2,14,200,100,100,0,200,,,0,,8.2,13.2 +p5-kill-r2,15,200,100,100,0,200,,,0,,11.0,27.6 +p5-kill-r2,16,200,100,100,0,200,,,0,,6.6,10.1 +p5-kill-r2,17,200,100,100,88,112,,,0,,6.7,48.6 +p5-kill-r2,18,200,100,100,100,100,,,0,,6.3,7.0 +p5-kill-r2,19,200,100,100,100,100,,,0,,6.5,8.9 +p5-kill-r2,20,200,100,100,100,100,,,0,,6.7,7.3 +p5-kill-r2,21,200,100,100,100,100,,,0,,6.2,6.6 +p5-kill-r2,22,200,100,100,100,100,,,0,,6.6,7.3 +p5-kill-r2,23,200,100,100,100,100,,,0,,6.5,8.6 +p5-kill-r2,24,200,100,100,100,100,,,0,,6.8,34.1 +p5-kill-r2,25,200,100,100,100,100,,,0,,6.3,7.5 +p5-kill-r2,26,200,100,100,100,100,,,0,,6.8,8.8 +p5-kill-r2,27,200,100,100,100,100,,,0,,6.3,13.8 +p5-kill-r2,28,200,100,100,100,100,,,0,,7.3,9.0 +p5-kill-r2,29,200,100,100,100,100,,,0,,6.4,8.6 +p5-kill-r2,30,200,100,100,100,100,,,0,,6.3,9.3 +p5-kill-r2,31,200,100,100,100,100,,,0,,6.4,7.6 +p5-kill-r2,32,200,100,100,100,100,,,0,,5.9,6.8 +p5-kill-r2,33,200,100,100,100,100,,,0,,6.4,7.4 +p5-kill-r2,34,200,100,100,100,100,,,0,,7.4,10.8 +p5-kill-r2,35,8,4,4,4,4,,,0,,7.8,7.8 +p6-pause-r2,-10,190,95,95,95,95,,,0,,7.1,27.1 +p6-pause-r2,-9,200,100,100,100,100,,,0,,6.4,8.2 +p6-pause-r2,-8,200,100,100,100,100,,,0,,7.0,8.3 +p6-pause-r2,-7,200,100,100,100,100,,,0,,6.8,9.1 +p6-pause-r2,-6,200,100,100,100,100,,,0,,6.6,7.3 +p6-pause-r2,-5,200,100,100,100,100,,,0,,6.6,8.3 +p6-pause-r2,-4,200,100,100,100,100,,,0,,6.5,8.7 +p6-pause-r2,-3,200,100,100,100,100,,,0,,7.1,11.4 +p6-pause-r2,-2,200,100,100,100,100,,,0,,6.7,8.2 +p6-pause-r2,-1,200,99,100,99,100,,,1,504:1,7.1,7.8 +p6-pause-r2,0,200,2,98,0,100,,,100,504:100,7.6,22.7 +p6-pause-r2,1,200,0,100,0,100,,,100,504:100,6.6,7.6 +p6-pause-r2,2,200,1,99,0,100,,,100,504:100,7.9,9.0 +p6-pause-r2,3,200,0,100,0,100,,,100,504:100,6.8,7.8 +p6-pause-r2,4,200,3,97,0,100,,,100,504:100,6.9,23.8 +p6-pause-r2,5,200,47,100,47,100,,,53,504:53,9947.6,10000.6 +p6-pause-r2,6,200,100,100,6,194,,,0,,7.2,9530.3 +p6-pause-r2,7,200,100,100,0,200,,,0,,7.5,14.1 +p6-pause-r2,8,200,100,100,0,200,,,0,,7.5,13.9 +p6-pause-r2,9,200,100,100,0,200,,,0,,7.9,8.7 +p6-pause-r2,10,200,100,100,0,200,,,0,,6.7,11.8 +p6-pause-r2,11,200,100,100,0,200,,,0,,7.2,8.5 +p6-pause-r2,12,200,100,100,0,200,,,0,,6.8,7.7 +p6-pause-r2,13,200,100,100,0,200,,,0,,6.9,8.2 +p6-pause-r2,14,200,100,100,0,200,,,0,,7.2,8.2 +p6-pause-r2,15,200,100,100,0,200,,,0,,7.2,9.5 +p6-pause-r2,16,200,100,100,97,103,,,0,,6.8,8.2 +p6-pause-r2,17,200,100,100,100,100,,,0,,7.0,13.0 +p6-pause-r2,18,200,100,100,100,100,,,0,,5.9,6.6 +p6-pause-r2,19,200,100,100,100,100,,,0,,6.3,7.3 +p6-pause-r2,20,200,100,100,100,100,,,0,,7.6,15.7 +p6-pause-r2,21,200,100,100,100,100,,,0,,6.8,8.7 +p6-pause-r2,22,200,100,100,100,100,,,0,,6.9,7.5 +p6-pause-r2,23,200,100,100,100,100,,,0,,7.0,8.5 +p6-pause-r2,24,200,100,100,100,100,,,0,,6.4,8.5 +p6-pause-r2,25,200,100,100,100,100,,,0,,6.2,6.8 +p6-pause-r2,26,200,100,100,100,100,,,0,,6.6,7.4 +p6-pause-r2,27,200,100,100,100,100,,,0,,6.9,10.6 +p6-pause-r2,28,200,100,100,100,100,,,0,,7.6,15.4 +p6-pause-r2,29,200,100,100,100,100,,,0,,7.3,14.1 +p6-pause-r2,30,200,100,100,100,100,,,0,,6.8,14.6 +p6-pause-r2,31,200,100,100,100,100,,,0,,6.6,8.2 +p6-pause-r2,32,200,100,100,100,100,,,0,,6.5,7.0 +p6-pause-r2,33,200,100,100,100,100,,,0,,7.0,9.0 +p6-pause-r2,34,200,100,100,100,100,,,0,,7.1,9.4 +p6-pause-r2,35,10,5,5,5,5,,,0,,7.2,7.2