test: capture API failover under sustained load

This commit is contained in:
2026-10-15 14:01:21 +03:00
parent 006ec24d20
commit dc1e8a3aff
11 changed files with 555 additions and 0 deletions
+207
View File
@@ -0,0 +1,207 @@
{
"kind": "failover",
"label": "baseline",
"started": "2026-10-15T10:55:52Z",
"profile": {
"name": "failover",
"limits": {
"api_replicas": 2,
"api_cpus": 1.0,
"api_memory": "512m",
"db_cpus": 2.0,
"db_memory": "1g",
"db_max_connections": 100,
"cache_cpus": 0.5,
"cache_memory": "128m",
"lb_cpus": 1.0,
"lb_memory": "256m",
"gen_cpus": 2.0,
"gen_memory": "1g",
"sampler_cpus": 0.25,
"sampler_memory": "256m"
},
"users": 4,
"read_messages": 1000,
"write_rate": 100.0,
"read_rate": 100.0,
"gen_procs": 2,
"session_ttl": 21600,
"seed": 20261014,
"modes": [
"stop",
"kill",
"pause"
],
"targets": 2,
"before_s": 10.0,
"fail_s": 15.0,
"after_s": 10.0,
"restore_s": 10.0,
"rollout_before_s": 0.0,
"rollout_budget_s": 0.0,
"rollout_after_s": 0.0,
"bounds": [
[
"stop",
2.5
],
[
"kill",
2.5
],
[
"pause",
6.5
]
]
},
"host": {
"platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O",
"machine": "arm64",
"cpu_model": "Apple M4",
"cpu_count": 10,
"ram_gb": 16.0,
"swap_used_mib": 0.0,
"power": "Battery Power",
"memory_pressure": null,
"memory_free_percent": 79,
"swapouts": 0,
"loadavg_1m": 1.34
},
"docker": {
"ServerVersion": "28.0.4",
"OperatingSystem": "Docker Desktop",
"KernelVersion": "6.10.14-linuxkit",
"Architecture": "aarch64",
"NCPU": 10,
"MemTotal": 8218034176,
"CgroupVersion": "2",
"CgroupDriver": "cgroupfs",
"compose_version": "2.34.0-desktop.1",
"other_running_containers": []
},
"budget": {
"limits_cpus": 7.75,
"limits_memory_gib": 3.62,
"vm_cpus": 10,
"vm_memory_gib": 7.65
},
"host_warnings": [
"host on battery power: CPU may be throttled"
],
"image": {
"tag": "mtusi-messenger-vladimirceva:bench-baseline",
"id": "sha256:ac17d94cda500d4a65418c184deec9428e706ff40a68a7789b068c973c7d4e05",
"created": "2026-10-15T10:39:12.700675009Z",
"src_sha256": "0349e9a2701784673391c0ba4bc2eedba7013098d6ff7f871261aab36f78354a",
"versions": {
"python": "3.13.15",
"fastapi": "0.142.1",
"starlette": "1.7.0",
"uvicorn": "0.54.0",
"pydantic": "2.13.5",
"sqlalchemy": "2.1.1",
"asyncpg": "0.31.0",
"alembic": "1.20.0",
"redis": "8.1.0",
"argon2-cffi": "25.1.0"
},
"app_defaults": {
"pool_size": 10,
"pool_overflow": 5,
"pool_timeout": 5.0,
"operation_timeout": 6.0,
"statement_timeout_ms": 5000,
"kdf_concurrency": 4,
"keep_alive": 60,
"shutdown_timeout": 5
}
},
"code": {
"src_sha256": "0349e9a2701784673391c0ba4bc2eedba7013098d6ff7f871261aab36f78354a",
"git_commit": "006ec24d208670f6405bc1bd02f2aafe207e2b30",
"git_dirty": false
},
"setup": {
"users": 4,
"read_chat": "691d7f26-cb3d-4fbf-8037-95c2b0ff2f98",
"read_messages": 1000,
"cursors": 20,
"fill_s": 0.94,
"fill_rps": 1069.0
},
"stand": {
"postgres": {
"image": "postgres:18.6-trixie@sha256:5a5a84b19854a9ffaa54082c166ff4ec27473a361e496e5ea167f298f2da9722",
"version": "18.6 (Debian 18.6-1.pgdg13+2)",
"settings": {
"checkpoint_timeout": "5min",
"fsync": "on",
"jit": "on",
"max_connections": "100",
"max_wal_size": "1GB",
"password_encryption": "scram-sha-256",
"shared_buffers": "128MB",
"synchronous_commit": "on",
"wal_level": "replica"
}
},
"valkey": {
"image": "valkey/valkey:9.1.2-trixie@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd",
"version": "9.1.2"
},
"haproxy": {
"image": "haproxy:3.4.6-trixie@sha256:e273915691a8eab3f105bf3e39fd4c29a4ecf334e33421903b5589d25e14db26"
},
"limits": {
"api": {
"nano_cpus": 1000000000,
"memory": 536870912,
"memory_swap": 536870912,
"published_ports": {}
},
"postgres": {
"nano_cpus": 2000000000,
"memory": 1073741824,
"memory_swap": 1073741824,
"published_ports": {}
},
"valkey": {
"nano_cpus": 500000000,
"memory": 134217728,
"memory_swap": 134217728,
"published_ports": {}
},
"haproxy": {
"nano_cpus": 1000000000,
"memory": 268435456,
"memory_swap": 268435456,
"published_ports": {}
}
},
"api_replicas": 2,
"app_env": {
"APP_HOST": "0.0.0.0",
"APP_PORT": "8080",
"CACHE_HOST": "valkey",
"CACHE_PORT": "6379",
"DB_HOST": "postgres",
"DB_NAME": "bench",
"DB_PORT": "5432",
"DB_USER": "bench",
"IDEMPOTENCY_TTL_SECONDS": "86400",
"LOG_LEVEL": "INFO",
"SESSION_TTL_SECONDS": "21600",
"SHUTDOWN_TIMEOUT": "5"
}
},
"sampler_ticks": 281,
"finished": "2026-10-15T11:00:43Z",
"host_state_end": {
"slept_seconds": 0.0,
"swapouts_start": 0,
"swapouts_end": 0,
"power_end": "Battery Power",
"swapouts_delta": 0
}
}
+7
View File
@@ -0,0 +1,7 @@
phase,mode,replica,target,slot,action_at_ms,restore_at_ms,detection_s,bound_s,over_bound,lb_return_s,errors_before,errors_fail,errors_after,error_kinds,first_error_s,last_error_s,recovery_s,ok_rps_before,ok_rps_fail,ok_rps_after,p95_before_ms,p95_fail_ms,p95_after_ms,max_fail_ms,target_after_down,writes,retried,recovered,rows,distinct_texts,verified,lb_errors
p1-stop-r1,stop,r1,cce5762ff431,api-2,1792061771104.8,1792061786580.0,1.79,2.5,False,2.03,0,1,0,IncompleteReadError:1,0.09,0.09,1.0,200.0,199.9,199.9,6.5,13.3,6.9,512.6,0,4500,0,0,4500,4500,True,
p2-kill-r1,kill,r1,cce5762ff431,api-2,1792061817739.5,1792061832912.1,1.21,2.5,False,1.8,1,0,0,IncompleteReadError:1,-0.0,-0.0,0.0,199.8,200.0,199.9,6.4,9.0,6.7,536.1,0,4500,1,1,4500,4500,True,
p3-pause-r1,pause,r1,cce5762ff431,api-2,1792061864401.4,1792061879442.8,5.89,6.5,False,1.04,1,584,0,504:585,-0.01,5.83,6.0,199.7,161.2,199.9,6.9,7.8,6.7,9998.4,0,4500,12,12,4500,4500,True,504:585
p4-stop-r2,stop,r2,dad5d68a3f49,api-1,1792061911082.5,1792061926559.0,1.27,2.5,False,0.81,0,0,0,,,,0.0,200.2,199.9,200.0,6.6,8.6,6.6,528.0,0,4500,0,0,4500,4500,True,
p5-kill-r2,kill,r2,dad5d68a3f49,api-1,1792061957859.1,1792061973051.8,0.26,2.5,False,1.92,0,0,0,,,,0.0,199.8,200.1,199.9,6.5,7.5,6.6,507.9,0,4500,0,0,4500,4500,True,
p6-pause-r2,pause,r2,dad5d68a3f49,api-1,1792062004497.9,1792062019540.0,6.06,6.5,False,0.98,1,553,0,504:554,-0.01,5.52,6.0,199.9,163.2,199.9,6.8,7.6,6.9,10000.6,0,4500,548,548,4500,4500,True,504:554
1 phase mode replica target slot action_at_ms restore_at_ms detection_s bound_s over_bound lb_return_s errors_before errors_fail errors_after error_kinds first_error_s last_error_s recovery_s ok_rps_before ok_rps_fail ok_rps_after p95_before_ms p95_fail_ms p95_after_ms max_fail_ms target_after_down writes retried recovered rows distinct_texts verified lb_errors
2 p1-stop-r1 stop r1 cce5762ff431 api-2 1792061771104.8 1792061786580.0 1.79 2.5 False 2.03 0 1 0 IncompleteReadError:1 0.09 0.09 1.0 200.0 199.9 199.9 6.5 13.3 6.9 512.6 0 4500 0 0 4500 4500 True
3 p2-kill-r1 kill r1 cce5762ff431 api-2 1792061817739.5 1792061832912.1 1.21 2.5 False 1.8 1 0 0 IncompleteReadError:1 -0.0 -0.0 0.0 199.8 200.0 199.9 6.4 9.0 6.7 536.1 0 4500 1 1 4500 4500 True
4 p3-pause-r1 pause r1 cce5762ff431 api-2 1792061864401.4 1792061879442.8 5.89 6.5 False 1.04 1 584 0 504:585 -0.01 5.83 6.0 199.7 161.2 199.9 6.9 7.8 6.7 9998.4 0 4500 12 12 4500 4500 True 504:585
5 p4-stop-r2 stop r2 dad5d68a3f49 api-1 1792061911082.5 1792061926559.0 1.27 2.5 False 0.81 0 0 0 0.0 200.2 199.9 200.0 6.6 8.6 6.6 528.0 0 4500 0 0 4500 4500 True
6 p5-kill-r2 kill r2 dad5d68a3f49 api-1 1792061957859.1 1792061973051.8 0.26 2.5 False 1.92 0 0 0 0.0 199.8 200.1 199.9 6.5 7.5 6.6 507.9 0 4500 0 0 4500 4500 True
7 p6-pause-r2 pause r2 dad5d68a3f49 api-1 1792062004497.9 1792062019540.0 6.06 6.5 False 0.98 1 553 0 504:554 -0.01 5.52 6.0 199.9 163.2 199.9 6.8 7.6 6.9 10000.6 0 4500 548 548 4500 4500 True 504:554
+19
View File
@@ -0,0 +1,19 @@
phase,window,api_cpu_cores,api_cpu_max,api_throttled_max,api_memory_max_mib,db_cpu_cores,db_memory_mib,lb_cpu_cores,cache_cpu_cores,gen_cpu_cores,vm_busy_share,api_cpu_us_per_req,db_cpu_us_per_req
p1-stop-r1,before,0.47,0.243,0.0,70.1,0.085,143.3,0.015,0.006,0.028,0.0814,2350.0,425.0
p1-stop-r1,fail,0.343,0.343,0.0,71.3,0.071,143.3,0.013,0.005,0.024,0.0563,1715.9,355.2
p1-stop-r1,after,0.484,0.243,0.0,71.3,0.091,137.2,0.016,0.006,0.026,0.0726,2421.2,455.2
p2-kill-r1,before,0.51,0.259,0.0,71.3,0.089,138.5,0.017,0.006,0.03,0.074,2552.6,445.4
p2-kill-r1,fail,0.351,0.351,0.0,72.1,0.069,138.3,0.014,0.006,0.024,0.0597,1755.0,345.0
p2-kill-r1,after,0.489,0.252,0.0,71.8,0.097,147.8,0.015,0.006,0.029,0.0737,2446.2,485.2
p3-pause-r1,before,0.509,0.267,0.0,72.3,0.099,149.4,0.016,0.007,0.03,0.0784,2548.8,495.7
p3-pause-r1,fail,0.326,0.306,0.0,86.7,0.065,162.2,0.017,0.006,0.035,0.0563,2022.3,403.2
p3-pause-r1,after,0.522,0.275,0.0,94.1,0.104,162.5,0.017,0.007,0.029,0.0812,2611.3,520.3
p4-stop-r2,before,0.496,0.255,0.0,94.1,0.115,163.5,0.016,0.006,0.029,0.0801,2477.5,574.4
p4-stop-r2,fail,0.35,0.35,0.0,95.1,0.072,163.4,0.014,0.006,0.023,0.0578,1750.9,360.2
p4-stop-r2,after,0.521,0.275,0.005,94.8,0.094,168.3,0.016,0.006,0.027,0.0768,2605.0,470.0
p5-kill-r2,before,0.529,0.266,0.0,94.6,0.096,169.1,0.017,0.006,0.03,0.0758,2647.6,480.5
p5-kill-r2,fail,0.344,0.344,0.0,95.1,0.085,169.1,0.013,0.006,0.023,0.0622,1719.1,424.8
p5-kill-r2,after,0.488,0.251,0.0,94.9,0.092,175.9,0.016,0.006,0.027,0.0746,2441.2,460.2
p6-pause-r2,before,0.502,0.257,0.0,94.6,0.094,177.0,0.016,0.006,0.031,0.0756,2511.3,470.2
p6-pause-r2,fail,0.328,0.311,0.0,94.9,0.065,177.7,0.018,0.006,0.039,0.0555,2009.8,398.3
p6-pause-r2,after,0.505,0.266,0.0051,94.7,0.116,193.4,0.017,0.006,0.029,0.0808,2526.3,580.3
1 phase window api_cpu_cores api_cpu_max api_throttled_max api_memory_max_mib db_cpu_cores db_memory_mib lb_cpu_cores cache_cpu_cores gen_cpu_cores vm_busy_share api_cpu_us_per_req db_cpu_us_per_req
2 p1-stop-r1 before 0.47 0.243 0.0 70.1 0.085 143.3 0.015 0.006 0.028 0.0814 2350.0 425.0
3 p1-stop-r1 fail 0.343 0.343 0.0 71.3 0.071 143.3 0.013 0.005 0.024 0.0563 1715.9 355.2
4 p1-stop-r1 after 0.484 0.243 0.0 71.3 0.091 137.2 0.016 0.006 0.026 0.0726 2421.2 455.2
5 p2-kill-r1 before 0.51 0.259 0.0 71.3 0.089 138.5 0.017 0.006 0.03 0.074 2552.6 445.4
6 p2-kill-r1 fail 0.351 0.351 0.0 72.1 0.069 138.3 0.014 0.006 0.024 0.0597 1755.0 345.0
7 p2-kill-r1 after 0.489 0.252 0.0 71.8 0.097 147.8 0.015 0.006 0.029 0.0737 2446.2 485.2
8 p3-pause-r1 before 0.509 0.267 0.0 72.3 0.099 149.4 0.016 0.007 0.03 0.0784 2548.8 495.7
9 p3-pause-r1 fail 0.326 0.306 0.0 86.7 0.065 162.2 0.017 0.006 0.035 0.0563 2022.3 403.2
10 p3-pause-r1 after 0.522 0.275 0.0 94.1 0.104 162.5 0.017 0.007 0.029 0.0812 2611.3 520.3
11 p4-stop-r2 before 0.496 0.255 0.0 94.1 0.115 163.5 0.016 0.006 0.029 0.0801 2477.5 574.4
12 p4-stop-r2 fail 0.35 0.35 0.0 95.1 0.072 163.4 0.014 0.006 0.023 0.0578 1750.9 360.2
13 p4-stop-r2 after 0.521 0.275 0.005 94.8 0.094 168.3 0.016 0.006 0.027 0.0768 2605.0 470.0
14 p5-kill-r2 before 0.529 0.266 0.0 94.6 0.096 169.1 0.017 0.006 0.03 0.0758 2647.6 480.5
15 p5-kill-r2 fail 0.344 0.344 0.0 95.1 0.085 169.1 0.013 0.006 0.023 0.0622 1719.1 424.8
16 p5-kill-r2 after 0.488 0.251 0.0 94.9 0.092 175.9 0.016 0.006 0.027 0.0746 2441.2 460.2
17 p6-pause-r2 before 0.502 0.257 0.0 94.6 0.094 177.0 0.016 0.006 0.031 0.0756 2511.3 470.2
18 p6-pause-r2 fail 0.328 0.311 0.0 94.9 0.065 177.7 0.018 0.006 0.039 0.0555 2009.8 398.3
19 p6-pause-r2 after 0.505 0.266 0.0051 94.7 0.116 193.4 0.017 0.006 0.029 0.0808 2526.3 580.3
+45
View File
@@ -0,0 +1,45 @@
# Отказ реплики под нагрузкой: baseline
Код: src_sha256 `0349e9a270178467`, commit 006ec24d2086. Хост: Apple M4, 16.0 ГиБ; ВМ Docker 10 CPU, 7.7 ГиБ, Docker 28.0.4, compose 2.34.0-desktop.1. Начало 2026-10-15T10:55:52Z, конец 2026-10-15T11:00:43Z.
Нагрузка: открытая модель, 100 записей с `Idempotency-Key` и 100 чтений страницы истории в секунду через HAProxy стенда; две реплики с лимитами рабочего стенда. Фаза: 10 с до действия, действие, 15 с отказа, возврат той же реплики, ещё 10 с.
Отметки — часы ВМ Docker: действие и возврат — события Docker, исключение и возврат в LB — журнал HAProxy, запросы — генератор (момент по расписанию). Обнаружение — от действия до `DOWN` слота; восстановление — от действия до начала секунды, после которой до возврата нет ошибок и успешных ≥ 90 % поданной нагрузки.
| Фаза | Режим | Реплика | Обнаружение, с (граница) | Возврат в LB, с | Ошибки до / отказ / после | Виды ошибок | Восстановление, с | Успешных/с до / отказ / после | p95, мс до / отказ / после | max в отказе, мс | Повтор: не прошло → прошло | Сверка |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| p1-stop-r1 | stop | r1 (cce5762ff431) | 1.79 (2.5) | 2.03 | 0 / 1 / 0 | IncompleteReadError:1 | 1 | 200.0 / 199.9 / 199.9 | 6.5 / 13.3 / 6.9 | 512.6 | 0 → 0 | ok: 4500 ключей = 4500 строк |
| p2-kill-r1 | kill | r1 (cce5762ff431) | 1.21 (2.5) | 1.8 | 1 / 0 / 0 | IncompleteReadError:1 | 0 | 199.8 / 200.0 / 199.9 | 6.4 / 9 / 6.7 | 536.1 | 1 → 1 | ok: 4500 ключей = 4500 строк |
| p3-pause-r1 | pause | r1 (cce5762ff431) | 5.89 (6.5) | 1.04 | 1 / 584 / 0 | 504:585 | 6 | 199.7 / 161.2 / 199.9 | 6.9 / 7.8 / 6.7 | 9998.4 | 12 → 12 | ok: 4500 ключей = 4500 строк |
| p4-stop-r2 | stop | r2 (dad5d68a3f49) | 1.27 (2.5) | 0.81 | 0 / 0 / 0 | — | 0 | 200.2 / 199.9 / 200.0 | 6.6 / 8.6 / 6.6 | 528 | 0 → 0 | ok: 4500 ключей = 4500 строк |
| p5-kill-r2 | kill | r2 (dad5d68a3f49) | 0.26 (2.5) | 1.92 | 0 / 0 / 0 | — | 0 | 199.8 / 200.1 / 199.9 | 6.5 / 7.5 / 6.6 | 507.9 | 0 → 0 | ok: 4500 ключей = 4500 строк |
| p6-pause-r2 | pause | r2 (dad5d68a3f49) | 6.06 (6.5) | 0.98 | 1 / 553 / 0 | 504:554 | 6 | 199.9 / 163.2 / 199.9 | 6.8 / 7.6 / 6.9 | 10000.6 | 548 → 548 | ok: 4500 ключей = 4500 строк |
## CPU и память по окнам фазы
Сэмплер (cgroup каждого контейнера раз в секунду). Окна: до действия (от первого запроса + 2 с), отказ (действие → возврат), после (возврат в LB → конец нагрузки). CPU реплик — сумма; на успешный запрос — только по окнам с нагрузкой.
| Фаза | Окно | API, ядер (макс. реплика) | PostgreSQL | HAProxy | Valkey | Генератор | ВМ занята | API, мкс/запрос | PostgreSQL, мкс/запрос |
|---|---|---|---|---|---|---|---|---|---|
| p1-stop-r1 | before | 0.47 (0.243) | 0.085 | 0.015 | 0.006 | 0.028 | 0.0814 | 2350 | 425 |
| p1-stop-r1 | fail | 0.343 (0.343) | 0.071 | 0.013 | 0.005 | 0.024 | 0.0563 | 1715.9 | 355.2 |
| p1-stop-r1 | after | 0.484 (0.243) | 0.091 | 0.016 | 0.006 | 0.026 | 0.0726 | 2421.2 | 455.2 |
| p2-kill-r1 | before | 0.51 (0.259) | 0.089 | 0.017 | 0.006 | 0.03 | 0.074 | 2552.6 | 445.4 |
| p2-kill-r1 | fail | 0.351 (0.351) | 0.069 | 0.014 | 0.006 | 0.024 | 0.0597 | 1755 | 345 |
| p2-kill-r1 | after | 0.489 (0.252) | 0.097 | 0.015 | 0.006 | 0.029 | 0.0737 | 2446.2 | 485.2 |
| p3-pause-r1 | before | 0.509 (0.267) | 0.099 | 0.016 | 0.007 | 0.03 | 0.0784 | 2548.8 | 495.7 |
| p3-pause-r1 | fail | 0.326 (0.306) | 0.065 | 0.017 | 0.006 | 0.035 | 0.0563 | 2022.3 | 403.2 |
| p3-pause-r1 | after | 0.522 (0.275) | 0.104 | 0.017 | 0.007 | 0.029 | 0.0812 | 2611.3 | 520.3 |
| p4-stop-r2 | before | 0.496 (0.255) | 0.115 | 0.016 | 0.006 | 0.029 | 0.0801 | 2477.5 | 574.4 |
| p4-stop-r2 | fail | 0.35 (0.35) | 0.072 | 0.014 | 0.006 | 0.023 | 0.0578 | 1750.9 | 360.2 |
| p4-stop-r2 | after | 0.521 (0.275) | 0.094 | 0.016 | 0.006 | 0.027 | 0.0768 | 2605 | 470 |
| p5-kill-r2 | before | 0.529 (0.266) | 0.096 | 0.017 | 0.006 | 0.03 | 0.0758 | 2647.6 | 480.5 |
| p5-kill-r2 | fail | 0.344 (0.344) | 0.085 | 0.013 | 0.006 | 0.023 | 0.0622 | 1719.1 | 424.8 |
| p5-kill-r2 | after | 0.488 (0.251) | 0.092 | 0.016 | 0.006 | 0.027 | 0.0746 | 2441.2 | 460.2 |
| p6-pause-r2 | before | 0.502 (0.257) | 0.094 | 0.016 | 0.006 | 0.031 | 0.0756 | 2511.3 | 470.2 |
| p6-pause-r2 | fail | 0.328 (0.311) | 0.065 | 0.018 | 0.006 | 0.039 | 0.0555 | 2009.8 | 398.3 |
| p6-pause-r2 | after | 0.505 (0.266) | 0.116 | 0.017 | 0.006 | 0.029 | 0.0808 | 2526.3 | 580.3 |
## Предупреждения о хосте
- host on battery power: CPU may be throttled
+277
View File
@@ -0,0 +1,277 @@
phase,second,sent,ok_write,ok_read,ok_target,ok_other,ok_old,ok_new,errors,error_kinds,p95_ms,max_ms
p1-stop-r1,-10,194,97,97,97,97,,,0,,5.6,11.6
p1-stop-r1,-9,200,100,100,100,100,,,0,,6.8,10.8
p1-stop-r1,-8,200,100,100,100,100,,,0,,6.6,35.7
p1-stop-r1,-7,200,100,100,100,100,,,0,,6.7,9.3
p1-stop-r1,-6,200,100,100,100,100,,,0,,6.4,7.5
p1-stop-r1,-5,200,100,100,100,100,,,0,,6.0,7.3
p1-stop-r1,-4,200,100,100,100,100,,,0,,6.1,7.3
p1-stop-r1,-3,200,100,100,100,100,,,0,,6.6,10.3
p1-stop-r1,-2,200,100,100,100,100,,,0,,6.6,9.0
p1-stop-r1,-1,200,100,100,100,100,,,0,,6.6,7.7
p1-stop-r1,0,200,100,99,8,191,,,1,IncompleteReadError:1,506.1,512.6
p1-stop-r1,1,200,100,100,0,200,,,0,,506.6,507.1
p1-stop-r1,2,200,100,100,0,200,,,0,,7.4,8.5
p1-stop-r1,3,200,100,100,0,200,,,0,,7.4,10.1
p1-stop-r1,4,200,100,100,0,200,,,0,,8.5,10.8
p1-stop-r1,5,200,100,100,0,200,,,0,,8.6,10.6
p1-stop-r1,6,200,100,100,0,200,,,0,,7.3,9.1
p1-stop-r1,7,200,100,100,0,200,,,0,,7.1,9.0
p1-stop-r1,8,200,100,100,0,200,,,0,,8.1,16.2
p1-stop-r1,9,200,100,100,0,200,,,0,,7.4,11.2
p1-stop-r1,10,200,100,100,0,200,,,0,,7.1,8.6
p1-stop-r1,11,200,100,100,0,200,,,0,,8.1,16.0
p1-stop-r1,12,200,100,100,0,200,,,0,,8.4,9.8
p1-stop-r1,13,200,100,100,0,200,,,0,,7.6,12.0
p1-stop-r1,14,200,100,100,0,200,,,0,,7.4,8.3
p1-stop-r1,15,200,100,100,0,200,,,0,,7.7,10.5
p1-stop-r1,16,200,100,100,0,200,,,0,,7.7,9.1
p1-stop-r1,17,200,100,100,50,150,,,0,,8.0,31.9
p1-stop-r1,18,200,100,100,100,100,,,0,,6.2,9.5
p1-stop-r1,19,200,100,100,100,100,,,0,,6.7,11.9
p1-stop-r1,20,200,100,100,100,100,,,0,,6.7,9.6
p1-stop-r1,21,200,100,100,100,100,,,0,,6.8,10.2
p1-stop-r1,22,200,100,100,100,100,,,0,,6.8,8.7
p1-stop-r1,23,200,100,100,100,100,,,0,,7.1,10.2
p1-stop-r1,24,200,100,100,100,100,,,0,,7.2,8.2
p1-stop-r1,25,200,100,100,100,100,,,0,,6.7,39.2
p1-stop-r1,26,200,100,100,100,100,,,0,,6.5,7.5
p1-stop-r1,27,200,100,100,100,100,,,0,,6.7,8.4
p1-stop-r1,28,200,100,100,100,100,,,0,,7.3,10.4
p1-stop-r1,29,200,100,100,100,100,,,0,,8.7,15.9
p1-stop-r1,30,200,100,100,100,100,,,0,,6.5,7.8
p1-stop-r1,31,200,100,100,100,100,,,0,,6.1,9.6
p1-stop-r1,32,200,100,100,100,100,,,0,,6.1,7.0
p1-stop-r1,33,200,100,100,100,100,,,0,,6.6,8.7
p1-stop-r1,34,200,100,100,100,100,,,0,,6.8,13.4
p1-stop-r1,35,6,3,3,3,3,,,0,,5.4,5.4
p2-kill-r1,-10,188,94,94,94,94,,,0,,7.2,30.6
p2-kill-r1,-9,200,100,100,100,100,,,0,,6.4,7.5
p2-kill-r1,-8,200,100,100,100,100,,,0,,6.4,7.3
p2-kill-r1,-7,200,100,100,100,100,,,0,,6.5,9.2
p2-kill-r1,-6,200,100,100,100,100,,,0,,6.8,13.3
p2-kill-r1,-5,200,100,100,100,100,,,0,,6.2,6.7
p2-kill-r1,-4,200,100,100,100,100,,,0,,6.4,7.6
p2-kill-r1,-3,200,100,100,100,100,,,0,,6.6,9.6
p2-kill-r1,-2,200,100,100,100,100,,,0,,6.2,6.7
p2-kill-r1,-1,200,99,100,99,100,,,1,IncompleteReadError:1,6.2,7.4
p2-kill-r1,0,200,100,100,0,200,,,0,,507.0,536.1
p2-kill-r1,1,200,100,100,0,200,,,0,,505.6,507.4
p2-kill-r1,2,200,100,100,0,200,,,0,,7.8,10.0
p2-kill-r1,3,200,100,100,0,200,,,0,,6.8,9.0
p2-kill-r1,4,200,100,100,0,200,,,0,,7.7,13.3
p2-kill-r1,5,200,100,100,0,200,,,0,,6.9,9.0
p2-kill-r1,6,200,100,100,0,200,,,0,,7.0,9.6
p2-kill-r1,7,200,100,100,0,200,,,0,,7.4,9.2
p2-kill-r1,8,200,100,100,0,200,,,0,,7.4,9.4
p2-kill-r1,9,200,100,100,0,200,,,0,,7.3,9.7
p2-kill-r1,10,200,100,100,0,200,,,0,,7.6,9.7
p2-kill-r1,11,200,100,100,0,200,,,0,,7.5,9.1
p2-kill-r1,12,200,100,100,0,200,,,0,,8.3,9.4
p2-kill-r1,13,200,100,100,0,200,,,0,,8.4,9.5
p2-kill-r1,14,200,100,100,0,200,,,0,,8.1,11.4
p2-kill-r1,15,200,100,100,0,200,,,0,,7.9,16.9
p2-kill-r1,16,200,100,100,3,197,,,0,,6.5,67.9
p2-kill-r1,17,200,100,100,100,100,,,0,,6.3,52.7
p2-kill-r1,18,200,100,100,100,100,,,0,,6.4,7.1
p2-kill-r1,19,200,100,100,100,100,,,0,,8.5,10.0
p2-kill-r1,20,200,100,100,100,100,,,0,,7.1,10.0
p2-kill-r1,21,200,100,100,100,100,,,0,,6.6,8.9
p2-kill-r1,22,200,100,100,100,100,,,0,,6.5,8.3
p2-kill-r1,23,200,100,100,100,100,,,0,,6.1,11.9
p2-kill-r1,24,200,100,100,100,100,,,0,,6.6,34.6
p2-kill-r1,25,200,100,100,100,100,,,0,,6.7,8.9
p2-kill-r1,26,200,100,100,100,100,,,0,,6.3,8.2
p2-kill-r1,27,200,100,100,100,100,,,0,,7.1,16.5
p2-kill-r1,28,200,100,100,100,100,,,0,,6.3,6.9
p2-kill-r1,29,200,100,100,100,100,,,0,,6.3,6.9
p2-kill-r1,30,200,100,100,100,100,,,0,,6.8,8.2
p2-kill-r1,31,200,100,100,100,100,,,0,,6.8,9.0
p2-kill-r1,32,200,100,100,100,100,,,0,,6.9,8.6
p2-kill-r1,33,200,100,100,100,100,,,0,,6.1,6.8
p2-kill-r1,34,200,100,100,100,100,,,0,,6.9,14.5
p2-kill-r1,35,12,6,6,6,6,,,0,,6.5,6.5
p3-pause-r1,-10,196,98,98,98,98,,,0,,7.4,17.6
p3-pause-r1,-9,200,100,100,100,100,,,0,,6.9,9.4
p3-pause-r1,-8,200,100,100,100,100,,,0,,6.5,8.0
p3-pause-r1,-7,200,100,100,100,100,,,0,,6.7,7.9
p3-pause-r1,-6,200,100,100,100,100,,,0,,6.7,8.3
p3-pause-r1,-5,200,100,100,100,100,,,0,,6.7,8.8
p3-pause-r1,-4,200,100,100,100,100,,,0,,6.7,8.2
p3-pause-r1,-3,200,100,100,100,100,,,0,,7.5,13.7
p3-pause-r1,-2,200,100,100,100,100,,,0,,7.4,21.2
p3-pause-r1,-1,200,100,99,99,100,,,1,504:1,6.6,7.1
p3-pause-r1,0,200,98,2,0,100,,,100,504:100,8.5,16.3
p3-pause-r1,1,200,99,1,0,100,,,100,504:100,7.2,24.5
p3-pause-r1,2,200,96,4,0,100,,,100,504:100,8.0,20.2
p3-pause-r1,3,200,99,1,0,100,,,100,504:100,8.6,42.7
p3-pause-r1,4,200,97,3,0,100,,,100,504:100,7.6,18.0
p3-pause-r1,5,200,99,17,5,111,,,84,504:84,30.5,9998.4
p3-pause-r1,6,200,100,100,0,200,,,0,,7.2,9.2
p3-pause-r1,7,200,100,100,0,200,,,0,,8.7,13.1
p3-pause-r1,8,200,100,100,0,200,,,0,,7.6,9.9
p3-pause-r1,9,200,100,100,0,200,,,0,,8.7,10.5
p3-pause-r1,10,200,100,100,0,200,,,0,,7.2,8.2
p3-pause-r1,11,200,100,100,0,200,,,0,,7.4,8.0
p3-pause-r1,12,200,100,100,0,200,,,0,,6.8,8.1
p3-pause-r1,13,200,100,100,0,200,,,0,,7.5,9.1
p3-pause-r1,14,200,100,100,0,200,,,0,,7.5,8.5
p3-pause-r1,15,200,100,100,0,200,,,0,,6.2,9.8
p3-pause-r1,16,200,100,100,91,109,,,0,,6.6,7.3
p3-pause-r1,17,200,100,100,100,100,,,0,,6.4,14.6
p3-pause-r1,18,200,100,100,100,100,,,0,,6.4,7.2
p3-pause-r1,19,200,100,100,100,100,,,0,,6.6,9.7
p3-pause-r1,20,200,100,100,100,100,,,0,,6.9,17.5
p3-pause-r1,21,200,100,100,100,100,,,0,,6.3,7.0
p3-pause-r1,22,200,100,100,100,100,,,0,,6.9,9.5
p3-pause-r1,23,200,100,100,100,100,,,0,,6.5,7.6
p3-pause-r1,24,200,100,100,100,100,,,0,,6.3,7.1
p3-pause-r1,25,200,100,100,100,100,,,0,,7.0,8.5
p3-pause-r1,26,200,100,100,100,100,,,0,,6.6,10.3
p3-pause-r1,27,200,100,100,100,100,,,0,,7.1,8.3
p3-pause-r1,28,200,100,100,100,100,,,0,,6.8,16.1
p3-pause-r1,29,200,100,100,100,100,,,0,,6.5,10.3
p3-pause-r1,30,200,100,100,100,100,,,0,,6.9,15.4
p3-pause-r1,31,200,100,100,100,100,,,0,,6.7,7.5
p3-pause-r1,32,200,100,100,100,100,,,0,,6.5,7.6
p3-pause-r1,33,200,100,100,100,100,,,0,,7.2,8.9
p3-pause-r1,34,200,100,100,100,100,,,0,,6.7,8.1
p3-pause-r1,35,4,2,2,2,2,,,0,,4.9,4.9
p4-stop-r2,-10,192,96,96,96,96,,,0,,7.8,32.4
p4-stop-r2,-9,200,100,100,100,100,,,0,,7.1,11.2
p4-stop-r2,-8,200,100,100,100,100,,,0,,6.1,7.8
p4-stop-r2,-7,200,100,100,100,100,,,0,,6.5,8.6
p4-stop-r2,-6,200,100,100,100,100,,,0,,6.1,10.3
p4-stop-r2,-5,200,100,100,100,100,,,0,,6.2,7.3
p4-stop-r2,-4,200,100,100,100,100,,,0,,7.3,8.7
p4-stop-r2,-3,200,100,100,100,100,,,0,,6.8,7.7
p4-stop-r2,-2,200,100,100,100,100,,,0,,6.0,7.0
p4-stop-r2,-1,200,100,100,100,100,,,0,,6.7,7.4
p4-stop-r2,0,200,100,100,7,193,,,0,,506.4,511.7
p4-stop-r2,1,200,100,100,0,200,,,0,,505.7,528.0
p4-stop-r2,2,200,100,100,0,200,,,0,,7.0,9.0
p4-stop-r2,3,200,100,100,0,200,,,0,,7.5,10.8
p4-stop-r2,4,200,100,100,0,200,,,0,,7.8,10.3
p4-stop-r2,5,200,100,100,0,200,,,0,,8.5,9.8
p4-stop-r2,6,200,100,100,0,200,,,0,,7.0,9.0
p4-stop-r2,7,200,100,100,0,200,,,0,,6.7,8.2
p4-stop-r2,8,200,100,100,0,200,,,0,,6.9,15.2
p4-stop-r2,9,200,100,100,0,200,,,0,,7.8,10.4
p4-stop-r2,10,200,100,100,0,200,,,0,,8.5,9.9
p4-stop-r2,11,200,100,100,0,200,,,0,,8.6,10.7
p4-stop-r2,12,200,100,100,0,200,,,0,,7.2,8.3
p4-stop-r2,13,200,100,100,0,200,,,0,,6.8,8.6
p4-stop-r2,14,200,100,100,0,200,,,0,,6.6,8.9
p4-stop-r2,15,200,100,100,0,200,,,0,,7.4,18.4
p4-stop-r2,16,200,100,100,72,128,,,0,,6.7,31.1
p4-stop-r2,17,200,100,100,100,100,,,0,,6.3,7.2
p4-stop-r2,18,200,100,100,100,100,,,0,,6.1,7.3
p4-stop-r2,19,200,100,100,100,100,,,0,,6.3,7.3
p4-stop-r2,20,200,100,100,100,100,,,0,,6.3,8.9
p4-stop-r2,21,200,100,100,100,100,,,0,,7.3,15.6
p4-stop-r2,22,200,100,100,100,100,,,0,,6.5,8.3
p4-stop-r2,23,200,100,100,100,100,,,0,,6.4,11.1
p4-stop-r2,24,200,100,100,100,100,,,0,,6.5,30.3
p4-stop-r2,25,200,100,100,100,100,,,0,,7.4,9.5
p4-stop-r2,26,200,100,100,100,100,,,0,,7.8,9.5
p4-stop-r2,27,200,100,100,100,100,,,0,,6.3,7.0
p4-stop-r2,28,200,100,100,100,100,,,0,,5.9,17.8
p4-stop-r2,29,200,100,100,100,100,,,0,,5.9,6.8
p4-stop-r2,30,200,100,100,100,100,,,0,,6.2,6.7
p4-stop-r2,31,200,100,100,100,100,,,0,,7.6,12.3
p4-stop-r2,32,200,100,100,100,100,,,0,,6.3,12.0
p4-stop-r2,33,200,100,100,100,100,,,0,,7.4,11.7
p4-stop-r2,34,200,100,100,100,100,,,0,,6.2,6.6
p4-stop-r2,35,8,4,4,4,4,,,0,,6.5,6.5
p5-kill-r2,-10,192,96,96,96,96,,,0,,7.1,26.8
p5-kill-r2,-9,200,100,100,100,100,,,0,,6.3,7.3
p5-kill-r2,-8,200,100,100,100,100,,,0,,6.5,8.9
p5-kill-r2,-7,200,100,100,100,100,,,0,,6.3,6.9
p5-kill-r2,-6,200,100,100,100,100,,,0,,7.5,13.3
p5-kill-r2,-5,200,100,100,100,100,,,0,,6.2,6.8
p5-kill-r2,-4,200,100,100,100,100,,,0,,6.4,9.3
p5-kill-r2,-3,200,100,100,100,100,,,0,,8.0,20.5
p5-kill-r2,-2,200,100,100,100,100,,,0,,6.3,7.4
p5-kill-r2,-1,200,100,100,100,100,,,0,,6.5,7.9
p5-kill-r2,0,200,100,100,0,200,,,0,,506.2,507.9
p5-kill-r2,1,200,100,100,0,200,,,0,,6.2,8.6
p5-kill-r2,2,200,100,100,0,200,,,0,,7.3,8.0
p5-kill-r2,3,200,100,100,0,200,,,0,,6.5,7.1
p5-kill-r2,4,200,100,100,0,200,,,0,,7.9,10.4
p5-kill-r2,5,200,100,100,0,200,,,0,,6.6,9.3
p5-kill-r2,6,200,100,100,0,200,,,0,,6.8,8.7
p5-kill-r2,7,200,100,100,0,200,,,0,,7.1,8.4
p5-kill-r2,8,200,100,100,0,200,,,0,,7.5,15.0
p5-kill-r2,9,200,100,100,0,200,,,0,,6.5,7.6
p5-kill-r2,10,200,100,100,0,200,,,0,,7.8,9.9
p5-kill-r2,11,200,100,100,0,200,,,0,,7.0,8.7
p5-kill-r2,12,200,100,100,0,200,,,0,,7.7,9.7
p5-kill-r2,13,200,100,100,0,200,,,0,,7.5,9.8
p5-kill-r2,14,200,100,100,0,200,,,0,,8.2,13.2
p5-kill-r2,15,200,100,100,0,200,,,0,,11.0,27.6
p5-kill-r2,16,200,100,100,0,200,,,0,,6.6,10.1
p5-kill-r2,17,200,100,100,88,112,,,0,,6.7,48.6
p5-kill-r2,18,200,100,100,100,100,,,0,,6.3,7.0
p5-kill-r2,19,200,100,100,100,100,,,0,,6.5,8.9
p5-kill-r2,20,200,100,100,100,100,,,0,,6.7,7.3
p5-kill-r2,21,200,100,100,100,100,,,0,,6.2,6.6
p5-kill-r2,22,200,100,100,100,100,,,0,,6.6,7.3
p5-kill-r2,23,200,100,100,100,100,,,0,,6.5,8.6
p5-kill-r2,24,200,100,100,100,100,,,0,,6.8,34.1
p5-kill-r2,25,200,100,100,100,100,,,0,,6.3,7.5
p5-kill-r2,26,200,100,100,100,100,,,0,,6.8,8.8
p5-kill-r2,27,200,100,100,100,100,,,0,,6.3,13.8
p5-kill-r2,28,200,100,100,100,100,,,0,,7.3,9.0
p5-kill-r2,29,200,100,100,100,100,,,0,,6.4,8.6
p5-kill-r2,30,200,100,100,100,100,,,0,,6.3,9.3
p5-kill-r2,31,200,100,100,100,100,,,0,,6.4,7.6
p5-kill-r2,32,200,100,100,100,100,,,0,,5.9,6.8
p5-kill-r2,33,200,100,100,100,100,,,0,,6.4,7.4
p5-kill-r2,34,200,100,100,100,100,,,0,,7.4,10.8
p5-kill-r2,35,8,4,4,4,4,,,0,,7.8,7.8
p6-pause-r2,-10,190,95,95,95,95,,,0,,7.1,27.1
p6-pause-r2,-9,200,100,100,100,100,,,0,,6.4,8.2
p6-pause-r2,-8,200,100,100,100,100,,,0,,7.0,8.3
p6-pause-r2,-7,200,100,100,100,100,,,0,,6.8,9.1
p6-pause-r2,-6,200,100,100,100,100,,,0,,6.6,7.3
p6-pause-r2,-5,200,100,100,100,100,,,0,,6.6,8.3
p6-pause-r2,-4,200,100,100,100,100,,,0,,6.5,8.7
p6-pause-r2,-3,200,100,100,100,100,,,0,,7.1,11.4
p6-pause-r2,-2,200,100,100,100,100,,,0,,6.7,8.2
p6-pause-r2,-1,200,99,100,99,100,,,1,504:1,7.1,7.8
p6-pause-r2,0,200,2,98,0,100,,,100,504:100,7.6,22.7
p6-pause-r2,1,200,0,100,0,100,,,100,504:100,6.6,7.6
p6-pause-r2,2,200,1,99,0,100,,,100,504:100,7.9,9.0
p6-pause-r2,3,200,0,100,0,100,,,100,504:100,6.8,7.8
p6-pause-r2,4,200,3,97,0,100,,,100,504:100,6.9,23.8
p6-pause-r2,5,200,47,100,47,100,,,53,504:53,9947.6,10000.6
p6-pause-r2,6,200,100,100,6,194,,,0,,7.2,9530.3
p6-pause-r2,7,200,100,100,0,200,,,0,,7.5,14.1
p6-pause-r2,8,200,100,100,0,200,,,0,,7.5,13.9
p6-pause-r2,9,200,100,100,0,200,,,0,,7.9,8.7
p6-pause-r2,10,200,100,100,0,200,,,0,,6.7,11.8
p6-pause-r2,11,200,100,100,0,200,,,0,,7.2,8.5
p6-pause-r2,12,200,100,100,0,200,,,0,,6.8,7.7
p6-pause-r2,13,200,100,100,0,200,,,0,,6.9,8.2
p6-pause-r2,14,200,100,100,0,200,,,0,,7.2,8.2
p6-pause-r2,15,200,100,100,0,200,,,0,,7.2,9.5
p6-pause-r2,16,200,100,100,97,103,,,0,,6.8,8.2
p6-pause-r2,17,200,100,100,100,100,,,0,,7.0,13.0
p6-pause-r2,18,200,100,100,100,100,,,0,,5.9,6.6
p6-pause-r2,19,200,100,100,100,100,,,0,,6.3,7.3
p6-pause-r2,20,200,100,100,100,100,,,0,,7.6,15.7
p6-pause-r2,21,200,100,100,100,100,,,0,,6.8,8.7
p6-pause-r2,22,200,100,100,100,100,,,0,,6.9,7.5
p6-pause-r2,23,200,100,100,100,100,,,0,,7.0,8.5
p6-pause-r2,24,200,100,100,100,100,,,0,,6.4,8.5
p6-pause-r2,25,200,100,100,100,100,,,0,,6.2,6.8
p6-pause-r2,26,200,100,100,100,100,,,0,,6.6,7.4
p6-pause-r2,27,200,100,100,100,100,,,0,,6.9,10.6
p6-pause-r2,28,200,100,100,100,100,,,0,,7.6,15.4
p6-pause-r2,29,200,100,100,100,100,,,0,,7.3,14.1
p6-pause-r2,30,200,100,100,100,100,,,0,,6.8,14.6
p6-pause-r2,31,200,100,100,100,100,,,0,,6.6,8.2
p6-pause-r2,32,200,100,100,100,100,,,0,,6.5,7.0
p6-pause-r2,33,200,100,100,100,100,,,0,,7.0,9.0
p6-pause-r2,34,200,100,100,100,100,,,0,,7.1,9.4
p6-pause-r2,35,10,5,5,5,5,,,0,,7.2,7.2
1 phase second sent ok_write ok_read ok_target ok_other ok_old ok_new errors error_kinds p95_ms max_ms
2 p1-stop-r1 -10 194 97 97 97 97 0 5.6 11.6
3 p1-stop-r1 -9 200 100 100 100 100 0 6.8 10.8
4 p1-stop-r1 -8 200 100 100 100 100 0 6.6 35.7
5 p1-stop-r1 -7 200 100 100 100 100 0 6.7 9.3
6 p1-stop-r1 -6 200 100 100 100 100 0 6.4 7.5
7 p1-stop-r1 -5 200 100 100 100 100 0 6.0 7.3
8 p1-stop-r1 -4 200 100 100 100 100 0 6.1 7.3
9 p1-stop-r1 -3 200 100 100 100 100 0 6.6 10.3
10 p1-stop-r1 -2 200 100 100 100 100 0 6.6 9.0
11 p1-stop-r1 -1 200 100 100 100 100 0 6.6 7.7
12 p1-stop-r1 0 200 100 99 8 191 1 IncompleteReadError:1 506.1 512.6
13 p1-stop-r1 1 200 100 100 0 200 0 506.6 507.1
14 p1-stop-r1 2 200 100 100 0 200 0 7.4 8.5
15 p1-stop-r1 3 200 100 100 0 200 0 7.4 10.1
16 p1-stop-r1 4 200 100 100 0 200 0 8.5 10.8
17 p1-stop-r1 5 200 100 100 0 200 0 8.6 10.6
18 p1-stop-r1 6 200 100 100 0 200 0 7.3 9.1
19 p1-stop-r1 7 200 100 100 0 200 0 7.1 9.0
20 p1-stop-r1 8 200 100 100 0 200 0 8.1 16.2
21 p1-stop-r1 9 200 100 100 0 200 0 7.4 11.2
22 p1-stop-r1 10 200 100 100 0 200 0 7.1 8.6
23 p1-stop-r1 11 200 100 100 0 200 0 8.1 16.0
24 p1-stop-r1 12 200 100 100 0 200 0 8.4 9.8
25 p1-stop-r1 13 200 100 100 0 200 0 7.6 12.0
26 p1-stop-r1 14 200 100 100 0 200 0 7.4 8.3
27 p1-stop-r1 15 200 100 100 0 200 0 7.7 10.5
28 p1-stop-r1 16 200 100 100 0 200 0 7.7 9.1
29 p1-stop-r1 17 200 100 100 50 150 0 8.0 31.9
30 p1-stop-r1 18 200 100 100 100 100 0 6.2 9.5
31 p1-stop-r1 19 200 100 100 100 100 0 6.7 11.9
32 p1-stop-r1 20 200 100 100 100 100 0 6.7 9.6
33 p1-stop-r1 21 200 100 100 100 100 0 6.8 10.2
34 p1-stop-r1 22 200 100 100 100 100 0 6.8 8.7
35 p1-stop-r1 23 200 100 100 100 100 0 7.1 10.2
36 p1-stop-r1 24 200 100 100 100 100 0 7.2 8.2
37 p1-stop-r1 25 200 100 100 100 100 0 6.7 39.2
38 p1-stop-r1 26 200 100 100 100 100 0 6.5 7.5
39 p1-stop-r1 27 200 100 100 100 100 0 6.7 8.4
40 p1-stop-r1 28 200 100 100 100 100 0 7.3 10.4
41 p1-stop-r1 29 200 100 100 100 100 0 8.7 15.9
42 p1-stop-r1 30 200 100 100 100 100 0 6.5 7.8
43 p1-stop-r1 31 200 100 100 100 100 0 6.1 9.6
44 p1-stop-r1 32 200 100 100 100 100 0 6.1 7.0
45 p1-stop-r1 33 200 100 100 100 100 0 6.6 8.7
46 p1-stop-r1 34 200 100 100 100 100 0 6.8 13.4
47 p1-stop-r1 35 6 3 3 3 3 0 5.4 5.4
48 p2-kill-r1 -10 188 94 94 94 94 0 7.2 30.6
49 p2-kill-r1 -9 200 100 100 100 100 0 6.4 7.5
50 p2-kill-r1 -8 200 100 100 100 100 0 6.4 7.3
51 p2-kill-r1 -7 200 100 100 100 100 0 6.5 9.2
52 p2-kill-r1 -6 200 100 100 100 100 0 6.8 13.3
53 p2-kill-r1 -5 200 100 100 100 100 0 6.2 6.7
54 p2-kill-r1 -4 200 100 100 100 100 0 6.4 7.6
55 p2-kill-r1 -3 200 100 100 100 100 0 6.6 9.6
56 p2-kill-r1 -2 200 100 100 100 100 0 6.2 6.7
57 p2-kill-r1 -1 200 99 100 99 100 1 IncompleteReadError:1 6.2 7.4
58 p2-kill-r1 0 200 100 100 0 200 0 507.0 536.1
59 p2-kill-r1 1 200 100 100 0 200 0 505.6 507.4
60 p2-kill-r1 2 200 100 100 0 200 0 7.8 10.0
61 p2-kill-r1 3 200 100 100 0 200 0 6.8 9.0
62 p2-kill-r1 4 200 100 100 0 200 0 7.7 13.3
63 p2-kill-r1 5 200 100 100 0 200 0 6.9 9.0
64 p2-kill-r1 6 200 100 100 0 200 0 7.0 9.6
65 p2-kill-r1 7 200 100 100 0 200 0 7.4 9.2
66 p2-kill-r1 8 200 100 100 0 200 0 7.4 9.4
67 p2-kill-r1 9 200 100 100 0 200 0 7.3 9.7
68 p2-kill-r1 10 200 100 100 0 200 0 7.6 9.7
69 p2-kill-r1 11 200 100 100 0 200 0 7.5 9.1
70 p2-kill-r1 12 200 100 100 0 200 0 8.3 9.4
71 p2-kill-r1 13 200 100 100 0 200 0 8.4 9.5
72 p2-kill-r1 14 200 100 100 0 200 0 8.1 11.4
73 p2-kill-r1 15 200 100 100 0 200 0 7.9 16.9
74 p2-kill-r1 16 200 100 100 3 197 0 6.5 67.9
75 p2-kill-r1 17 200 100 100 100 100 0 6.3 52.7
76 p2-kill-r1 18 200 100 100 100 100 0 6.4 7.1
77 p2-kill-r1 19 200 100 100 100 100 0 8.5 10.0
78 p2-kill-r1 20 200 100 100 100 100 0 7.1 10.0
79 p2-kill-r1 21 200 100 100 100 100 0 6.6 8.9
80 p2-kill-r1 22 200 100 100 100 100 0 6.5 8.3
81 p2-kill-r1 23 200 100 100 100 100 0 6.1 11.9
82 p2-kill-r1 24 200 100 100 100 100 0 6.6 34.6
83 p2-kill-r1 25 200 100 100 100 100 0 6.7 8.9
84 p2-kill-r1 26 200 100 100 100 100 0 6.3 8.2
85 p2-kill-r1 27 200 100 100 100 100 0 7.1 16.5
86 p2-kill-r1 28 200 100 100 100 100 0 6.3 6.9
87 p2-kill-r1 29 200 100 100 100 100 0 6.3 6.9
88 p2-kill-r1 30 200 100 100 100 100 0 6.8 8.2
89 p2-kill-r1 31 200 100 100 100 100 0 6.8 9.0
90 p2-kill-r1 32 200 100 100 100 100 0 6.9 8.6
91 p2-kill-r1 33 200 100 100 100 100 0 6.1 6.8
92 p2-kill-r1 34 200 100 100 100 100 0 6.9 14.5
93 p2-kill-r1 35 12 6 6 6 6 0 6.5 6.5
94 p3-pause-r1 -10 196 98 98 98 98 0 7.4 17.6
95 p3-pause-r1 -9 200 100 100 100 100 0 6.9 9.4
96 p3-pause-r1 -8 200 100 100 100 100 0 6.5 8.0
97 p3-pause-r1 -7 200 100 100 100 100 0 6.7 7.9
98 p3-pause-r1 -6 200 100 100 100 100 0 6.7 8.3
99 p3-pause-r1 -5 200 100 100 100 100 0 6.7 8.8
100 p3-pause-r1 -4 200 100 100 100 100 0 6.7 8.2
101 p3-pause-r1 -3 200 100 100 100 100 0 7.5 13.7
102 p3-pause-r1 -2 200 100 100 100 100 0 7.4 21.2
103 p3-pause-r1 -1 200 100 99 99 100 1 504:1 6.6 7.1
104 p3-pause-r1 0 200 98 2 0 100 100 504:100 8.5 16.3
105 p3-pause-r1 1 200 99 1 0 100 100 504:100 7.2 24.5
106 p3-pause-r1 2 200 96 4 0 100 100 504:100 8.0 20.2
107 p3-pause-r1 3 200 99 1 0 100 100 504:100 8.6 42.7
108 p3-pause-r1 4 200 97 3 0 100 100 504:100 7.6 18.0
109 p3-pause-r1 5 200 99 17 5 111 84 504:84 30.5 9998.4
110 p3-pause-r1 6 200 100 100 0 200 0 7.2 9.2
111 p3-pause-r1 7 200 100 100 0 200 0 8.7 13.1
112 p3-pause-r1 8 200 100 100 0 200 0 7.6 9.9
113 p3-pause-r1 9 200 100 100 0 200 0 8.7 10.5
114 p3-pause-r1 10 200 100 100 0 200 0 7.2 8.2
115 p3-pause-r1 11 200 100 100 0 200 0 7.4 8.0
116 p3-pause-r1 12 200 100 100 0 200 0 6.8 8.1
117 p3-pause-r1 13 200 100 100 0 200 0 7.5 9.1
118 p3-pause-r1 14 200 100 100 0 200 0 7.5 8.5
119 p3-pause-r1 15 200 100 100 0 200 0 6.2 9.8
120 p3-pause-r1 16 200 100 100 91 109 0 6.6 7.3
121 p3-pause-r1 17 200 100 100 100 100 0 6.4 14.6
122 p3-pause-r1 18 200 100 100 100 100 0 6.4 7.2
123 p3-pause-r1 19 200 100 100 100 100 0 6.6 9.7
124 p3-pause-r1 20 200 100 100 100 100 0 6.9 17.5
125 p3-pause-r1 21 200 100 100 100 100 0 6.3 7.0
126 p3-pause-r1 22 200 100 100 100 100 0 6.9 9.5
127 p3-pause-r1 23 200 100 100 100 100 0 6.5 7.6
128 p3-pause-r1 24 200 100 100 100 100 0 6.3 7.1
129 p3-pause-r1 25 200 100 100 100 100 0 7.0 8.5
130 p3-pause-r1 26 200 100 100 100 100 0 6.6 10.3
131 p3-pause-r1 27 200 100 100 100 100 0 7.1 8.3
132 p3-pause-r1 28 200 100 100 100 100 0 6.8 16.1
133 p3-pause-r1 29 200 100 100 100 100 0 6.5 10.3
134 p3-pause-r1 30 200 100 100 100 100 0 6.9 15.4
135 p3-pause-r1 31 200 100 100 100 100 0 6.7 7.5
136 p3-pause-r1 32 200 100 100 100 100 0 6.5 7.6
137 p3-pause-r1 33 200 100 100 100 100 0 7.2 8.9
138 p3-pause-r1 34 200 100 100 100 100 0 6.7 8.1
139 p3-pause-r1 35 4 2 2 2 2 0 4.9 4.9
140 p4-stop-r2 -10 192 96 96 96 96 0 7.8 32.4
141 p4-stop-r2 -9 200 100 100 100 100 0 7.1 11.2
142 p4-stop-r2 -8 200 100 100 100 100 0 6.1 7.8
143 p4-stop-r2 -7 200 100 100 100 100 0 6.5 8.6
144 p4-stop-r2 -6 200 100 100 100 100 0 6.1 10.3
145 p4-stop-r2 -5 200 100 100 100 100 0 6.2 7.3
146 p4-stop-r2 -4 200 100 100 100 100 0 7.3 8.7
147 p4-stop-r2 -3 200 100 100 100 100 0 6.8 7.7
148 p4-stop-r2 -2 200 100 100 100 100 0 6.0 7.0
149 p4-stop-r2 -1 200 100 100 100 100 0 6.7 7.4
150 p4-stop-r2 0 200 100 100 7 193 0 506.4 511.7
151 p4-stop-r2 1 200 100 100 0 200 0 505.7 528.0
152 p4-stop-r2 2 200 100 100 0 200 0 7.0 9.0
153 p4-stop-r2 3 200 100 100 0 200 0 7.5 10.8
154 p4-stop-r2 4 200 100 100 0 200 0 7.8 10.3
155 p4-stop-r2 5 200 100 100 0 200 0 8.5 9.8
156 p4-stop-r2 6 200 100 100 0 200 0 7.0 9.0
157 p4-stop-r2 7 200 100 100 0 200 0 6.7 8.2
158 p4-stop-r2 8 200 100 100 0 200 0 6.9 15.2
159 p4-stop-r2 9 200 100 100 0 200 0 7.8 10.4
160 p4-stop-r2 10 200 100 100 0 200 0 8.5 9.9
161 p4-stop-r2 11 200 100 100 0 200 0 8.6 10.7
162 p4-stop-r2 12 200 100 100 0 200 0 7.2 8.3
163 p4-stop-r2 13 200 100 100 0 200 0 6.8 8.6
164 p4-stop-r2 14 200 100 100 0 200 0 6.6 8.9
165 p4-stop-r2 15 200 100 100 0 200 0 7.4 18.4
166 p4-stop-r2 16 200 100 100 72 128 0 6.7 31.1
167 p4-stop-r2 17 200 100 100 100 100 0 6.3 7.2
168 p4-stop-r2 18 200 100 100 100 100 0 6.1 7.3
169 p4-stop-r2 19 200 100 100 100 100 0 6.3 7.3
170 p4-stop-r2 20 200 100 100 100 100 0 6.3 8.9
171 p4-stop-r2 21 200 100 100 100 100 0 7.3 15.6
172 p4-stop-r2 22 200 100 100 100 100 0 6.5 8.3
173 p4-stop-r2 23 200 100 100 100 100 0 6.4 11.1
174 p4-stop-r2 24 200 100 100 100 100 0 6.5 30.3
175 p4-stop-r2 25 200 100 100 100 100 0 7.4 9.5
176 p4-stop-r2 26 200 100 100 100 100 0 7.8 9.5
177 p4-stop-r2 27 200 100 100 100 100 0 6.3 7.0
178 p4-stop-r2 28 200 100 100 100 100 0 5.9 17.8
179 p4-stop-r2 29 200 100 100 100 100 0 5.9 6.8
180 p4-stop-r2 30 200 100 100 100 100 0 6.2 6.7
181 p4-stop-r2 31 200 100 100 100 100 0 7.6 12.3
182 p4-stop-r2 32 200 100 100 100 100 0 6.3 12.0
183 p4-stop-r2 33 200 100 100 100 100 0 7.4 11.7
184 p4-stop-r2 34 200 100 100 100 100 0 6.2 6.6
185 p4-stop-r2 35 8 4 4 4 4 0 6.5 6.5
186 p5-kill-r2 -10 192 96 96 96 96 0 7.1 26.8
187 p5-kill-r2 -9 200 100 100 100 100 0 6.3 7.3
188 p5-kill-r2 -8 200 100 100 100 100 0 6.5 8.9
189 p5-kill-r2 -7 200 100 100 100 100 0 6.3 6.9
190 p5-kill-r2 -6 200 100 100 100 100 0 7.5 13.3
191 p5-kill-r2 -5 200 100 100 100 100 0 6.2 6.8
192 p5-kill-r2 -4 200 100 100 100 100 0 6.4 9.3
193 p5-kill-r2 -3 200 100 100 100 100 0 8.0 20.5
194 p5-kill-r2 -2 200 100 100 100 100 0 6.3 7.4
195 p5-kill-r2 -1 200 100 100 100 100 0 6.5 7.9
196 p5-kill-r2 0 200 100 100 0 200 0 506.2 507.9
197 p5-kill-r2 1 200 100 100 0 200 0 6.2 8.6
198 p5-kill-r2 2 200 100 100 0 200 0 7.3 8.0
199 p5-kill-r2 3 200 100 100 0 200 0 6.5 7.1
200 p5-kill-r2 4 200 100 100 0 200 0 7.9 10.4
201 p5-kill-r2 5 200 100 100 0 200 0 6.6 9.3
202 p5-kill-r2 6 200 100 100 0 200 0 6.8 8.7
203 p5-kill-r2 7 200 100 100 0 200 0 7.1 8.4
204 p5-kill-r2 8 200 100 100 0 200 0 7.5 15.0
205 p5-kill-r2 9 200 100 100 0 200 0 6.5 7.6
206 p5-kill-r2 10 200 100 100 0 200 0 7.8 9.9
207 p5-kill-r2 11 200 100 100 0 200 0 7.0 8.7
208 p5-kill-r2 12 200 100 100 0 200 0 7.7 9.7
209 p5-kill-r2 13 200 100 100 0 200 0 7.5 9.8
210 p5-kill-r2 14 200 100 100 0 200 0 8.2 13.2
211 p5-kill-r2 15 200 100 100 0 200 0 11.0 27.6
212 p5-kill-r2 16 200 100 100 0 200 0 6.6 10.1
213 p5-kill-r2 17 200 100 100 88 112 0 6.7 48.6
214 p5-kill-r2 18 200 100 100 100 100 0 6.3 7.0
215 p5-kill-r2 19 200 100 100 100 100 0 6.5 8.9
216 p5-kill-r2 20 200 100 100 100 100 0 6.7 7.3
217 p5-kill-r2 21 200 100 100 100 100 0 6.2 6.6
218 p5-kill-r2 22 200 100 100 100 100 0 6.6 7.3
219 p5-kill-r2 23 200 100 100 100 100 0 6.5 8.6
220 p5-kill-r2 24 200 100 100 100 100 0 6.8 34.1
221 p5-kill-r2 25 200 100 100 100 100 0 6.3 7.5
222 p5-kill-r2 26 200 100 100 100 100 0 6.8 8.8
223 p5-kill-r2 27 200 100 100 100 100 0 6.3 13.8
224 p5-kill-r2 28 200 100 100 100 100 0 7.3 9.0
225 p5-kill-r2 29 200 100 100 100 100 0 6.4 8.6
226 p5-kill-r2 30 200 100 100 100 100 0 6.3 9.3
227 p5-kill-r2 31 200 100 100 100 100 0 6.4 7.6
228 p5-kill-r2 32 200 100 100 100 100 0 5.9 6.8
229 p5-kill-r2 33 200 100 100 100 100 0 6.4 7.4
230 p5-kill-r2 34 200 100 100 100 100 0 7.4 10.8
231 p5-kill-r2 35 8 4 4 4 4 0 7.8 7.8
232 p6-pause-r2 -10 190 95 95 95 95 0 7.1 27.1
233 p6-pause-r2 -9 200 100 100 100 100 0 6.4 8.2
234 p6-pause-r2 -8 200 100 100 100 100 0 7.0 8.3
235 p6-pause-r2 -7 200 100 100 100 100 0 6.8 9.1
236 p6-pause-r2 -6 200 100 100 100 100 0 6.6 7.3
237 p6-pause-r2 -5 200 100 100 100 100 0 6.6 8.3
238 p6-pause-r2 -4 200 100 100 100 100 0 6.5 8.7
239 p6-pause-r2 -3 200 100 100 100 100 0 7.1 11.4
240 p6-pause-r2 -2 200 100 100 100 100 0 6.7 8.2
241 p6-pause-r2 -1 200 99 100 99 100 1 504:1 7.1 7.8
242 p6-pause-r2 0 200 2 98 0 100 100 504:100 7.6 22.7
243 p6-pause-r2 1 200 0 100 0 100 100 504:100 6.6 7.6
244 p6-pause-r2 2 200 1 99 0 100 100 504:100 7.9 9.0
245 p6-pause-r2 3 200 0 100 0 100 100 504:100 6.8 7.8
246 p6-pause-r2 4 200 3 97 0 100 100 504:100 6.9 23.8
247 p6-pause-r2 5 200 47 100 47 100 53 504:53 9947.6 10000.6
248 p6-pause-r2 6 200 100 100 6 194 0 7.2 9530.3
249 p6-pause-r2 7 200 100 100 0 200 0 7.5 14.1
250 p6-pause-r2 8 200 100 100 0 200 0 7.5 13.9
251 p6-pause-r2 9 200 100 100 0 200 0 7.9 8.7
252 p6-pause-r2 10 200 100 100 0 200 0 6.7 11.8
253 p6-pause-r2 11 200 100 100 0 200 0 7.2 8.5
254 p6-pause-r2 12 200 100 100 0 200 0 6.8 7.7
255 p6-pause-r2 13 200 100 100 0 200 0 6.9 8.2
256 p6-pause-r2 14 200 100 100 0 200 0 7.2 8.2
257 p6-pause-r2 15 200 100 100 0 200 0 7.2 9.5
258 p6-pause-r2 16 200 100 100 97 103 0 6.8 8.2
259 p6-pause-r2 17 200 100 100 100 100 0 7.0 13.0
260 p6-pause-r2 18 200 100 100 100 100 0 5.9 6.6
261 p6-pause-r2 19 200 100 100 100 100 0 6.3 7.3
262 p6-pause-r2 20 200 100 100 100 100 0 7.6 15.7
263 p6-pause-r2 21 200 100 100 100 100 0 6.8 8.7
264 p6-pause-r2 22 200 100 100 100 100 0 6.9 7.5
265 p6-pause-r2 23 200 100 100 100 100 0 7.0 8.5
266 p6-pause-r2 24 200 100 100 100 100 0 6.4 8.5
267 p6-pause-r2 25 200 100 100 100 100 0 6.2 6.8
268 p6-pause-r2 26 200 100 100 100 100 0 6.6 7.4
269 p6-pause-r2 27 200 100 100 100 100 0 6.9 10.6
270 p6-pause-r2 28 200 100 100 100 100 0 7.6 15.4
271 p6-pause-r2 29 200 100 100 100 100 0 7.3 14.1
272 p6-pause-r2 30 200 100 100 100 100 0 6.8 14.6
273 p6-pause-r2 31 200 100 100 100 100 0 6.6 8.2
274 p6-pause-r2 32 200 100 100 100 100 0 6.5 7.0
275 p6-pause-r2 33 200 100 100 100 100 0 7.0 9.0
276 p6-pause-r2 34 200 100 100 100 100 0 7.1 9.4
277 p6-pause-r2 35 10 5 5 5 5 0 7.2 7.2