forked from mtusi2026/mtusi2026lab7
913 lines
25 KiB
JSON
913 lines
25 KiB
JSON
{
|
|
"date_utc": "2026-11-22T11:28:32.906048+00:00",
|
|
"project": "messenger-lab7-test",
|
|
"load": {
|
|
"date_utc": "2026-11-22T11:28:28.640765+00:00",
|
|
"workload": "synthetic POST messages; fixed arrival rate; no retries",
|
|
"duration_seconds": 10,
|
|
"elapsed_seconds": 9.957,
|
|
"arrival_rate": 10,
|
|
"concurrency_limit": 2,
|
|
"successful_requests": 100,
|
|
"failed_requests": 0,
|
|
"error_ratio": 0.0,
|
|
"successful_rps": 10.043,
|
|
"successful_latency_ms": {
|
|
"p50": 19.239,
|
|
"p95": 54.353,
|
|
"p99": 60.554
|
|
},
|
|
"dataset": {
|
|
"users": 3,
|
|
"chats": 1,
|
|
"messages": 100
|
|
},
|
|
"resource_samples": [
|
|
{
|
|
"elapsed_seconds": 2.075,
|
|
"containers": [
|
|
{
|
|
"Name": "messenger-lab7-test-api-1",
|
|
"CPUPerc": "21.07%",
|
|
"MemUsage": "108.2MiB / 1.5GiB",
|
|
"MemPerc": "7.04%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-api-2",
|
|
"CPUPerc": "23.76%",
|
|
"MemUsage": "112MiB / 1.5GiB",
|
|
"MemPerc": "7.29%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-broker-1",
|
|
"CPUPerc": "79.50%",
|
|
"MemUsage": "187.1MiB / 768MiB",
|
|
"MemPerc": "24.36%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-cleanup-1",
|
|
"CPUPerc": "0.00%",
|
|
"MemUsage": "77.41MiB / 256MiB",
|
|
"MemPerc": "30.24%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-collector-scrape-1",
|
|
"CPUPerc": "0.17%",
|
|
"MemUsage": "2.785MiB / 64MiB",
|
|
"MemPerc": "4.35%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-grafana-1",
|
|
"CPUPerc": "0.79%",
|
|
"MemUsage": "275MiB / 512MiB",
|
|
"MemPerc": "53.71%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-loki-1",
|
|
"CPUPerc": "0.62%",
|
|
"MemUsage": "52.68MiB / 512MiB",
|
|
"MemPerc": "10.29%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-minio-1",
|
|
"CPUPerc": "0.00%",
|
|
"MemUsage": "216.3MiB / 512MiB",
|
|
"MemPerc": "42.25%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-nginx-1",
|
|
"CPUPerc": "1.09%",
|
|
"MemUsage": "4.566MiB / 128MiB",
|
|
"MemPerc": "3.57%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-openbao-1",
|
|
"CPUPerc": "0.23%",
|
|
"MemUsage": "24.39MiB / 256MiB",
|
|
"MemPerc": "9.53%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-otel-collector-1",
|
|
"CPUPerc": "0.36%",
|
|
"MemUsage": "55.35MiB / 256MiB",
|
|
"MemPerc": "21.62%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-postgres-1",
|
|
"CPUPerc": "1.65%",
|
|
"MemUsage": "77.88MiB / 512MiB",
|
|
"MemPerc": "15.21%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-prometheus-1",
|
|
"CPUPerc": "0.11%",
|
|
"MemUsage": "34.97MiB / 512MiB",
|
|
"MemPerc": "6.83%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-publisher-1",
|
|
"CPUPerc": "0.71%",
|
|
"MemUsage": "72.55MiB / 256MiB",
|
|
"MemPerc": "28.34%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-redis-1",
|
|
"CPUPerc": "0.44%",
|
|
"MemUsage": "8.848MiB / 512MiB",
|
|
"MemPerc": "1.73%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-tempo-1",
|
|
"CPUPerc": "0.42%",
|
|
"MemUsage": "156.3MiB / 512MiB",
|
|
"MemPerc": "30.53%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-worker-1",
|
|
"CPUPerc": "0.01%",
|
|
"MemUsage": "71.65MiB / 2GiB",
|
|
"MemPerc": "3.50%"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"elapsed_seconds": 7.125,
|
|
"containers": [
|
|
{
|
|
"Name": "messenger-lab7-test-api-1",
|
|
"CPUPerc": "9.94%",
|
|
"MemUsage": "108.2MiB / 1.5GiB",
|
|
"MemPerc": "7.05%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-api-2",
|
|
"CPUPerc": "23.59%",
|
|
"MemUsage": "112.5MiB / 1.5GiB",
|
|
"MemPerc": "7.33%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-broker-1",
|
|
"CPUPerc": "0.21%",
|
|
"MemUsage": "113.3MiB / 768MiB",
|
|
"MemPerc": "14.76%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-cleanup-1",
|
|
"CPUPerc": "0.00%",
|
|
"MemUsage": "77.41MiB / 256MiB",
|
|
"MemPerc": "30.24%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-collector-scrape-1",
|
|
"CPUPerc": "0.12%",
|
|
"MemUsage": "2.789MiB / 64MiB",
|
|
"MemPerc": "4.36%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-grafana-1",
|
|
"CPUPerc": "0.64%",
|
|
"MemUsage": "275.1MiB / 512MiB",
|
|
"MemPerc": "53.73%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-loki-1",
|
|
"CPUPerc": "0.78%",
|
|
"MemUsage": "54.74MiB / 512MiB",
|
|
"MemPerc": "10.69%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-minio-1",
|
|
"CPUPerc": "17.44%",
|
|
"MemUsage": "216.4MiB / 512MiB",
|
|
"MemPerc": "42.26%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-nginx-1",
|
|
"CPUPerc": "1.98%",
|
|
"MemUsage": "4.605MiB / 128MiB",
|
|
"MemPerc": "3.60%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-openbao-1",
|
|
"CPUPerc": "0.32%",
|
|
"MemUsage": "24.14MiB / 256MiB",
|
|
"MemPerc": "9.43%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-otel-collector-1",
|
|
"CPUPerc": "0.34%",
|
|
"MemUsage": "55.63MiB / 256MiB",
|
|
"MemPerc": "21.73%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-postgres-1",
|
|
"CPUPerc": "2.53%",
|
|
"MemUsage": "77.8MiB / 512MiB",
|
|
"MemPerc": "15.20%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-prometheus-1",
|
|
"CPUPerc": "0.15%",
|
|
"MemUsage": "35.52MiB / 512MiB",
|
|
"MemPerc": "6.94%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-publisher-1",
|
|
"CPUPerc": "0.76%",
|
|
"MemUsage": "72.8MiB / 256MiB",
|
|
"MemPerc": "28.44%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-redis-1",
|
|
"CPUPerc": "4.46%",
|
|
"MemUsage": "8.859MiB / 512MiB",
|
|
"MemPerc": "1.73%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-tempo-1",
|
|
"CPUPerc": "0.43%",
|
|
"MemUsage": "158.1MiB / 512MiB",
|
|
"MemPerc": "30.88%"
|
|
},
|
|
{
|
|
"Name": "messenger-lab7-test-worker-1",
|
|
"CPUPerc": "0.01%",
|
|
"MemUsage": "71.65MiB / 2GiB",
|
|
"MemPerc": "3.50%"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"limits": "Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark."
|
|
},
|
|
"api_replicas": 2,
|
|
"dashboard_uid": "messenger-overview",
|
|
"api_correlation": {
|
|
"trace_id": "a6f7f0852688e953ee912aad2380a83d",
|
|
"span_id": "b89754c41f5660ed",
|
|
"log": {
|
|
"timestamp": "2026-11-22T11:28:29.340657+00:00",
|
|
"severity": "INFO",
|
|
"service_name": "messenger-api",
|
|
"event": "http.request",
|
|
"trace_id": "a6f7f0852688e953ee912aad2380a83d",
|
|
"span_id": "b89754c41f5660ed",
|
|
"method": "POST",
|
|
"route": "/api/v1/chats/{chat_id}/messages",
|
|
"status_code": 201,
|
|
"request_id": "4d079858-abdf-4a2b-8628-993a3871dad2",
|
|
"duration_seconds": 0.013454396000042834
|
|
},
|
|
"spans": [
|
|
{
|
|
"name": "POST /api/v1/chats/{chat_id}/messages",
|
|
"kind": "SPAN_KIND_SERVER",
|
|
"spanId": "uJdUxB9WYO0=",
|
|
"startTimeUnixNano": "1795346909327149486",
|
|
"endTimeUnixNano": "1795346909340768007",
|
|
"status": {
|
|
"code": "STATUS_CODE_OK"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"worker_correlation": {
|
|
"trace_id": "1e7c6f66330d96e8f014c48ab46a8273",
|
|
"span_id": "f170b2b2da2c20be",
|
|
"log": {
|
|
"timestamp": "2026-11-22T11:28:29.879326+00:00",
|
|
"severity": "INFO",
|
|
"service_name": "messenger-worker",
|
|
"event": "job.received",
|
|
"trace_id": "1e7c6f66330d96e8f014c48ab46a8273",
|
|
"span_id": "f170b2b2da2c20be",
|
|
"job_id": "cdba33de-b90f-4f13-9176-7c8bf885b2d4",
|
|
"attachment_id": "4c513979-d4e4-4f3d-9817-0c07da12c1d1"
|
|
},
|
|
"spans": [
|
|
{
|
|
"name": "attachment.process.v1",
|
|
"kind": "SPAN_KIND_CONSUMER",
|
|
"spanId": "8XCystosIL4=",
|
|
"startTimeUnixNano": "1795346909879301199",
|
|
"endTimeUnixNano": "1795346910685398243",
|
|
"status": {
|
|
"code": "STATUS_CODE_OK"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"metrics": {
|
|
"http_requests": [
|
|
{
|
|
"metric": {},
|
|
"value": [
|
|
1795346912.17,
|
|
"145"
|
|
]
|
|
}
|
|
],
|
|
"http_duration": [
|
|
{
|
|
"metric": {},
|
|
"value": [
|
|
1795346912.205,
|
|
"145"
|
|
]
|
|
}
|
|
],
|
|
"job_success": [
|
|
{
|
|
"metric": {},
|
|
"value": [
|
|
1795346912.22,
|
|
"1"
|
|
]
|
|
}
|
|
],
|
|
"job_duration": [
|
|
{
|
|
"metric": {},
|
|
"value": [
|
|
1795346912.235,
|
|
"1"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"upload_trace": {
|
|
"trace_id": "1e7c6f66330d96e8f014c48ab46a8273",
|
|
"chain": [
|
|
{
|
|
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
|
"kind": "SPAN_KIND_SERVER",
|
|
"span_id": "d114b425a264152c",
|
|
"parent_span_id": "",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "2c4c30811fd64842",
|
|
"parent_span_id": "d114b425a264152c",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "98dc6c86445d5dc5",
|
|
"parent_span_id": "d114b425a264152c",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "08d64db09c247de9",
|
|
"parent_span_id": "d114b425a264152c",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "rabbitmq.publish",
|
|
"kind": "SPAN_KIND_PRODUCER",
|
|
"span_id": "4dd40750b68877b4",
|
|
"parent_span_id": "d114b425a264152c",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "1"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "attachment.process.v1",
|
|
"kind": "SPAN_KIND_CONSUMER",
|
|
"span_id": "f170b2b2da2c20be",
|
|
"parent_span_id": "4dd40750b68877b4",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "1"
|
|
},
|
|
"outcome": {
|
|
"stringValue": "success"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "24e495260256a6e5",
|
|
"parent_span_id": "f170b2b2da2c20be",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "s3.write_result",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "8fc2173f4f8d04d4",
|
|
"parent_span_id": "f170b2b2da2c20be",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.commit_ready",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "9cfcde4ea56e423c",
|
|
"parent_span_id": "f170b2b2da2c20be",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.claim_job",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "50380257859c67ce",
|
|
"parent_span_id": "f170b2b2da2c20be",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "66bee75bfcabd324",
|
|
"parent_span_id": "50380257859c67ce",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "s3.read_staging",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "83752239ee47543a",
|
|
"parent_span_id": "f170b2b2da2c20be",
|
|
"job": {}
|
|
}
|
|
]
|
|
},
|
|
"incidents": [
|
|
{
|
|
"scenario": "postgres_unavailable",
|
|
"http_status": 503,
|
|
"correlation": {
|
|
"trace_id": "8f800efc469ce0ef5398db2c8c335b6a",
|
|
"span_id": "e745615ccc2bda58",
|
|
"log": {
|
|
"timestamp": "2026-11-22T11:28:37.928319+00:00",
|
|
"severity": "ERROR",
|
|
"service_name": "messenger-api",
|
|
"event": "http.request",
|
|
"trace_id": "8f800efc469ce0ef5398db2c8c335b6a",
|
|
"span_id": "e745615ccc2bda58",
|
|
"method": "POST",
|
|
"route": "/api/v1/users",
|
|
"status_code": 503,
|
|
"request_id": "05bba838-db0e-4aa6-ab40-29b040070f45",
|
|
"duration_seconds": 3.0050329410000813
|
|
},
|
|
"spans": [
|
|
{
|
|
"name": "POST /api/v1/users",
|
|
"kind": "SPAN_KIND_SERVER",
|
|
"spanId": "50VhXMwr2lg=",
|
|
"startTimeUnixNano": "1795346914923039414",
|
|
"endTimeUnixNano": "1795346917928534695",
|
|
"status": {
|
|
"code": "STATUS_CODE_ERROR"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"diagnostic": {
|
|
"timestamp": "2026-11-22T11:28:37.926802+00:00",
|
|
"severity": "ERROR",
|
|
"service_name": "messenger-api",
|
|
"event": "dependency.unavailable",
|
|
"trace_id": "8f800efc469ce0ef5398db2c8c335b6a",
|
|
"span_id": "e745615ccc2bda58",
|
|
"request_id": "05bba838-db0e-4aa6-ab40-29b040070f45",
|
|
"exception_type": "TimeoutError"
|
|
},
|
|
"db_span": "postgres.checkout",
|
|
"alert": "MessengerHighErrorRate",
|
|
"firing_after_seconds": 31.518,
|
|
"resolved_after_seconds": 63.702
|
|
},
|
|
{
|
|
"scenario": "worker_stopped_then_transient_retry",
|
|
"alert": "MessengerQueueLag",
|
|
"queue": {
|
|
"depth": 1.0,
|
|
"oldest_seconds": 24.761214,
|
|
"firing_after_seconds": 29.762,
|
|
"resolved": true,
|
|
"final_depth": 0
|
|
},
|
|
"recovered_status": "ready",
|
|
"trace_id": "791b7e703e211908b060e2a39d4e48e0",
|
|
"chain": [
|
|
{
|
|
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
|
"kind": "SPAN_KIND_SERVER",
|
|
"span_id": "cf1ac77789810a1f",
|
|
"parent_span_id": "",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "2450c7aa3ae880fe",
|
|
"parent_span_id": "cf1ac77789810a1f",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "b2b2c2217f3db619",
|
|
"parent_span_id": "cf1ac77789810a1f",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "ea27f177a7659807",
|
|
"parent_span_id": "cf1ac77789810a1f",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "rabbitmq.publish",
|
|
"kind": "SPAN_KIND_PRODUCER",
|
|
"span_id": "4a6dfcdc69e351c2",
|
|
"parent_span_id": "cf1ac77789810a1f",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "1"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "attachment.process.v1",
|
|
"kind": "SPAN_KIND_CONSUMER",
|
|
"span_id": "806f3c140711f68b",
|
|
"parent_span_id": "4a6dfcdc69e351c2",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "1"
|
|
},
|
|
"outcome": {
|
|
"stringValue": "error"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "postgres.claim_job",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "07b6c9b7b1b9fb14",
|
|
"parent_span_id": "806f3c140711f68b",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "4a720ce84bc25008",
|
|
"parent_span_id": "07b6c9b7b1b9fb14",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "168d3779d5b24f37",
|
|
"parent_span_id": "806f3c140711f68b",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "attachment.process.v1",
|
|
"kind": "SPAN_KIND_CONSUMER",
|
|
"span_id": "666e47e7dba62271",
|
|
"parent_span_id": "dcf62144796246df",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "2"
|
|
},
|
|
"outcome": {
|
|
"stringValue": "error"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "postgres.claim_job",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "747f3013c022c4ed",
|
|
"parent_span_id": "666e47e7dba62271",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "9873c8097758c2b5",
|
|
"parent_span_id": "747f3013c022c4ed",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "67fae4728d3811be",
|
|
"parent_span_id": "666e47e7dba62271",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "attachment.process.v1",
|
|
"kind": "SPAN_KIND_CONSUMER",
|
|
"span_id": "f88b949658c8f409",
|
|
"parent_span_id": "ea7db8f547bda4d2",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "3"
|
|
},
|
|
"outcome": {
|
|
"stringValue": "error"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "postgres.claim_job",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "e6a34e589b2b648d",
|
|
"parent_span_id": "f88b949658c8f409",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "53d56065ddee16c5",
|
|
"parent_span_id": "e6a34e589b2b648d",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "ac6d2b5e56697439",
|
|
"parent_span_id": "f88b949658c8f409",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "attachment.process.v1",
|
|
"kind": "SPAN_KIND_CONSUMER",
|
|
"span_id": "1386e063d11904e4",
|
|
"parent_span_id": "b85223b834e7224f",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "4"
|
|
},
|
|
"outcome": {
|
|
"stringValue": "error"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "postgres.claim_job",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "2fde6b64e9c941e5",
|
|
"parent_span_id": "1386e063d11904e4",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "ae13fa0652de10f6",
|
|
"parent_span_id": "2fde6b64e9c941e5",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "1982e1f794ce6ebc",
|
|
"parent_span_id": "1386e063d11904e4",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "rabbitmq.publish",
|
|
"kind": "SPAN_KIND_PRODUCER",
|
|
"span_id": "dcf62144796246df",
|
|
"parent_span_id": "cf1ac77789810a1f",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "2"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "rabbitmq.publish",
|
|
"kind": "SPAN_KIND_PRODUCER",
|
|
"span_id": "ea7db8f547bda4d2",
|
|
"parent_span_id": "cf1ac77789810a1f",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "3"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "rabbitmq.publish",
|
|
"kind": "SPAN_KIND_PRODUCER",
|
|
"span_id": "b85223b834e7224f",
|
|
"parent_span_id": "cf1ac77789810a1f",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "4"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "rabbitmq.publish",
|
|
"kind": "SPAN_KIND_PRODUCER",
|
|
"span_id": "ed1c95c1e48ebfdf",
|
|
"parent_span_id": "cf1ac77789810a1f",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "5"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "rabbitmq.publish",
|
|
"kind": "SPAN_KIND_PRODUCER",
|
|
"span_id": "9dcd7c3b7beab549",
|
|
"parent_span_id": "cf1ac77789810a1f",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "6"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "attachment.process.v1",
|
|
"kind": "SPAN_KIND_CONSUMER",
|
|
"span_id": "a8fb8ad60d7f0ee5",
|
|
"parent_span_id": "ed1c95c1e48ebfdf",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "5"
|
|
},
|
|
"outcome": {
|
|
"stringValue": "error"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "postgres.claim_job",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "aabc9e47a24348ab",
|
|
"parent_span_id": "a8fb8ad60d7f0ee5",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "afe21fb8aa86ba09",
|
|
"parent_span_id": "aabc9e47a24348ab",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "9eeccadf992914a5",
|
|
"parent_span_id": "a8fb8ad60d7f0ee5",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "attachment.process.v1",
|
|
"kind": "SPAN_KIND_CONSUMER",
|
|
"span_id": "59b1b50f324fe027",
|
|
"parent_span_id": "9dcd7c3b7beab549",
|
|
"job": {
|
|
"job.attempt": {
|
|
"intValue": "6"
|
|
},
|
|
"outcome": {
|
|
"stringValue": "success"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "postgres.claim_job",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "3d663bab303cc626",
|
|
"parent_span_id": "59b1b50f324fe027",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "81c4466b26233d49",
|
|
"parent_span_id": "3d663bab303cc626",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "s3.read_staging",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "b998c69c224f8495",
|
|
"parent_span_id": "59b1b50f324fe027",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.checkout",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "1a9414b035ae3c60",
|
|
"parent_span_id": "59b1b50f324fe027",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "s3.write_result",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "9580361a67ae3bfb",
|
|
"parent_span_id": "59b1b50f324fe027",
|
|
"job": {}
|
|
},
|
|
{
|
|
"name": "postgres.commit_ready",
|
|
"kind": "SPAN_KIND_INTERNAL",
|
|
"span_id": "a02dd09ac80de3b1",
|
|
"parent_span_id": "59b1b50f324fe027",
|
|
"job": {}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"scenario": "loki_unavailable",
|
|
"successful_messages": 204,
|
|
"dropped_after_export_failure": 8.0,
|
|
"dropped_at_enqueue": 0.0,
|
|
"samples": [
|
|
{
|
|
"seconds": 0.08,
|
|
"rss_bytes": 214970368.0,
|
|
"queued_batches": 0.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 5.08,
|
|
"rss_bytes": 215224320.0,
|
|
"queued_batches": 4.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 10.11,
|
|
"rss_bytes": 215232512.0,
|
|
"queued_batches": 9.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 15.11,
|
|
"rss_bytes": 215232512.0,
|
|
"queued_batches": 14.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 20.08,
|
|
"rss_bytes": 215244800.0,
|
|
"queued_batches": 19.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 25.16,
|
|
"rss_bytes": 215281664.0,
|
|
"queued_batches": 23.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 30.15,
|
|
"rss_bytes": 215285760.0,
|
|
"queued_batches": 28.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 35.1,
|
|
"rss_bytes": 217411584.0,
|
|
"queued_batches": 32.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 40.08,
|
|
"rss_bytes": 217411584.0,
|
|
"queued_batches": 37.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 45.1,
|
|
"rss_bytes": 217419776.0,
|
|
"queued_batches": 42.0,
|
|
"capacity_batches": 256.0
|
|
},
|
|
{
|
|
"seconds": 50.12,
|
|
"rss_bytes": 217419776.0,
|
|
"queued_batches": 47.0,
|
|
"capacity_batches": 256.0
|
|
}
|
|
],
|
|
"log_export_recovered": true
|
|
}
|
|
],
|
|
"privacy_checked_logs": 264,
|
|
"checks": {
|
|
"three_backends": true,
|
|
"dashboard_queries": true,
|
|
"server_consumer_traces": true,
|
|
"log_trace_correlation": true,
|
|
"low_cardinality": true,
|
|
"privacy": true,
|
|
"history_survives_restart": true
|
|
}
|
|
}
|