mirror of
https://github.com/devitq/homelab.git
synced 2026-11-21 08:27:40 +00:00
2154 lines
84 KiB
YAML
2154 lines
84 KiB
YAML
apiVersion: v1
|
|
data:
|
|
cilium-envoy-upstream.json: |
|
|
{
|
|
"apiVersion": "dashboard.grafana.app/v2",
|
|
"kind": "Dashboard",
|
|
"metadata": {
|
|
"name": "cilium-envoy-upstream",
|
|
"namespace": "default",
|
|
"uid": "8ff01bf5-fa52-4e7a-bd73-6e177d44a369",
|
|
"resourceVersion": "1788358446375990",
|
|
"generation": 1,
|
|
"creationTimestamp": "2026-09-02T14:14:06Z",
|
|
"labels": {
|
|
"grafana.app/deprecatedInternalID": "979467957358592"
|
|
},
|
|
"annotations": {
|
|
"grafana.app/createdBy": "user:cfx1picle4dmod",
|
|
"grafana.app/saved-from-ui": "Grafana v13.2.0 (f681b1359f)"
|
|
}
|
|
},
|
|
"spec": {
|
|
"annotations": [
|
|
{
|
|
"kind": "AnnotationQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "grafana",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "-- Grafana --"
|
|
},
|
|
"spec": {}
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"builtIn": true,
|
|
"legacyOptions": {
|
|
"type": "dashboard"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"cursorSync": "Off",
|
|
"description": "Detailed upstream cluster monitoring for Envoy proxy. Tracks request rates, latency distributions (P50/P95/P99), success rates, connection health, circuit breaker status, retry behavior, and timeout patterns for each upstream cluster. Use this dashboard to troubleshoot backend service issues, identify performance bottlenecks, and monitor cluster health across all Envoy pods. Supports multi-cluster selection for comparative analysis. The dashboards were generated using [envoy-mixin](https://github.com/adinhodovic/envoy-mixin). Open issues and create feature requests in the repository.",
|
|
"editable": false,
|
|
"elements": {
|
|
"panel-10": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 10,
|
|
"title": "Upstream Success Rate (Including 4xx errors)",
|
|
"description": "Overall success rate counting both 4xx and 5xx as failures. Reflects end-user experience including authentication, authorization, and validation errors. Lower rates may indicate API contract mismatches, breaking changes, or integration issues between services.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_xx{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n,\n envoy_response_code_class!~\"4|5\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n envoy_cluster_upstream_rq_xx{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n)\n* 100\n",
|
|
"legendFormat": "Success Rate"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"custom": {
|
|
"axisSoftMax": 100,
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-11": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 11,
|
|
"title": "Upstream Rate by Code Class",
|
|
"description": "Request rate breakdown by HTTP status code class (2xx/3xx/4xx/5xx). Normal traffic shows dominant 2xx responses. Sudden 4xx spikes suggest client-side issues or API changes. Any 5xx indicates backend failures. Monitor 5xx rate to detect cascading failures early.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_xx{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_response_code_class)\n",
|
|
"legendFormat": "{{ envoy_response_code_class }}xx"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-12": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 12,
|
|
"title": "Upstream Rate by Code",
|
|
"description": "Detailed request rate by specific HTTP status code (200, 404, 500, etc.). Use this to identify specific error patterns. For example, 503 may indicate overload, 502 suggests gateway issues, and 429 shows rate limiting activation. Helps pinpoint exact failure modes.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_response_code)\n",
|
|
"legendFormat": "{{ envoy_response_code }}"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-13": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 13,
|
|
"title": "Upstream Healthy Percent",
|
|
"description": "Percentage of healthy endpoints per cluster based on active health checks. 100% indicates all endpoints passing health checks. Drops suggest failing instances - check pod logs, resource usage, and health check configurations. Envoy removes unhealthy endpoints from load balancing rotation.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n envoy_cluster_membership_healthy{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }\n) by (envoy_cluster_name)\n/\nsum(\n envoy_cluster_membership_total{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }\n) by (envoy_cluster_name)\n* 100\n",
|
|
"legendFormat": "{{ envoy_cluster_name }}"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"custom": {
|
|
"axisSoftMax": 100,
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-14": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 14,
|
|
"title": "Upstream Concurrency",
|
|
"description": "Current request and connection concurrency for the selected upstream cluster. Rising active requests with relatively flat connections points to request-level pressure. Rising connections and connection overflow points to connection-level pressure.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n envoy_cluster_upstream_rq_active{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Active Requests"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n envoy_cluster_upstream_cx_active{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Active Connections"
|
|
}
|
|
},
|
|
"refId": "B",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"fillOpacity": 10
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-15": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 15,
|
|
"title": "Upstream Pending and Overflow",
|
|
"description": "Pending queue activity and overflow signals for the selected upstream cluster. Pending overflow can indicate connection-pool overflow or request-breaker overflow, so a high overflow rate with low pending-active can still mean request concurrency pressure rather than a full queue.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n envoy_cluster_upstream_rq_pending_active{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Pending Requests Active"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_pending_total{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Pending Requests Rate"
|
|
}
|
|
},
|
|
"refId": "B",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_pending_overflow{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Pending Overflow Rate"
|
|
}
|
|
},
|
|
"refId": "C",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n increase(\n envoy_cluster_upstream_cx_overflow{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Connection Overflow Rate"
|
|
}
|
|
},
|
|
"refId": "D",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"fillOpacity": 10
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-16": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 16,
|
|
"title": "Upstream Connections",
|
|
"description": "Connection lifecycle metrics for the selected upstream cluster. Overflow indicates connection-related breaker pressure. Destroyed tracks normal or failed connection teardown. Connect failures suggest network issues, DNS problems, TLS issues, or unreachable backends.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n increase(\n envoy_cluster_upstream_cx_overflow{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Overflow Connections"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n increase(\n envoy_cluster_upstream_cx_destroy{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Destroyed Connections"
|
|
}
|
|
},
|
|
"refId": "B",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n increase(\n envoy_cluster_upstream_cx_connect_fail{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Connect Failures"
|
|
}
|
|
},
|
|
"refId": "C",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"fillOpacity": 10
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-17": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 17,
|
|
"title": "Upstream Circuit Breakers",
|
|
"description": "Circuit breaker state for the selected upstream cluster. Non-zero values indicate protection mechanisms are currently open. Open Connections tracks the connection breaker, Open Pool Connections tracks connection-pool breaker pressure, and Open Requests tracks request breaker pressure rather than only a full pending queue.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n envoy_cluster_circuit_breakers_default_cx_open{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Open Connections"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n envoy_cluster_circuit_breakers_default_cx_pool_open{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Open Pool Connections"
|
|
}
|
|
},
|
|
"refId": "B",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n envoy_cluster_circuit_breakers_default_rq_open{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Open Requests"
|
|
}
|
|
},
|
|
"refId": "C",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-18": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 18,
|
|
"title": "Upstream Resets, Timeouts, and Retries",
|
|
"description": "Failure-mode signals for the selected upstream cluster. Tx resets often reflect local fast-fail behavior, breaker pressure, or local connection issues. Rx resets point to the upstream side closing early. Timeouts highlight slow backends, and retry overflow shows retry budget or retry concurrency pressure.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_tx_reset{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Tx Reset Rate"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_rx_reset{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Rx Reset Rate"
|
|
}
|
|
},
|
|
"refId": "B",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_timeout{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Timeout Rate"
|
|
}
|
|
},
|
|
"refId": "C",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_retry{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Retry Rate"
|
|
}
|
|
},
|
|
"refId": "D",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_retry_overflow{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "Retry Overflow Rate"
|
|
}
|
|
},
|
|
"refId": "E",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-19": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 19,
|
|
"title": "Upstream Rate by Pod",
|
|
"description": "Request distribution across Envoy proxy pods for this upstream cluster. Ideally shows even distribution. Imbalanced traffic may indicate pod scheduling issues, uneven client distribution, or connection affinity problems. Use to verify horizontal scaling effectiveness.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_total{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (pod, envoy_cluster_name)\n",
|
|
"legendFormat": "{{ pod }}"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-2": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 2,
|
|
"title": "Upstreams Count by Job",
|
|
"description": "Distribution of upstream clusters across different Prometheus job labels. Useful for understanding how backend services are organized across different Envoy deployments or environments. Imbalanced distribution may indicate configuration inconsistencies.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"expr": "count(\n envoy_cluster_upstream_rq_total{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n\n }\n) by (job)\n",
|
|
"instant": true,
|
|
"legendFormat": "{{ job }}"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "piechart",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"displayLabels": [
|
|
"percent"
|
|
],
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"values": [
|
|
"percent"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "upstreams"
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-20": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 20,
|
|
"title": "Upstream Active Connections by Pod",
|
|
"description": "Active connections per Envoy pod to this upstream cluster. Should correlate with request rates. High connections with low requests suggest connection pooling inefficiency or slow-draining connections. Uneven distribution may indicate pod-level issues requiring investigation.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n envoy_cluster_upstream_cx_active{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }\n) by (pod, envoy_cluster_name)\n",
|
|
"legendFormat": "{{ pod }}"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-3": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 3,
|
|
"title": "Upstream Rate by Envoy Cluster Name [1h]",
|
|
"description": "Request rate distribution across upstream clusters over the past hour (top 20). Identifies which backend services receive the most traffic. Use this to validate load distribution, detect traffic shifts after deployments, and identify hot services that may need scaling.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"expr": "topk(20,\n sum(\n rate(\n envoy_cluster_upstream_rq_total{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n\n }[1h]\n )\n ) by (envoy_cluster_name)\n)\n",
|
|
"instant": true,
|
|
"legendFormat": "{{ envoy_cluster_name }}"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "piechart",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"displayLabels": [
|
|
"percent"
|
|
],
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"values": [
|
|
"percent"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-4": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 4,
|
|
"title": "Upstream Rate by Code Class [1h]",
|
|
"description": "Breakdown of upstream responses by HTTP status code class over the past hour. Healthy services typically show 95%+ 2xx responses. High 4xx proportions suggest API contract issues or client misconfigurations. Any 5xx responses indicate backend failures requiring immediate investigation.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_xx{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n\n }[1h]\n )\n) by (envoy_response_code_class)\n",
|
|
"instant": true,
|
|
"legendFormat": "{{ envoy_response_code_class }}xx"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "piechart",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"displayLabels": [
|
|
"percent"
|
|
],
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"values": [
|
|
"percent"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-5": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 5,
|
|
"title": "Upstream Active Connections by Envoy Cluster Name",
|
|
"description": "Distribution of currently active TCP connections to upstream services. Shows which backend clusters are consuming the most connections. Disproportionately high connection counts may indicate connection pooling issues, slow backends, or HTTP/1.1 connection inefficiency. Compare with request rates to assess connection reuse.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"expr": "topk(20,\n sum(\n envoy_cluster_upstream_cx_active{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n\n }\n ) by (envoy_cluster_name)\n)\n",
|
|
"instant": true,
|
|
"legendFormat": "{{ envoy_cluster_name }}"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "piechart",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"displayLabels": [
|
|
"percent"
|
|
],
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"values": [
|
|
"percent"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-7": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 7,
|
|
"title": "Upstream Rate",
|
|
"description": "Request rate per upstream cluster over time. Each line represents traffic to a specific backend service. Sudden drops may indicate circuit breaker activation, upstream failures, or routing changes. Gradual increases suggest growing load. Use this to identify which clusters need scaling or optimization.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_total{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n) by (envoy_cluster_name)\n",
|
|
"legendFormat": "{{ envoy_cluster_name }}"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-8": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 8,
|
|
"title": "Upstream Latency",
|
|
"description": "Backend service response time percentiles. P50 represents typical latency, while P95/P99 show tail latency affecting a subset of requests. Rising P99 often precedes visible performance degradation. Investigate backends when P95 exceeds SLO targets. Exemplars link to distributed traces for root cause analysis.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "histogram_quantile(\n 0.5,\n sum(\n rate(\n envoy_cluster_upstream_rq_time_bucket{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n ) by (le)\n)\n",
|
|
"legendFormat": "P50"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "histogram_quantile(\n 0.95,\n sum(\n rate(\n envoy_cluster_upstream_rq_time_bucket{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n ) by (le)\n)\n",
|
|
"legendFormat": "P95"
|
|
}
|
|
},
|
|
"refId": "B",
|
|
"hidden": false
|
|
}
|
|
},
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(\n 0.99,\n sum(\n rate(\n envoy_cluster_upstream_rq_time_bucket{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n ) by (le)\n)\n",
|
|
"legendFormat": "P99"
|
|
}
|
|
},
|
|
"refId": "C",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"custom": {
|
|
"fillOpacity": 10
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"panel-9": {
|
|
"kind": "Panel",
|
|
"spec": {
|
|
"id": 9,
|
|
"title": "Upstream Success Rate (Excluding 4xx errors)",
|
|
"description": "Backend success rate treating 4xx as successful (client errors, not backend failures). Drops below 99.9% indicate backend health issues. Correlate with circuit breaker metrics, health check status, and backend logs. Sustained low rates may trigger automatic circuit breaking.",
|
|
"links": [],
|
|
"data": {
|
|
"kind": "QueryGroup",
|
|
"spec": {
|
|
"queries": [
|
|
{
|
|
"kind": "PanelQuery",
|
|
"spec": {
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "$datasource"
|
|
},
|
|
"spec": {
|
|
"exemplar": false,
|
|
"expr": "sum(\n rate(\n envoy_cluster_upstream_rq_xx{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n,\n envoy_response_code_class!=\"5\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n envoy_cluster_upstream_rq_xx{\n cluster=\"$cluster\",\nnamespace=~\"$namespace\",\njob=~\"$job\",\npod=~\"$pod\"\n\n,\nenvoy_cluster_name=\"$envoy_cluster_name\"\n\n }[$__rate_interval]\n )\n)\n* 100\n",
|
|
"legendFormat": "Success Rate"
|
|
}
|
|
},
|
|
"refId": "A",
|
|
"hidden": false
|
|
}
|
|
}
|
|
],
|
|
"transformations": [],
|
|
"queryOptions": {}
|
|
}
|
|
},
|
|
"vizConfig": {
|
|
"kind": "VizConfig",
|
|
"group": "timeseries",
|
|
"version": "v11.4.0",
|
|
"spec": {
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"showLegend": true,
|
|
"sortBy": "mean",
|
|
"sortDesc": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"custom": {
|
|
"axisSoftMax": 100,
|
|
"axisSoftMin": 0,
|
|
"fillOpacity": 100,
|
|
"lineWidth": 1,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"layout": {
|
|
"kind": "RowsLayout",
|
|
"spec": {
|
|
"rows": [
|
|
{
|
|
"kind": "RowsLayoutRow",
|
|
"spec": {
|
|
"title": "Summary",
|
|
"collapse": false,
|
|
"layout": {
|
|
"kind": "GridLayout",
|
|
"spec": {
|
|
"items": [
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 0,
|
|
"y": 4,
|
|
"width": 6,
|
|
"height": 6,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-2"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 6,
|
|
"y": 4,
|
|
"width": 6,
|
|
"height": 6,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-3"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 12,
|
|
"y": 4,
|
|
"width": 6,
|
|
"height": 6,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-4"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 18,
|
|
"y": 4,
|
|
"width": 6,
|
|
"height": 6,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-5"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "RowsLayoutRow",
|
|
"spec": {
|
|
"title": "$envoy_cluster_name",
|
|
"collapse": false,
|
|
"repeat": {
|
|
"mode": "variable",
|
|
"value": "envoy_cluster_name"
|
|
},
|
|
"layout": {
|
|
"kind": "GridLayout",
|
|
"spec": {
|
|
"items": [
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-7"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 12,
|
|
"y": 0,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-8"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 0,
|
|
"y": 8,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-9"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 12,
|
|
"y": 8,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-10"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 0,
|
|
"y": 16,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-11"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 12,
|
|
"y": 16,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-12"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 0,
|
|
"y": 24,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-13"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 12,
|
|
"y": 24,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-14"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 0,
|
|
"y": 32,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-15"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 12,
|
|
"y": 32,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-16"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 0,
|
|
"y": 40,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-17"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 12,
|
|
"y": 40,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-18"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 0,
|
|
"y": 48,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-19"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"kind": "GridLayoutItem",
|
|
"spec": {
|
|
"x": 12,
|
|
"y": 48,
|
|
"width": 12,
|
|
"height": 8,
|
|
"element": {
|
|
"kind": "ElementReference",
|
|
"name": "panel-20"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"links": [
|
|
{
|
|
"title": "Envoy",
|
|
"type": "dashboards",
|
|
"icon": "",
|
|
"tooltip": "",
|
|
"tags": [
|
|
"envoy",
|
|
"envoy-mixin",
|
|
"gateway"
|
|
],
|
|
"asDropdown": true,
|
|
"targetBlank": true,
|
|
"includeVars": false,
|
|
"keepTime": true
|
|
}
|
|
],
|
|
"liveNow": false,
|
|
"preload": false,
|
|
"tags": [
|
|
"envoy",
|
|
"envoy-mixin",
|
|
"gateway"
|
|
],
|
|
"timeSettings": {
|
|
"timezone": "utc",
|
|
"from": "now-6h",
|
|
"to": "now",
|
|
"autoRefresh": "",
|
|
"autoRefreshIntervals": [
|
|
"5s",
|
|
"10s",
|
|
"30s",
|
|
"1m",
|
|
"5m",
|
|
"15m",
|
|
"30m",
|
|
"1h",
|
|
"2h",
|
|
"1d"
|
|
],
|
|
"hideTimepicker": false,
|
|
"fiscalYearStartMonth": 0
|
|
},
|
|
"title": "Cilium / Envoy / Upstream",
|
|
"variables": [
|
|
{
|
|
"kind": "DatasourceVariable",
|
|
"spec": {
|
|
"name": "datasource",
|
|
"pluginId": "prometheus",
|
|
"refresh": "never",
|
|
"regex": "",
|
|
"current": {
|
|
"text": "default",
|
|
"value": "default"
|
|
},
|
|
"options": [],
|
|
"multi": false,
|
|
"includeAll": false,
|
|
"label": "Data source",
|
|
"hide": "dontHide",
|
|
"skipUrlSync": false,
|
|
"allowCustomValue": true
|
|
}
|
|
},
|
|
{
|
|
"kind": "QueryVariable",
|
|
"spec": {
|
|
"name": "cluster",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"label": "Cluster",
|
|
"hide": "hideVariable",
|
|
"refresh": "onTimeRangeChanged",
|
|
"skipUrlSync": false,
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "${datasource}"
|
|
},
|
|
"spec": {
|
|
"__legacyStringValue": "label_values(envoy_cluster_upstream_rq_xx{}, cluster)"
|
|
}
|
|
},
|
|
"regex": "",
|
|
"sort": "alphabeticalAsc",
|
|
"options": [],
|
|
"multi": false,
|
|
"includeAll": false,
|
|
"allowCustomValue": true
|
|
}
|
|
},
|
|
{
|
|
"kind": "QueryVariable",
|
|
"spec": {
|
|
"name": "namespace",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"label": "Namespace",
|
|
"hide": "dontHide",
|
|
"refresh": "onTimeRangeChanged",
|
|
"skipUrlSync": false,
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "${datasource}"
|
|
},
|
|
"spec": {
|
|
"__legacyStringValue": "label_values(envoy_cluster_upstream_rq_xx{cluster=\"$cluster\"}, namespace)"
|
|
}
|
|
},
|
|
"regex": "",
|
|
"sort": "alphabeticalAsc",
|
|
"options": [],
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"allowCustomValue": true
|
|
}
|
|
},
|
|
{
|
|
"kind": "QueryVariable",
|
|
"spec": {
|
|
"name": "job",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"label": "Job",
|
|
"hide": "dontHide",
|
|
"refresh": "onTimeRangeChanged",
|
|
"skipUrlSync": false,
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "${datasource}"
|
|
},
|
|
"spec": {
|
|
"__legacyStringValue": "label_values(envoy_cluster_upstream_rq_xx{cluster=\"$cluster\", namespace=~\"$namespace\"}, job)"
|
|
}
|
|
},
|
|
"regex": "",
|
|
"sort": "alphabeticalAsc",
|
|
"options": [],
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"allowCustomValue": true
|
|
}
|
|
},
|
|
{
|
|
"kind": "QueryVariable",
|
|
"spec": {
|
|
"name": "envoy_cluster_name",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"label": "Envoy Cluster Name",
|
|
"hide": "dontHide",
|
|
"refresh": "onTimeRangeChanged",
|
|
"skipUrlSync": false,
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "${datasource}"
|
|
},
|
|
"spec": {
|
|
"__legacyStringValue": "label_values(envoy_cluster_upstream_rq_xx{cluster=\"$cluster\", namespace=~\"$namespace\", job=~\"$job\"}, envoy_cluster_name)"
|
|
}
|
|
},
|
|
"regex": "",
|
|
"sort": "alphabeticalAsc",
|
|
"options": [],
|
|
"multi": true,
|
|
"includeAll": false,
|
|
"allowCustomValue": true
|
|
}
|
|
},
|
|
{
|
|
"kind": "QueryVariable",
|
|
"spec": {
|
|
"name": "pod",
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"label": "Pod",
|
|
"hide": "dontHide",
|
|
"refresh": "onTimeRangeChanged",
|
|
"skipUrlSync": false,
|
|
"query": {
|
|
"kind": "DataQuery",
|
|
"group": "prometheus",
|
|
"version": "v0",
|
|
"datasource": {
|
|
"name": "${datasource}"
|
|
},
|
|
"spec": {
|
|
"__legacyStringValue": "label_values(envoy_cluster_upstream_rq_xx{cluster=\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", envoy_cluster_name=~\"$envoy_cluster_name\"}, pod)"
|
|
}
|
|
},
|
|
"regex": "",
|
|
"sort": "alphabeticalAsc",
|
|
"options": [],
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"allowCustomValue": true
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: cilium-envoy-upstream-grafana-dashboard
|
|
namespace: kube-system
|
|
labels:
|
|
grafana_dashboard: "1"
|