apiVersion: v1 data: etcd.json: | { "description": "etcd sample Grafana dashboard with Prometheus", "panels": [ { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, "id": 1, "interval": "1m", "options": { "colorMode": "none", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ] } }, "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "sum(etcd_server_has_leader{job=~\".*etcd.*\", job=\"$cluster\"})", "legendFormat": "{{cluster}} - {{namespace}}\n" } ], "title": "Up", "type": "stat" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" }, "unit": "ops" } }, "gridPos": { "h": 7, "w": 10, "x": 6, "y": 0 }, "id": 2, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "sum(rate(grpc_server_started_total{job=~\".*etcd.*\", job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))", "legendFormat": "RPC rate" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "sum(rate(grpc_server_handled_total{job=~\".*etcd.*\", job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))", "legendFormat": "RPC failed rate" } ], "title": "RPC rate", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" } } }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "id": 3, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "sum(grpc_server_started_total{job=~\".*etcd.*\",job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", "legendFormat": "Watch streams" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "sum(grpc_server_started_total{job=~\".*etcd.*\",job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", "legendFormat": "Lease streams" } ], "title": "Active streams", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" }, "unit": "bytes" } }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 25 }, "id": 4, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "etcd_mvcc_db_total_size_in_bytes{job=~\".*etcd.*\", job=\"$cluster\"}", "legendFormat": "{{instance}} DB size" } ], "title": "DB size", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" }, "unit": "s" } }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 25 }, "id": 5, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance, le))", "legendFormat": "{{instance}} WAL fsync" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance, le))", "legendFormat": "{{instance}} DB fsync" } ], "title": "Disk sync duration", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" }, "unit": "bytes" } }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 25 }, "id": 6, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "process_resident_memory_bytes{job=~\".*etcd.*\", job=\"$cluster\"}", "legendFormat": "{{instance}} resident memory" } ], "title": "Memory", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" }, "unit": "Bps" } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 50 }, "id": 7, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])", "legendFormat": "{{instance}} client traffic in" } ], "title": "Client traffic in", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" }, "unit": "Bps" } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 50 }, "id": 8, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])", "legendFormat": "{{instance}} client traffic out" } ], "title": "Client traffic out", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" }, "unit": "Bps" } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 50 }, "id": 9, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance)", "legendFormat": "{{instance}} peer traffic in" } ], "title": "Peer traffic in", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" }, "unit": "Bps" } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 50 }, "id": 10, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance)", "legendFormat": "{{instance}} peer traffic out" } ], "title": "Peer traffic out", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" } } }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 75 }, "id": 11, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\", job=\"$cluster\"}[1d])", "legendFormat": "{{instance}} total leader elections per day" } ], "title": "Raft proposals", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" } } }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 75 }, "id": 12, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\", job=\"$cluster\"}[1d])", "legendFormat": "{{instance}} total leader elections per day" } ], "title": "Total leader elections per day", "type": "timeseries" }, { "datasource": { "type": "datasource", "uid": "-- Mixed --" }, "fieldConfig": { "defaults": { "custom": { "fillOpacity": 0, "lineWidth": 2, "showPoints": "never" }, "unit": "s" } }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 75 }, "id": 13, "interval": "1m", "pluginVersion": "v10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "expr": "histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])))", "legendFormat": "{{instance}} peer round trip time" } ], "title": "Peer round trip time", "type": "timeseries" } ], "refresh": "10s", "schemaVersion": 36, "tags": [ "etcd-mixin" ], "templating": { "list": [ { "label": "Data Source", "name": "datasource", "query": "prometheus", "type": "datasource" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "label": "cluster", "name": "cluster", "query": "label_values(etcd_server_has_leader{job=~\".*etcd.*\"}, job)", "refresh": 2, "type": "query" } ] }, "time": { "from": "now-15m", "to": "now" }, "timezone": "utc", "title": "Kubernetes / etcd", "uid": "c2f4e12cdf69feb95caa41a5a1b423d9" } kind: ConfigMap metadata: name: etcd-grafana-dashboard namespace: kube-system labels: grafana_dashboard: "1"