feat: setup observability in k8s #69
@@ -0,0 +1,79 @@
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: oci://ghcr.io/grafana-community/helm-charts/grafana
|
||||
path: .
|
||||
targetRevision: 12.3.0
|
||||
helm:
|
||||
valuesObject:
|
||||
envValueFrom:
|
||||
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
|
||||
secretKeyRef:
|
||||
name: grafana-secrets
|
||||
key: client-secret
|
||||
|
||||
grafana.ini:
|
||||
server:
|
||||
root_url: https://grafana.internal.itqdev.xyz
|
||||
|
||||
auth:
|
||||
disable_login_form: false
|
||||
oauth_auto_login: false
|
||||
|
||||
auth.generic_oauth:
|
||||
enabled: true
|
||||
name: Keycloak
|
||||
allow_sign_up: true
|
||||
client_id: grafana-private
|
||||
client_secret: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET}
|
||||
scopes: openid profile email
|
||||
use_pkce: true
|
||||
|
||||
auth_url: https://id.itqdev.xyz/realms/master/protocol/openid-connect/auth
|
||||
token_url: https://id.itqdev.xyz/realms/master/protocol/openid-connect/token
|
||||
api_url: https://id.itqdev.xyz/realms/master/protocol/openid-connect/userinfo
|
||||
|
||||
role_attribute_path: >
|
||||
contains(groups[*], 'admin') && 'Admin' ||
|
||||
contains(groups[*], 'editor') && 'Editor' ||
|
||||
'Viewer'
|
||||
role_attribute_strict: false
|
||||
|
||||
use_refresh_token: true
|
||||
id_token_attribute_name: preferred_username
|
||||
|
||||
sidecar:
|
||||
dashboards:
|
||||
enabled: true
|
||||
label: grafana_dashboard
|
||||
labelValue: "1"
|
||||
searchNamespace: ALL
|
||||
folderAnnotation: grafana_folder
|
||||
provider:
|
||||
folder: MovieNight
|
||||
allowUiUpdates: false
|
||||
datasources:
|
||||
enabled: true
|
||||
label: grafana_datasource
|
||||
labelValue: "1"
|
||||
searchNamespace: ALL
|
||||
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: grafana
|
||||
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
enabled: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ApplyOutOfSyncOnly=true
|
||||
- ServerSideApply=true
|
||||
@@ -0,0 +1,58 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: victoriametrics-agent
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://victoriametrics.github.io/helm-charts/
|
||||
chart: victoria-metrics-agent
|
||||
targetRevision: 0.11.0
|
||||
helm:
|
||||
releaseName: vmagent
|
||||
valuesObject:
|
||||
remoteWriteUrls:
|
||||
- http://vmsingle-victoria-metrics-single-server.observability.svc:8428/api/v1/write
|
||||
config:
|
||||
global:
|
||||
scrape_interval: 10s
|
||||
scrape_configs:
|
||||
- job_name: 'movienight-backend'
|
||||
metrics_path: /actuator/prometheus
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
|
||||
action: keep
|
||||
regex: backend
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: keep
|
||||
regex: movienight
|
||||
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
|
||||
action: keep
|
||||
regex: movienight
|
||||
- source_labels: [__meta_kubernetes_pod_container_port_name]
|
||||
action: keep
|
||||
regex: management
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: replace
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
action: replace
|
||||
target_label: pod
|
||||
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: observability
|
||||
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ApplyOutOfSyncOnly=true
|
||||
- ServerSideApply=true
|
||||
@@ -0,0 +1,67 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: victoriametrics-alert
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://victoriametrics.github.io/helm-charts/
|
||||
chart: victoria-metrics-alert
|
||||
targetRevision: 0.12.0
|
||||
helm:
|
||||
releaseName: vmalert
|
||||
valuesObject:
|
||||
server:
|
||||
datasource:
|
||||
url: http://vmsingle-victoria-metrics-single-server.observability.svc:8428
|
||||
config:
|
||||
alerts:
|
||||
groups:
|
||||
- name: movienight.rules
|
||||
rules:
|
||||
- alert: MovieNightBackendDown
|
||||
expr: avg(up{job="movienight-backend"}) < 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: MovieNight backend is not fully available
|
||||
description: vmagent is scraping fewer healthy MovieNight backend targets than expected.
|
||||
- alert: MovieNightHigh5xxRatio
|
||||
expr: sum(rate(http_server_requests_seconds_count{job="movienight-backend",status=~"5..",uri!~"/actuator.*"}[5m])) / clamp_min(sum(rate(http_server_requests_seconds_count{job="movienight-backend",uri!~"/actuator.*"}[5m])), 0.001) > 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: MovieNight backend 5xx ratio is high
|
||||
description: More than 5 percent of non-actuator HTTP requests are returning 5xx responses.
|
||||
- alert: MovieNightNoScrapeData
|
||||
expr: absent(up{job="movienight-backend"})
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: MovieNight backend scrape data is missing
|
||||
description: VictoriaMetrics has no up metric for the movienight-backend scrape job.
|
||||
alertmanager:
|
||||
enabled: true
|
||||
config:
|
||||
route:
|
||||
group_by: ['alertname']
|
||||
receiver: blackhole
|
||||
receivers:
|
||||
- name: blackhole
|
||||
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: observability
|
||||
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ApplyOutOfSyncOnly=true
|
||||
- ServerSideApply=true
|
||||
@@ -0,0 +1,34 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: victoriametrics-operator
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://victoriametrics.github.io/helm-charts/
|
||||
chart: victoria-metrics-operator
|
||||
targetRevision: 0.38.0
|
||||
helm:
|
||||
releaseName: victoriametrics-operator
|
||||
valuesObject:
|
||||
admissionWebhooks:
|
||||
enabled: false
|
||||
createCRD: true
|
||||
operator:
|
||||
# This enables the operator to watch for CRs in all namespaces
|
||||
# or we can specify namespaces.
|
||||
disableNamespaceRestriction: true
|
||||
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: observability
|
||||
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ApplyOutOfSyncOnly=true
|
||||
- ServerSideApply=true
|
||||
@@ -0,0 +1,32 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: victoriametrics-single
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://victoriametrics.github.io/helm-charts/
|
||||
chart: victoria-metrics-single
|
||||
targetRevision: 0.20.0
|
||||
helm:
|
||||
releaseName: vmsingle
|
||||
valuesObject:
|
||||
server:
|
||||
retentionPeriod: 30d
|
||||
persistentVolume:
|
||||
enabled: true
|
||||
size: 4Gi
|
||||
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: observability
|
||||
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- ApplyOutOfSyncOnly=true
|
||||
- ServerSideApply=true
|
||||
@@ -123,3 +123,19 @@ app.kubernetes.io/component: {{ $component }}
|
||||
key: password
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "movienight.victoriaMetricsName" -}}
|
||||
{{- printf "%s-victoriametrics" (include "movienight.fullname" .) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "movienight.vmagentName" -}}
|
||||
{{- printf "%s-vmagent" (include "movienight.fullname" .) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "movienight.victoriaMetricsURL" -}}
|
||||
{{- if .Values.observability.grafana.datasource.url -}}
|
||||
{{- .Values.observability.grafana.datasource.url -}}
|
||||
{{- else -}}
|
||||
{{- .Values.observability.victoriaMetrics.url -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
@@ -48,8 +48,21 @@ spec:
|
||||
- name: http
|
||||
containerPort: {{ .Values.backend.service.port }}
|
||||
protocol: TCP
|
||||
{{- if or $postgresEnv .Values.backend.env }}
|
||||
{{- if .Values.backend.management.enabled }}
|
||||
- name: management
|
||||
containerPort: {{ .Values.backend.management.port }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
{{- if or $postgresEnv .Values.backend.env .Values.backend.management.enabled .Values.observability.enabled }}
|
||||
env:
|
||||
{{- if .Values.backend.management.enabled }}
|
||||
- name: MANAGEMENT_SERVER_PORT
|
||||
value: {{ .Values.backend.management.port | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.observability.enabled }}
|
||||
- name: MANAGEMENT_METRICS_TAGS_APPLICATION
|
||||
value: {{ .Values.observability.metrics.applicationTag | quote }}
|
||||
{{- end }}
|
||||
{{- if $postgresEnv }}
|
||||
{{- $postgresEnv | nindent 12 }}
|
||||
{{- end }}
|
||||
|
||||
@@ -16,6 +16,12 @@ spec:
|
||||
port: {{ .Values.backend.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
{{- if .Values.backend.management.enabled }}
|
||||
- name: management
|
||||
port: {{ .Values.backend.management.port }}
|
||||
targetPort: management
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
selector:
|
||||
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "backend") | nindent 4 }}
|
||||
{{- end }}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,828 @@
|
||||
{{- if and .Values.observability.enabled .Values.observability.grafana.dashboards.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "movienight.fullname" . }}-grafana-red-dashboard
|
||||
labels:
|
||||
{{- include "movienight.componentLabels" (dict "root" . "component" "grafana-dashboard") | nindent 4 }}
|
||||
{{- with .Values.observability.grafana.dashboards.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.global.annotations .Values.observability.grafana.dashboards.annotations }}
|
||||
annotations:
|
||||
{{- with .Values.global.annotations }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.observability.grafana.dashboards.annotations }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
data:
|
||||
movienight-red.json: |
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "avg(up{job=~\"$job\"})",
|
||||
"instant": false,
|
||||
"legendFormat": "up",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Availability",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval]))",
|
||||
"instant": false,
|
||||
"legendFormat": "requests",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Request Rate",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 0.01
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0.05
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",status=~\"5..\",uri!~\"/actuator.*\"}[$__rate_interval])) / clamp_min(sum(rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])), 0.001)",
|
||||
"instant": false,
|
||||
"legendFormat": "5xx ratio",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Error Ratio",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 0.5
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 0
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(http_server_requests_seconds_sum{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])) / clamp_min(sum(rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])), 0.001)",
|
||||
"instant": false,
|
||||
"legendFormat": "mean",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Mean Duration",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (method, uri, status) (rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval]))",
|
||||
"instant": false,
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Request Rate by Route",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 4
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.95, sum by (le, uri) (rate(http_server_requests_seconds_bucket{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])))",
|
||||
"instant": false,
|
||||
"legendFormat": "p95",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (uri) (rate(http_server_requests_seconds_sum{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])) / clamp_min(sum by (uri) (rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])), 0.001)",
|
||||
"instant": false,
|
||||
"legendFormat": "mean",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Duration by Route",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",status=~\"2..\",uri!~\"/actuator.*\"}[$__rate_interval]))",
|
||||
"instant": false,
|
||||
"legendFormat": "2xx",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",status=~\"4..\",uri!~\"/actuator.*\"}[$__rate_interval]))",
|
||||
"instant": false,
|
||||
"legendFormat": "4xx",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",status=~\"5..\",uri!~\"/actuator.*\"}[$__rate_interval]))",
|
||||
"instant": false,
|
||||
"legendFormat": "5xx",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Requests by Status Class",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 12
|
||||
},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(process_cpu_usage{job=~\"$job\"})",
|
||||
"instant": false,
|
||||
"legendFormat": "process CPU",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(jvm_memory_used_bytes{job=~\"$job\",area=\"heap\"})",
|
||||
"instant": false,
|
||||
"legendFormat": "heap used",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Runtime Signals",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"preload": false,
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 42,
|
||||
"tags": [
|
||||
"movienight",
|
||||
"red"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": {{ .Values.observability.grafana.datasource.name | quote }},
|
||||
"value": {{ .Values.observability.grafana.datasource.uid | quote }}
|
||||
},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"multi": false,
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": {{ .Values.observability.vmagent.backendJobName | quote }},
|
||||
"value": {{ .Values.observability.vmagent.backendJobName | quote }}
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(http_server_requests_seconds_count, job)",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Job",
|
||||
"multi": false,
|
||||
"name": "job",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values(http_server_requests_seconds_count, job)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "MovieNight RED",
|
||||
"uid": "movienight-red",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,39 @@
|
||||
{{- if and .Values.observability.enabled .Values.observability.grafana.datasource.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "movienight.fullname" . }}-grafana-datasource
|
||||
labels:
|
||||
{{- include "movienight.componentLabels" (dict "root" . "component" "grafana-datasource") | nindent 4 }}
|
||||
{{- with .Values.observability.grafana.datasource.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.global.annotations .Values.observability.grafana.datasource.annotations }}
|
||||
annotations:
|
||||
{{- with .Values.global.annotations }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.observability.grafana.datasource.annotations }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
data:
|
||||
victoriametrics.yaml: |
|
||||
apiVersion: 1
|
||||
prune: true
|
||||
|
||||
datasources:
|
||||
- name: {{ .Values.observability.grafana.datasource.name | quote }}
|
||||
type: prometheus
|
||||
access: proxy
|
||||
orgId: 1
|
||||
uid: {{ .Values.observability.grafana.datasource.uid | quote }}
|
||||
url: {{ include "movienight.victoriaMetricsURL" . | quote }}
|
||||
basicAuth: false
|
||||
isDefault: {{ .Values.observability.grafana.datasource.isDefault }}
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
queryTimeout: 10s
|
||||
timeInterval: {{ .Values.observability.vmagent.scrapeInterval | quote }}
|
||||
{{- end }}
|
||||
@@ -180,6 +180,18 @@
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"management": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"port": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
},
|
||||
"env": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -241,6 +253,31 @@
|
||||
"routes": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"observability": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"victoriaMetrics": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"vmagent": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
},
|
||||
"grafana": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,6 +48,9 @@ backend:
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
management:
|
||||
enabled: true
|
||||
port: 8081
|
||||
env:
|
||||
- name: SERVER_PORT
|
||||
value: "8080"
|
||||
@@ -73,7 +76,7 @@ backend:
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: http
|
||||
port: management
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
@@ -81,7 +84,7 @@ backend:
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: http
|
||||
port: management
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
@@ -89,11 +92,11 @@ backend:
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
port: management
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 24
|
||||
failureThreshold: 12
|
||||
|
||||
gateway:
|
||||
enabled: false
|
||||
@@ -116,3 +119,32 @@ routes:
|
||||
backend:
|
||||
enabled: true
|
||||
pathPrefix: /
|
||||
|
||||
observability:
|
||||
enabled: true
|
||||
metrics:
|
||||
applicationTag: movienight
|
||||
httpServerRequestsHistogram: true
|
||||
vmagent:
|
||||
scrapeInterval: 10s
|
||||
backendJobName: movienight-backend
|
||||
victoriaMetrics:
|
||||
url: "http://vmsingle-victoria-metrics-single-server.observability.svc:8428"
|
||||
operator:
|
||||
enabled: true
|
||||
grafana:
|
||||
datasource:
|
||||
enabled: true
|
||||
name: VictoriaMetrics
|
||||
uid: victoriametrics
|
||||
labels:
|
||||
grafana_datasource: "1"
|
||||
annotations: {}
|
||||
url: ""
|
||||
isDefault: true
|
||||
dashboards:
|
||||
enabled: true
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: MovieNight
|
||||
|
||||
@@ -26,7 +26,7 @@ class BusinessMetricsService(
|
||||
private val jellyfinSyncedItems: Counter = meterRegistry.counter("business_jellyfin_synced_items_total")
|
||||
private val jellyfinSyncDuration: Timer =
|
||||
Timer
|
||||
.builder("business_jellyfin_sync_duration_seconds")
|
||||
.builder("business_jellyfin_sync_duration")
|
||||
.publishPercentileHistogram()
|
||||
.register(meterRegistry)
|
||||
private val jellyfinSyncFailures: Counter = meterRegistry.counter("business_jellyfin_sync_failures_total")
|
||||
|
||||
@@ -58,9 +58,12 @@ spring:
|
||||
user-name-attribute: response
|
||||
|
||||
server:
|
||||
port: ${SERVER_PORT:8080}
|
||||
shutdown: graceful
|
||||
|
||||
management:
|
||||
server:
|
||||
port: ${MANAGEMENT_SERVER_PORT:8081}
|
||||
endpoints:
|
||||
web:
|
||||
base-path: /actuator
|
||||
@@ -76,6 +79,12 @@ management:
|
||||
enabled: true
|
||||
readinessstate:
|
||||
enabled: true
|
||||
metrics:
|
||||
tags:
|
||||
application: ${MANAGEMENT_METRICS_TAGS_APPLICATION:${spring.application.name}}
|
||||
distribution:
|
||||
percentiles-histogram:
|
||||
http.server.requests: ${MANAGEMENT_METRICS_DISTRIBUTION_PERCENTILES_HISTOGRAM_HTTP_SERVER_REQUESTS:false}
|
||||
info:
|
||||
env:
|
||||
enabled: true
|
||||
|
||||
Reference in New Issue
Block a user