Merge pull request #69 from devitq/feat/setup-observability-in-k8s

feat: setup observability in k8s
This commit was merged in pull request #69.
This commit is contained in:
ITQ
2026-05-23 09:50:31 +03:00
committed by GitHub
15 changed files with 2349 additions and 7 deletions
+79
View File
@@ -0,0 +1,79 @@
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: grafana
namespace: argocd
spec:
project: default
source:
repoURL: oci://ghcr.io/grafana-community/helm-charts/grafana
path: .
targetRevision: 12.3.0
helm:
valuesObject:
envValueFrom:
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
secretKeyRef:
name: grafana-secrets
key: client-secret
grafana.ini:
server:
root_url: https://grafana.internal.itqdev.xyz
auth:
disable_login_form: false
oauth_auto_login: false
auth.generic_oauth:
enabled: true
name: Keycloak
allow_sign_up: true
client_id: grafana-private
client_secret: ${GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET}
scopes: openid profile email
use_pkce: true
auth_url: https://id.itqdev.xyz/realms/master/protocol/openid-connect/auth
token_url: https://id.itqdev.xyz/realms/master/protocol/openid-connect/token
api_url: https://id.itqdev.xyz/realms/master/protocol/openid-connect/userinfo
role_attribute_path: >
contains(groups[*], 'admin') && 'Admin' ||
contains(groups[*], 'editor') && 'Editor' ||
'Viewer'
role_attribute_strict: false
use_refresh_token: true
id_token_attribute_name: preferred_username
sidecar:
dashboards:
enabled: true
label: grafana_dashboard
labelValue: "1"
searchNamespace: ALL
folderAnnotation: grafana_folder
provider:
folder: MovieNight
allowUiUpdates: false
datasources:
enabled: true
label: grafana_datasource
labelValue: "1"
searchNamespace: ALL
destination:
server: https://kubernetes.default.svc
namespace: grafana
syncPolicy:
automated:
prune: true
selfHeal: true
enabled: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
+58
View File
@@ -0,0 +1,58 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-agent
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-agent
targetRevision: 0.11.0
helm:
releaseName: vmagent
valuesObject:
remoteWriteUrls:
- http://vmsingle-victoria-metrics-single-server.observability.svc:8428/api/v1/write
config:
global:
scrape_interval: 10s
scrape_configs:
- job_name: 'movienight-backend'
metrics_path: /actuator/prometheus
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
action: keep
regex: backend
- source_labels: [__meta_kubernetes_namespace]
action: keep
regex: movienight
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
action: keep
regex: movienight
- source_labels: [__meta_kubernetes_pod_container_port_name]
action: keep
regex: management
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: pod
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
+67
View File
@@ -0,0 +1,67 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-alert
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-alert
targetRevision: 0.12.0
helm:
releaseName: vmalert
valuesObject:
server:
datasource:
url: http://vmsingle-victoria-metrics-single-server.observability.svc:8428
config:
alerts:
groups:
- name: movienight.rules
rules:
- alert: MovieNightBackendDown
expr: avg(up{job="movienight-backend"}) < 1
for: 2m
labels:
severity: critical
annotations:
summary: MovieNight backend is not fully available
description: vmagent is scraping fewer healthy MovieNight backend targets than expected.
- alert: MovieNightHigh5xxRatio
expr: sum(rate(http_server_requests_seconds_count{job="movienight-backend",status=~"5..",uri!~"/actuator.*"}[5m])) / clamp_min(sum(rate(http_server_requests_seconds_count{job="movienight-backend",uri!~"/actuator.*"}[5m])), 0.001) > 0.05
for: 5m
labels:
severity: warning
annotations:
summary: MovieNight backend 5xx ratio is high
description: More than 5 percent of non-actuator HTTP requests are returning 5xx responses.
- alert: MovieNightNoScrapeData
expr: absent(up{job="movienight-backend"})
for: 5m
labels:
severity: warning
annotations:
summary: MovieNight backend scrape data is missing
description: VictoriaMetrics has no up metric for the movienight-backend scrape job.
alertmanager:
enabled: true
config:
route:
group_by: ['alertname']
receiver: blackhole
receivers:
- name: blackhole
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
@@ -0,0 +1,34 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-operator
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-operator
targetRevision: 0.38.0
helm:
releaseName: victoriametrics-operator
valuesObject:
admissionWebhooks:
enabled: false
createCRD: true
operator:
# This enables the operator to watch for CRs in all namespaces
# or we can specify namespaces.
disableNamespaceRestriction: true
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
+32
View File
@@ -0,0 +1,32 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-single
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-single
targetRevision: 0.20.0
helm:
releaseName: vmsingle
valuesObject:
server:
retentionPeriod: 30d
persistentVolume:
enabled: true
size: 4Gi
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
@@ -123,3 +123,19 @@ app.kubernetes.io/component: {{ $component }}
key: password key: password
{{- end -}} {{- end -}}
{{- end -}} {{- end -}}
{{- define "movienight.victoriaMetricsName" -}}
{{- printf "%s-victoriametrics" (include "movienight.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "movienight.vmagentName" -}}
{{- printf "%s-vmagent" (include "movienight.fullname" .) | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- define "movienight.victoriaMetricsURL" -}}
{{- if .Values.observability.grafana.datasource.url -}}
{{- .Values.observability.grafana.datasource.url -}}
{{- else -}}
{{- .Values.observability.victoriaMetrics.url -}}
{{- end -}}
{{- end -}}
@@ -48,8 +48,21 @@ spec:
- name: http - name: http
containerPort: {{ .Values.backend.service.port }} containerPort: {{ .Values.backend.service.port }}
protocol: TCP protocol: TCP
{{- if or $postgresEnv .Values.backend.env }} {{- if .Values.backend.management.enabled }}
- name: management
containerPort: {{ .Values.backend.management.port }}
protocol: TCP
{{- end }}
{{- if or $postgresEnv .Values.backend.env .Values.backend.management.enabled .Values.observability.enabled }}
env: env:
{{- if .Values.backend.management.enabled }}
- name: MANAGEMENT_SERVER_PORT
value: {{ .Values.backend.management.port | quote }}
{{- end }}
{{- if .Values.observability.enabled }}
- name: MANAGEMENT_METRICS_TAGS_APPLICATION
value: {{ .Values.observability.metrics.applicationTag | quote }}
{{- end }}
{{- if $postgresEnv }} {{- if $postgresEnv }}
{{- $postgresEnv | nindent 12 }} {{- $postgresEnv | nindent 12 }}
{{- end }} {{- end }}
@@ -16,6 +16,12 @@ spec:
port: {{ .Values.backend.service.port }} port: {{ .Values.backend.service.port }}
targetPort: http targetPort: http
protocol: TCP protocol: TCP
{{- if .Values.backend.management.enabled }}
- name: management
port: {{ .Values.backend.management.port }}
targetPort: management
protocol: TCP
{{- end }}
selector: selector:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "backend") | nindent 4 }} {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "backend") | nindent 4 }}
{{- end }} {{- end }}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,828 @@
{{- if and .Values.observability.enabled .Values.observability.grafana.dashboards.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "movienight.fullname" . }}-grafana-red-dashboard
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "grafana-dashboard") | nindent 4 }}
{{- with .Values.observability.grafana.dashboards.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if or .Values.global.annotations .Values.observability.grafana.dashboards.annotations }}
annotations:
{{- with .Values.global.annotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.observability.grafana.dashboards.annotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
data:
movienight-red.json: |
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "avg(up{job=~\"$job\"})",
"instant": false,
"legendFormat": "up",
"range": true,
"refId": "A"
}
],
"title": "Availability",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval]))",
"instant": false,
"legendFormat": "requests",
"range": true,
"refId": "A"
}
],
"title": "Request Rate",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 0.01
},
{
"color": "red",
"value": 0.05
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",status=~\"5..\",uri!~\"/actuator.*\"}[$__rate_interval])) / clamp_min(sum(rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])), 0.001)",
"instant": false,
"legendFormat": "5xx ratio",
"range": true,
"refId": "A"
}
],
"title": "Error Ratio",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 0.5
},
{
"color": "red",
"value": 1
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(rate(http_server_requests_seconds_sum{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])) / clamp_min(sum(rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])), 0.001)",
"instant": false,
"legendFormat": "mean",
"range": true,
"refId": "A"
}
],
"title": "Mean Duration",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"id": 5,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (method, uri, status) (rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval]))",
"instant": false,
"range": true,
"refId": "A"
}
],
"title": "Request Rate by Route",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by (le, uri) (rate(http_server_requests_seconds_bucket{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])))",
"instant": false,
"legendFormat": "p95",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (uri) (rate(http_server_requests_seconds_sum{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])) / clamp_min(sum by (uri) (rate(http_server_requests_seconds_count{job=~\"$job\",uri!~\"/actuator.*\"}[$__rate_interval])), 0.001)",
"instant": false,
"legendFormat": "mean",
"range": true,
"refId": "B"
}
],
"title": "Duration by Route",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"id": 7,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",status=~\"2..\",uri!~\"/actuator.*\"}[$__rate_interval]))",
"instant": false,
"legendFormat": "2xx",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",status=~\"4..\",uri!~\"/actuator.*\"}[$__rate_interval]))",
"instant": false,
"legendFormat": "4xx",
"range": true,
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(rate(http_server_requests_seconds_count{job=~\"$job\",status=~\"5..\",uri!~\"/actuator.*\"}[$__rate_interval]))",
"instant": false,
"legendFormat": "5xx",
"range": true,
"refId": "C"
}
],
"title": "Requests by Status Class",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 12
},
"id": 8,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(process_cpu_usage{job=~\"$job\"})",
"instant": false,
"legendFormat": "process CPU",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum(jvm_memory_used_bytes{job=~\"$job\",area=\"heap\"})",
"instant": false,
"legendFormat": "heap used",
"range": true,
"refId": "B"
}
],
"title": "Runtime Signals",
"type": "timeseries"
}
],
"preload": false,
"refresh": "30s",
"schemaVersion": 42,
"tags": [
"movienight",
"red"
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": {{ .Values.observability.grafana.datasource.name | quote }},
"value": {{ .Values.observability.grafana.datasource.uid | quote }}
},
"hide": 0,
"includeAll": false,
"label": "Datasource",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"type": "datasource"
},
{
"current": {
"selected": false,
"text": {{ .Values.observability.vmagent.backendJobName | quote }},
"value": {{ .Values.observability.vmagent.backendJobName | quote }}
},
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"definition": "label_values(http_server_requests_seconds_count, job)",
"hide": 0,
"includeAll": false,
"label": "Job",
"multi": false,
"name": "job",
"options": [],
"query": {
"query": "label_values(http_server_requests_seconds_count, job)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "MovieNight RED",
"uid": "movienight-red",
"version": 1,
"weekStart": ""
}
{{- end }}
@@ -0,0 +1,39 @@
{{- if and .Values.observability.enabled .Values.observability.grafana.datasource.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "movienight.fullname" . }}-grafana-datasource
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "grafana-datasource") | nindent 4 }}
{{- with .Values.observability.grafana.datasource.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if or .Values.global.annotations .Values.observability.grafana.datasource.annotations }}
annotations:
{{- with .Values.global.annotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.observability.grafana.datasource.annotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
data:
victoriametrics.yaml: |
apiVersion: 1
prune: true
datasources:
- name: {{ .Values.observability.grafana.datasource.name | quote }}
type: prometheus
access: proxy
orgId: 1
uid: {{ .Values.observability.grafana.datasource.uid | quote }}
url: {{ include "movienight.victoriaMetricsURL" . | quote }}
basicAuth: false
isDefault: {{ .Values.observability.grafana.datasource.isDefault }}
editable: false
jsonData:
httpMethod: POST
queryTimeout: 10s
timeInterval: {{ .Values.observability.vmagent.scrapeInterval | quote }}
{{- end }}
+37
View File
@@ -180,6 +180,18 @@
}, },
"additionalProperties": true "additionalProperties": true
}, },
"management": {
"type": "object",
"properties": {
"enabled": {
"type": "boolean"
},
"port": {
"type": "integer"
}
},
"additionalProperties": true
},
"env": { "env": {
"type": "array", "type": "array",
"items": { "items": {
@@ -241,6 +253,31 @@
"routes": { "routes": {
"type": "object", "type": "object",
"additionalProperties": true "additionalProperties": true
},
"observability": {
"type": "object",
"properties": {
"enabled": {
"type": "boolean"
},
"metrics": {
"type": "object",
"additionalProperties": true
},
"victoriaMetrics": {
"type": "object",
"additionalProperties": true
},
"vmagent": {
"type": "object",
"additionalProperties": true
},
"grafana": {
"type": "object",
"additionalProperties": true
}
},
"additionalProperties": true
} }
} }
} }
+37 -5
View File
@@ -48,6 +48,9 @@ backend:
service: service:
type: ClusterIP type: ClusterIP
port: 8080 port: 8080
management:
enabled: true
port: 8081
env: env:
- name: SERVER_PORT - name: SERVER_PORT
value: "8080" value: "8080"
@@ -73,7 +76,7 @@ backend:
livenessProbe: livenessProbe:
httpGet: httpGet:
path: /actuator/health/liveness path: /actuator/health/liveness
port: http port: management
initialDelaySeconds: 20 initialDelaySeconds: 20
periodSeconds: 10 periodSeconds: 10
timeoutSeconds: 5 timeoutSeconds: 5
@@ -81,7 +84,7 @@ backend:
readinessProbe: readinessProbe:
httpGet: httpGet:
path: /actuator/health/readiness path: /actuator/health/readiness
port: http port: management
initialDelaySeconds: 10 initialDelaySeconds: 10
periodSeconds: 10 periodSeconds: 10
timeoutSeconds: 5 timeoutSeconds: 5
@@ -89,11 +92,11 @@ backend:
startupProbe: startupProbe:
httpGet: httpGet:
path: /actuator/health path: /actuator/health
port: http port: management
initialDelaySeconds: 5 initialDelaySeconds: 10
periodSeconds: 5 periodSeconds: 5
timeoutSeconds: 5 timeoutSeconds: 5
failureThreshold: 24 failureThreshold: 12
gateway: gateway:
enabled: false enabled: false
@@ -116,3 +119,32 @@ routes:
backend: backend:
enabled: true enabled: true
pathPrefix: / pathPrefix: /
observability:
enabled: true
metrics:
applicationTag: movienight
httpServerRequestsHistogram: true
vmagent:
scrapeInterval: 10s
backendJobName: movienight-backend
victoriaMetrics:
url: "http://vmsingle-victoria-metrics-single-server.observability.svc:8428"
operator:
enabled: true
grafana:
datasource:
enabled: true
name: VictoriaMetrics
uid: victoriametrics
labels:
grafana_datasource: "1"
annotations: {}
url: ""
isDefault: true
dashboards:
enabled: true
labels:
grafana_dashboard: "1"
annotations:
grafana_folder: MovieNight
@@ -26,7 +26,7 @@ class BusinessMetricsService(
private val jellyfinSyncedItems: Counter = meterRegistry.counter("business_jellyfin_synced_items_total") private val jellyfinSyncedItems: Counter = meterRegistry.counter("business_jellyfin_synced_items_total")
private val jellyfinSyncDuration: Timer = private val jellyfinSyncDuration: Timer =
Timer Timer
.builder("business_jellyfin_sync_duration_seconds") .builder("business_jellyfin_sync_duration")
.publishPercentileHistogram() .publishPercentileHistogram()
.register(meterRegistry) .register(meterRegistry)
private val jellyfinSyncFailures: Counter = meterRegistry.counter("business_jellyfin_sync_failures_total") private val jellyfinSyncFailures: Counter = meterRegistry.counter("business_jellyfin_sync_failures_total")
+9
View File
@@ -58,9 +58,12 @@ spring:
user-name-attribute: response user-name-attribute: response
server: server:
port: ${SERVER_PORT:8080}
shutdown: graceful shutdown: graceful
management: management:
server:
port: ${MANAGEMENT_SERVER_PORT:8081}
endpoints: endpoints:
web: web:
base-path: /actuator base-path: /actuator
@@ -76,6 +79,12 @@ management:
enabled: true enabled: true
readinessstate: readinessstate:
enabled: true enabled: true
metrics:
tags:
application: ${MANAGEMENT_METRICS_TAGS_APPLICATION:${spring.application.name}}
distribution:
percentiles-histogram:
http.server.requests: ${MANAGEMENT_METRICS_DISTRIBUTION_PERCENTILES_HISTOGRAM_HTTP_SERVER_REQUESTS:false}
info: info:
env: env:
enabled: true enabled: true