From 26619c6399291a39520ca9496a4aeb0f477ee369 Mon Sep 17 00:00:00 2001 From: ITQ Date: Sat, 23 May 2026 09:44:01 +0300 Subject: [PATCH] hotfix(helm): --- deploy/argocd/victoriametrics-agent.yaml | 13 +- deploy/argocd/victoriametrics-alert.yaml | 46 ++++- deploy/helm/movienight/templates/_helpers.tpl | 18 +- .../observability/victoriametrics.yaml | 142 --------------- .../templates/observability/vmagent.yaml | 169 ------------------ deploy/helm/movienight/values.yaml | 55 +----- .../metrics/BusinessMetricsService.kt | 2 +- src/main/resources/application.yaml | 9 + 8 files changed, 67 insertions(+), 387 deletions(-) delete mode 100644 deploy/helm/movienight/templates/observability/victoriametrics.yaml delete mode 100644 deploy/helm/movienight/templates/observability/vmagent.yaml diff --git a/deploy/argocd/victoriametrics-agent.yaml b/deploy/argocd/victoriametrics-agent.yaml index 53491cb..b984493 100644 --- a/deploy/argocd/victoriametrics-agent.yaml +++ b/deploy/argocd/victoriametrics-agent.yaml @@ -12,10 +12,11 @@ spec: helm: releaseName: vmagent valuesObject: - remoteWrite: - - url: http://vmsingle-victoria-metrics-single.observability.svc:8428/api/v1/write + remoteWriteUrls: + - http://vmsingle-victoria-metrics-single-server.observability.svc:8428/api/v1/write config: - scrape_interval: 10s + global: + scrape_interval: 10s scrape_configs: - job_name: 'movienight-backend' metrics_path: /actuator/prometheus @@ -25,6 +26,12 @@ spec: - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component] action: keep regex: backend + - source_labels: [__meta_kubernetes_namespace] + action: keep + regex: movienight + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + action: keep + regex: movienight - source_labels: [__meta_kubernetes_pod_container_port_name] action: keep regex: management diff --git a/deploy/argocd/victoriametrics-alert.yaml b/deploy/argocd/victoriametrics-alert.yaml index 19d68ad..3035280 100644 --- a/deploy/argocd/victoriametrics-alert.yaml +++ b/deploy/argocd/victoriametrics-alert.yaml @@ -14,14 +14,44 @@ spec: valuesObject: server: datasource: - url: http://vmsingle-victoria-metrics-single.observability.svc:8428 - notifier: - config: | - route: - group_by: ['alertname'] - receiver: 'blackhole' - receivers: - - name: 'blackhole' + url: http://vmsingle-victoria-metrics-single-server.observability.svc:8428 + config: + alerts: + groups: + - name: movienight.rules + rules: + - alert: MovieNightBackendDown + expr: avg(up{job="movienight-backend"}) < 1 + for: 2m + labels: + severity: critical + annotations: + summary: MovieNight backend is not fully available + description: vmagent is scraping fewer healthy MovieNight backend targets than expected. + - alert: MovieNightHigh5xxRatio + expr: sum(rate(http_server_requests_seconds_count{job="movienight-backend",status=~"5..",uri!~"/actuator.*"}[5m])) / clamp_min(sum(rate(http_server_requests_seconds_count{job="movienight-backend",uri!~"/actuator.*"}[5m])), 0.001) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: MovieNight backend 5xx ratio is high + description: More than 5 percent of non-actuator HTTP requests are returning 5xx responses. + - alert: MovieNightNoScrapeData + expr: absent(up{job="movienight-backend"}) + for: 5m + labels: + severity: warning + annotations: + summary: MovieNight backend scrape data is missing + description: VictoriaMetrics has no up metric for the movienight-backend scrape job. + alertmanager: + enabled: true + config: + route: + group_by: ['alertname'] + receiver: blackhole + receivers: + - name: blackhole destination: server: https://kubernetes.default.svc diff --git a/deploy/helm/movienight/templates/_helpers.tpl b/deploy/helm/movienight/templates/_helpers.tpl index e32e83f..f86ef93 100644 --- a/deploy/helm/movienight/templates/_helpers.tpl +++ b/deploy/helm/movienight/templates/_helpers.tpl @@ -136,22 +136,6 @@ app.kubernetes.io/component: {{ $component }} {{- if .Values.observability.grafana.datasource.url -}} {{- .Values.observability.grafana.datasource.url -}} {{- else -}} -{{- printf "http://%s.%s.svc:%v" (include "movienight.victoriaMetricsName" .) .Release.Namespace .Values.observability.victoriaMetrics.service.port -}} -{{- end -}} -{{- end -}} - -{{- define "movienight.vmagentRemoteWriteURL" -}} -{{- if .Values.observability.vmagent.remoteWriteUrl -}} -{{- .Values.observability.vmagent.remoteWriteUrl -}} -{{- else -}} -{{- printf "http://%s:%v/api/v1/write" (include "movienight.victoriaMetricsName" .) .Values.observability.victoriaMetrics.service.port -}} -{{- end -}} -{{- end -}} - -{{- define "movienight.backendMetricsTarget" -}} -{{- if .Values.backend.management.enabled -}} -{{- printf "%s-backend:%v" (include "movienight.fullname" .) .Values.backend.management.port -}} -{{- else -}} -{{- printf "%s-backend:%v" (include "movienight.fullname" .) .Values.backend.service.port -}} +{{- .Values.observability.victoriaMetrics.url -}} {{- end -}} {{- end -}} diff --git a/deploy/helm/movienight/templates/observability/victoriametrics.yaml b/deploy/helm/movienight/templates/observability/victoriametrics.yaml deleted file mode 100644 index e829ec3..0000000 --- a/deploy/helm/movienight/templates/observability/victoriametrics.yaml +++ /dev/null @@ -1,142 +0,0 @@ -{{- if and .Values.observability.enabled .Values.observability.victoriaMetrics.enabled }} -{{- if .Values.observability.victoriaMetrics.persistence.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "movienight.victoriaMetricsName" . }} - labels: - {{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }} - {{- with .Values.global.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.observability.victoriaMetrics.persistence.size }} - {{- with .Values.observability.victoriaMetrics.persistence.storageClass }} - storageClassName: {{ . | quote }} - {{- end }} ---- -{{- end }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "movienight.victoriaMetricsName" . }} - labels: - {{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }} - {{- with .Values.global.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 6 }} - template: - metadata: - labels: - {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 8 }} - {{- with .Values.observability.victoriaMetrics.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.observability.victoriaMetrics.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ include "movienight.serviceAccountName" . }} - {{- with .Values.global.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.observability.victoriaMetrics.podSecurityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: victoriametrics - image: "{{ .Values.observability.victoriaMetrics.image.repository }}:{{ .Values.observability.victoriaMetrics.image.tag }}" - imagePullPolicy: {{ .Values.observability.victoriaMetrics.image.pullPolicy }} - args: - - -storageDataPath=/var/lib/victoriametrics - - -retentionPeriod={{ .Values.observability.victoriaMetrics.retentionPeriod }} - - -httpListenAddr=:{{ .Values.observability.victoriaMetrics.service.port }} - {{- with .Values.observability.victoriaMetrics.extraArgs }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.observability.victoriaMetrics.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: {{ .Values.observability.victoriaMetrics.service.port }} - protocol: TCP - readinessProbe: - httpGet: - path: /-/ready - port: http - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - livenessProbe: - httpGet: - path: /-/healthy - port: http - initialDelaySeconds: 10 - periodSeconds: 30 - timeoutSeconds: 5 - {{- with .Values.observability.victoriaMetrics.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - volumeMounts: - - name: data - mountPath: /var/lib/victoriametrics - volumes: - - name: data - {{- if .Values.observability.victoriaMetrics.persistence.enabled }} - persistentVolumeClaim: - claimName: {{ include "movienight.victoriaMetricsName" . }} - {{- else }} - emptyDir: {} - {{- end }} - {{- with .Values.observability.victoriaMetrics.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.observability.victoriaMetrics.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.observability.victoriaMetrics.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "movienight.victoriaMetricsName" . }} - labels: - {{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }} - {{- with .Values.global.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.observability.victoriaMetrics.service.type }} - ports: - - name: http - port: {{ .Values.observability.victoriaMetrics.service.port }} - targetPort: http - protocol: TCP - selector: - {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }} -{{- end }} diff --git a/deploy/helm/movienight/templates/observability/vmagent.yaml b/deploy/helm/movienight/templates/observability/vmagent.yaml deleted file mode 100644 index fdce0e4..0000000 --- a/deploy/helm/movienight/templates/observability/vmagent.yaml +++ /dev/null @@ -1,169 +0,0 @@ -{{- if and .Values.observability.enabled .Values.observability.vmagent.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "movienight.vmagentName" . }} - labels: - {{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }} - {{- with .Values.global.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -data: - scrape.yaml: | - global: - scrape_interval: {{ .Values.observability.vmagent.scrapeInterval }} - scrape_timeout: {{ .Values.observability.vmagent.scrapeTimeout }} - - scrape_configs: - {{- if .Values.backend.enabled }} - - job_name: {{ .Values.observability.vmagent.backendJobName | quote }} - metrics_path: /actuator/prometheus - static_configs: - - targets: - - {{ include "movienight.backendMetricsTarget" . | quote }} - labels: - app: {{ include "movienight.name" . | quote }} - release: {{ .Release.Name | quote }} - namespace: {{ .Release.Namespace | quote }} - {{- end }} - - job_name: {{ printf "%s-vmagent" (include "movienight.name" .) | quote }} - static_configs: - - targets: - - {{ printf "%s:%v" (include "movienight.vmagentName" .) .Values.observability.vmagent.service.port | quote }} - labels: - app: {{ include "movienight.name" . | quote }} - release: {{ .Release.Name | quote }} - namespace: {{ .Release.Namespace | quote }} - {{- if .Values.observability.victoriaMetrics.enabled }} - - job_name: {{ printf "%s-victoriametrics" (include "movienight.name" .) | quote }} - static_configs: - - targets: - - {{ printf "%s:%v" (include "movienight.victoriaMetricsName" .) .Values.observability.victoriaMetrics.service.port | quote }} - labels: - app: {{ include "movienight.name" . | quote }} - release: {{ .Release.Name | quote }} - namespace: {{ .Release.Namespace | quote }} - {{- end }} - {{- with .Values.observability.vmagent.extraScrapeConfigs }} - {{- toYaml . | nindent 6 }} - {{- end }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "movienight.vmagentName" . }} - labels: - {{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }} - {{- with .Values.global.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - replicas: 1 - selector: - matchLabels: - {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 6 }} - template: - metadata: - labels: - {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 8 }} - {{- with .Values.observability.vmagent.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.observability.vmagent.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ include "movienight.serviceAccountName" . }} - {{- with .Values.global.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.observability.vmagent.podSecurityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: vmagent - image: "{{ .Values.observability.vmagent.image.repository }}:{{ .Values.observability.vmagent.image.tag }}" - imagePullPolicy: {{ .Values.observability.vmagent.image.pullPolicy }} - args: - - -promscrape.config=/etc/vmagent/scrape.yaml - - -remoteWrite.url={{ include "movienight.vmagentRemoteWriteURL" . }} - - -httpListenAddr=:{{ .Values.observability.vmagent.service.port }} - {{- with .Values.observability.vmagent.extraArgs }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.observability.vmagent.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: {{ .Values.observability.vmagent.service.port }} - protocol: TCP - readinessProbe: - httpGet: - path: /-/ready - port: http - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - livenessProbe: - httpGet: - path: /-/healthy - port: http - initialDelaySeconds: 10 - periodSeconds: 30 - timeoutSeconds: 5 - {{- with .Values.observability.vmagent.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - volumeMounts: - - name: config - mountPath: /etc/vmagent - readOnly: true - - name: tmp - mountPath: /tmp - volumes: - - name: config - configMap: - name: {{ include "movienight.vmagentName" . }} - - name: tmp - emptyDir: {} - {{- with .Values.observability.vmagent.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.observability.vmagent.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.observability.vmagent.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "movienight.vmagentName" . }} - labels: - {{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }} - {{- with .Values.global.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.observability.vmagent.service.type }} - ports: - - name: http - port: {{ .Values.observability.vmagent.service.port }} - targetPort: http - protocol: TCP - selector: - {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 4 }} -{{- end }} diff --git a/deploy/helm/movienight/values.yaml b/deploy/helm/movienight/values.yaml index 60e1b33..d6a7dbc 100644 --- a/deploy/helm/movienight/values.yaml +++ b/deploy/helm/movienight/values.yaml @@ -93,10 +93,10 @@ backend: httpGet: path: /actuator/health port: management - initialDelaySeconds: 5 + initialDelaySeconds: 10 periodSeconds: 5 timeoutSeconds: 5 - failureThreshold: 24 + failureThreshold: 12 gateway: enabled: false @@ -121,56 +121,17 @@ routes: pathPrefix: / observability: - enabled: false + enabled: true metrics: applicationTag: movienight httpServerRequestsHistogram: true - victoriaMetrics: - enabled: true - image: - repository: docker.io/victoriametrics/victoria-metrics - tag: v1.134.0 - pullPolicy: IfNotPresent - service: - type: ClusterIP - port: 8428 - retentionPeriod: 30d - extraArgs: [] - persistence: - enabled: true - size: 10Gi - storageClass: "" - podAnnotations: {} - podLabels: {} - resources: {} - securityContext: {} - podSecurityContext: {} - nodeSelector: {} - tolerations: [] - affinity: {} vmagent: - enabled: true - image: - repository: docker.io/victoriametrics/vmagent - tag: v1.135.0 - pullPolicy: IfNotPresent - service: - type: ClusterIP - port: 8429 - scrapeInterval: 5s - scrapeTimeout: 5s - remoteWriteUrl: "" + scrapeInterval: 10s backendJobName: movienight-backend - extraArgs: [] - extraScrapeConfigs: [] - podAnnotations: {} - podLabels: {} - resources: {} - securityContext: {} - podSecurityContext: {} - nodeSelector: {} - tolerations: [] - affinity: {} + victoriaMetrics: + url: "http://vmsingle-victoria-metrics-single-server.observability.svc:8428" + operator: + enabled: true grafana: datasource: enabled: true diff --git a/src/main/kotlin/com/project/movienight/adapters/metrics/BusinessMetricsService.kt b/src/main/kotlin/com/project/movienight/adapters/metrics/BusinessMetricsService.kt index 973071c..012c9e3 100644 --- a/src/main/kotlin/com/project/movienight/adapters/metrics/BusinessMetricsService.kt +++ b/src/main/kotlin/com/project/movienight/adapters/metrics/BusinessMetricsService.kt @@ -26,7 +26,7 @@ class BusinessMetricsService( private val jellyfinSyncedItems: Counter = meterRegistry.counter("business_jellyfin_synced_items_total") private val jellyfinSyncDuration: Timer = Timer - .builder("business_jellyfin_sync_duration_seconds") + .builder("business_jellyfin_sync_duration") .publishPercentileHistogram() .register(meterRegistry) private val jellyfinSyncFailures: Counter = meterRegistry.counter("business_jellyfin_sync_failures_total") diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index a3b844c..2865e0d 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -58,9 +58,12 @@ spring: user-name-attribute: response server: + port: ${SERVER_PORT:8080} shutdown: graceful management: + server: + port: ${MANAGEMENT_SERVER_PORT:8081} endpoints: web: base-path: /actuator @@ -76,6 +79,12 @@ management: enabled: true readinessstate: enabled: true + metrics: + tags: + application: ${MANAGEMENT_METRICS_TAGS_APPLICATION:${spring.application.name}} + distribution: + percentiles-histogram: + http.server.requests: ${MANAGEMENT_METRICS_DISTRIBUTION_PERCENTILES_HISTOGRAM_HTTP_SERVER_REQUESTS:false} info: env: enabled: true