hotfix(helm):

deploy/argocd/victoriametrics-agent.yaml

@@ -12,10 +12,11 @@ spec:
     helm:
       releaseName: vmagent
       valuesObject:
-        remoteWrite:
+        remoteWriteUrls:
-          - url: http://vmsingle-victoria-metrics-single.observability.svc:8428/api/v1/write
+          - http://vmsingle-victoria-metrics-single-server.observability.svc:8428/api/v1/write
         config:
-          scrape_interval: 10s
+          global:
+            scrape_interval: 10s
           scrape_configs:
             - job_name: 'movienight-backend'
               metrics_path: /actuator/prometheus
@@ -25,6 +26,12 @@ spec:
                 - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
                   action: keep
                   regex: backend
+                - source_labels: [__meta_kubernetes_namespace]
+                  action: keep
+                  regex: movienight
+                - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+                  action: keep
+                  regex: movienight
                 - source_labels: [__meta_kubernetes_pod_container_port_name]
                   action: keep
                   regex: management

deploy/argocd/victoriametrics-alert.yaml

@@ -14,14 +14,44 @@ spec:
       valuesObject:
         server:
           datasource:
-            url: http://vmsingle-victoria-metrics-single.observability.svc:8428
+            url: http://vmsingle-victoria-metrics-single-server.observability.svc:8428
-          notifier:
+          config:
-            config: |
+            alerts:
-              route:
+              groups:
-                group_by: ['alertname']
+                - name: movienight.rules
-                receiver: 'blackhole'
+                  rules:
-              receivers:
+                    - alert: MovieNightBackendDown
-              - name: 'blackhole'
+                      expr: avg(up{job="movienight-backend"}) < 1
+                      for: 2m
+                      labels:
+                        severity: critical
+                      annotations:
+                        summary: MovieNight backend is not fully available
+                        description: vmagent is scraping fewer healthy MovieNight backend targets than expected.
+                    - alert: MovieNightHigh5xxRatio
+                      expr: sum(rate(http_server_requests_seconds_count{job="movienight-backend",status=~"5..",uri!~"/actuator.*"}[5m])) / clamp_min(sum(rate(http_server_requests_seconds_count{job="movienight-backend",uri!~"/actuator.*"}[5m])), 0.001) > 0.05
+                      for: 5m
+                      labels:
+                        severity: warning
+                      annotations:
+                        summary: MovieNight backend 5xx ratio is high
+                        description: More than 5 percent of non-actuator HTTP requests are returning 5xx responses.
+                    - alert: MovieNightNoScrapeData
+                      expr: absent(up{job="movienight-backend"})
+                      for: 5m
+                      labels:
+                        severity: warning
+                      annotations:
+                        summary: MovieNight backend scrape data is missing
+                        description: VictoriaMetrics has no up metric for the movienight-backend scrape job.
+        alertmanager:
+          enabled: true
+          config:
+            route:
+              group_by: ['alertname']
+              receiver: blackhole
+            receivers:
+              - name: blackhole
 
   destination:
     server: https://kubernetes.default.svc

deploy/helm/movienight/templates/_helpers.tpl

@@ -136,22 +136,6 @@ app.kubernetes.io/component: {{ $component }}
 {{- if .Values.observability.grafana.datasource.url -}}
 {{- .Values.observability.grafana.datasource.url -}}
 {{- else -}}
-{{- printf "http://%s.%s.svc:%v" (include "movienight.victoriaMetricsName" .) .Release.Namespace .Values.observability.victoriaMetrics.service.port -}}
+{{- .Values.observability.victoriaMetrics.url -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "movienight.vmagentRemoteWriteURL" -}}
-{{- if .Values.observability.vmagent.remoteWriteUrl -}}
-{{- .Values.observability.vmagent.remoteWriteUrl -}}
-{{- else -}}
-{{- printf "http://%s:%v/api/v1/write" (include "movienight.victoriaMetricsName" .) .Values.observability.victoriaMetrics.service.port -}}
-{{- end -}}
-{{- end -}}
-
-{{- define "movienight.backendMetricsTarget" -}}
-{{- if .Values.backend.management.enabled -}}
-{{- printf "%s-backend:%v" (include "movienight.fullname" .) .Values.backend.management.port -}}
-{{- else -}}
-{{- printf "%s-backend:%v" (include "movienight.fullname" .) .Values.backend.service.port -}}
 {{- end -}}
 {{- end -}}

deploy/helm/movienight/templates/observability/victoriametrics.yaml

@@ -1,142 +0,0 @@
-{{- if and .Values.observability.enabled .Values.observability.victoriaMetrics.enabled }}
-{{- if .Values.observability.victoriaMetrics.persistence.enabled }}
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: {{ include "movienight.victoriaMetricsName" . }}
-  labels:
-    {{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
-  {{- with .Values.global.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: {{ .Values.observability.victoriaMetrics.persistence.size }}
-  {{- with .Values.observability.victoriaMetrics.persistence.storageClass }}
-  storageClassName: {{ . | quote }}
-  {{- end }}
----
-{{- end }}
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "movienight.victoriaMetricsName" . }}
-  labels:
-    {{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
-  {{- with .Values.global.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 6 }}
-  template:
-    metadata:
-      labels:
-        {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 8 }}
-        {{- with .Values.observability.victoriaMetrics.podLabels }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
-      {{- with .Values.observability.victoriaMetrics.podAnnotations }}
-      annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-    spec:
-      serviceAccountName: {{ include "movienight.serviceAccountName" . }}
-      {{- with .Values.global.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.observability.victoriaMetrics.podSecurityContext }}
-      securityContext:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      containers:
-        - name: victoriametrics
-          image: "{{ .Values.observability.victoriaMetrics.image.repository }}:{{ .Values.observability.victoriaMetrics.image.tag }}"
-          imagePullPolicy: {{ .Values.observability.victoriaMetrics.image.pullPolicy }}
-          args:
-            - -storageDataPath=/var/lib/victoriametrics
-            - -retentionPeriod={{ .Values.observability.victoriaMetrics.retentionPeriod }}
-            - -httpListenAddr=:{{ .Values.observability.victoriaMetrics.service.port }}
-            {{- with .Values.observability.victoriaMetrics.extraArgs }}
-            {{- toYaml . | nindent 12 }}
-            {{- end }}
-          {{- with .Values.observability.victoriaMetrics.securityContext }}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          ports:
-            - name: http
-              containerPort: {{ .Values.observability.victoriaMetrics.service.port }}
-              protocol: TCP
-          readinessProbe:
-            httpGet:
-              path: /-/ready
-              port: http
-            initialDelaySeconds: 5
-            periodSeconds: 10
-            timeoutSeconds: 5
-          livenessProbe:
-            httpGet:
-              path: /-/healthy
-              port: http
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            timeoutSeconds: 5
-          {{- with .Values.observability.victoriaMetrics.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-            - name: data
-              mountPath: /var/lib/victoriametrics
-      volumes:
-        - name: data
-          {{- if .Values.observability.victoriaMetrics.persistence.enabled }}
-          persistentVolumeClaim:
-            claimName: {{ include "movienight.victoriaMetricsName" . }}
-          {{- else }}
-          emptyDir: {}
-          {{- end }}
-      {{- with .Values.observability.victoriaMetrics.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.observability.victoriaMetrics.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.observability.victoriaMetrics.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "movienight.victoriaMetricsName" . }}
-  labels:
-    {{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
-  {{- with .Values.global.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  type: {{ .Values.observability.victoriaMetrics.service.type }}
-  ports:
-    - name: http
-      port: {{ .Values.observability.victoriaMetrics.service.port }}
-      targetPort: http
-      protocol: TCP
-  selector:
-    {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
-{{- end }}

deploy/helm/movienight/templates/observability/vmagent.yaml

@@ -1,169 +0,0 @@
-{{- if and .Values.observability.enabled .Values.observability.vmagent.enabled }}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "movienight.vmagentName" . }}
-  labels:
-    {{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
-  {{- with .Values.global.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-data:
-  scrape.yaml: |
-    global:
-      scrape_interval: {{ .Values.observability.vmagent.scrapeInterval }}
-      scrape_timeout: {{ .Values.observability.vmagent.scrapeTimeout }}
-
-    scrape_configs:
-      {{- if .Values.backend.enabled }}
-      - job_name: {{ .Values.observability.vmagent.backendJobName | quote }}
-        metrics_path: /actuator/prometheus
-        static_configs:
-          - targets:
-              - {{ include "movienight.backendMetricsTarget" . | quote }}
-            labels:
-              app: {{ include "movienight.name" . | quote }}
-              release: {{ .Release.Name | quote }}
-              namespace: {{ .Release.Namespace | quote }}
-      {{- end }}
-      - job_name: {{ printf "%s-vmagent" (include "movienight.name" .) | quote }}
-        static_configs:
-          - targets:
-              - {{ printf "%s:%v" (include "movienight.vmagentName" .) .Values.observability.vmagent.service.port | quote }}
-            labels:
-              app: {{ include "movienight.name" . | quote }}
-              release: {{ .Release.Name | quote }}
-              namespace: {{ .Release.Namespace | quote }}
-      {{- if .Values.observability.victoriaMetrics.enabled }}
-      - job_name: {{ printf "%s-victoriametrics" (include "movienight.name" .) | quote }}
-        static_configs:
-          - targets:
-              - {{ printf "%s:%v" (include "movienight.victoriaMetricsName" .) .Values.observability.victoriaMetrics.service.port | quote }}
-            labels:
-              app: {{ include "movienight.name" . | quote }}
-              release: {{ .Release.Name | quote }}
-              namespace: {{ .Release.Namespace | quote }}
-      {{- end }}
-      {{- with .Values.observability.vmagent.extraScrapeConfigs }}
-      {{- toYaml . | nindent 6 }}
-      {{- end }}
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "movienight.vmagentName" . }}
-  labels:
-    {{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
-  {{- with .Values.global.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 6 }}
-  template:
-    metadata:
-      labels:
-        {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 8 }}
-        {{- with .Values.observability.vmagent.podLabels }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
-      {{- with .Values.observability.vmagent.podAnnotations }}
-      annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-    spec:
-      serviceAccountName: {{ include "movienight.serviceAccountName" . }}
-      {{- with .Values.global.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.observability.vmagent.podSecurityContext }}
-      securityContext:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      containers:
-        - name: vmagent
-          image: "{{ .Values.observability.vmagent.image.repository }}:{{ .Values.observability.vmagent.image.tag }}"
-          imagePullPolicy: {{ .Values.observability.vmagent.image.pullPolicy }}
-          args:
-            - -promscrape.config=/etc/vmagent/scrape.yaml
-            - -remoteWrite.url={{ include "movienight.vmagentRemoteWriteURL" . }}
-            - -httpListenAddr=:{{ .Values.observability.vmagent.service.port }}
-            {{- with .Values.observability.vmagent.extraArgs }}
-            {{- toYaml . | nindent 12 }}
-            {{- end }}
-          {{- with .Values.observability.vmagent.securityContext }}
-          securityContext:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          ports:
-            - name: http
-              containerPort: {{ .Values.observability.vmagent.service.port }}
-              protocol: TCP
-          readinessProbe:
-            httpGet:
-              path: /-/ready
-              port: http
-            initialDelaySeconds: 5
-            periodSeconds: 10
-            timeoutSeconds: 5
-          livenessProbe:
-            httpGet:
-              path: /-/healthy
-              port: http
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            timeoutSeconds: 5
-          {{- with .Values.observability.vmagent.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          volumeMounts:
-            - name: config
-              mountPath: /etc/vmagent
-              readOnly: true
-            - name: tmp
-              mountPath: /tmp
-      volumes:
-        - name: config
-          configMap:
-            name: {{ include "movienight.vmagentName" . }}
-        - name: tmp
-          emptyDir: {}
-      {{- with .Values.observability.vmagent.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.observability.vmagent.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.observability.vmagent.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ include "movienight.vmagentName" . }}
-  labels:
-    {{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
-  {{- with .Values.global.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  type: {{ .Values.observability.vmagent.service.type }}
-  ports:
-    - name: http
-      port: {{ .Values.observability.vmagent.service.port }}
-      targetPort: http
-      protocol: TCP
-  selector:
-    {{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
-{{- end }}

deploy/helm/movienight/values.yaml

@@ -93,10 +93,10 @@ backend:
     httpGet:
       path: /actuator/health
       port: management
-    initialDelaySeconds: 5
+    initialDelaySeconds: 10
     periodSeconds: 5
     timeoutSeconds: 5
-    failureThreshold: 24
+    failureThreshold: 12
 
 gateway:
   enabled: false
@@ -121,56 +121,17 @@ routes:
     pathPrefix: /
 
 observability:
-  enabled: false
+  enabled: true
   metrics:
     applicationTag: movienight
     httpServerRequestsHistogram: true
-  victoriaMetrics:
-    enabled: true
-    image:
-      repository: docker.io/victoriametrics/victoria-metrics
-      tag: v1.134.0
-      pullPolicy: IfNotPresent
-    service:
-      type: ClusterIP
-      port: 8428
-    retentionPeriod: 30d
-    extraArgs: []
-    persistence:
-      enabled: true
-      size: 10Gi
-      storageClass: ""
-    podAnnotations: {}
-    podLabels: {}
-    resources: {}
-    securityContext: {}
-    podSecurityContext: {}
-    nodeSelector: {}
-    tolerations: []
-    affinity: {}
   vmagent:
-    enabled: true
+    scrapeInterval: 10s
-    image:
-      repository: docker.io/victoriametrics/vmagent
-      tag: v1.135.0
-      pullPolicy: IfNotPresent
-    service:
-      type: ClusterIP
-      port: 8429
-    scrapeInterval: 5s
-    scrapeTimeout: 5s
-    remoteWriteUrl: ""
     backendJobName: movienight-backend
-    extraArgs: []
+  victoriaMetrics:
-    extraScrapeConfigs: []
+    url: "http://vmsingle-victoria-metrics-single-server.observability.svc:8428"
-    podAnnotations: {}
+    operator:
-    podLabels: {}
+      enabled: true
-    resources: {}
-    securityContext: {}
-    podSecurityContext: {}
-    nodeSelector: {}
-    tolerations: []
-    affinity: {}
   grafana:
     datasource:
       enabled: true

src/main/kotlin/com/project/movienight/adapters/metrics/BusinessMetricsService.kt

@@ -26,7 +26,7 @@ class BusinessMetricsService(
     private val jellyfinSyncedItems: Counter = meterRegistry.counter("business_jellyfin_synced_items_total")
     private val jellyfinSyncDuration: Timer =
         Timer
-            .builder("business_jellyfin_sync_duration_seconds")
+            .builder("business_jellyfin_sync_duration")
             .publishPercentileHistogram()
             .register(meterRegistry)
     private val jellyfinSyncFailures: Counter = meterRegistry.counter("business_jellyfin_sync_failures_total")

src/main/resources/application.yaml

@@ -58,9 +58,12 @@ spring:
             user-name-attribute: response
 
 server:
+  port: ${SERVER_PORT:8080}
   shutdown: graceful
 
 management:
+  server:
+    port: ${MANAGEMENT_SERVER_PORT:8081}
   endpoints:
     web:
       base-path: /actuator
@@ -76,6 +79,12 @@ management:
       enabled: true
     readinessstate:
       enabled: true
+  metrics:
+    tags:
+      application: ${MANAGEMENT_METRICS_TAGS_APPLICATION:${spring.application.name}}
+    distribution:
+      percentiles-histogram:
+        http.server.requests: ${MANAGEMENT_METRICS_DISTRIBUTION_PERCENTILES_HISTOGRAM_HTTP_SERVER_REQUESTS:false}
   info:
     env:
       enabled: true