Fix Observability Issues and Startup Probe Failure #70

Closed
devitq wants to merge 2 commits from fix-observability-issues-17959231904052988380 into feat/setup-observability-in-k8s
9 changed files with 155 additions and 375 deletions
+49
View File
@@ -0,0 +1,49 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-agent
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-agent
targetRevision: 0.11.0
helm:
releaseName: vmagent
valuesObject:
remoteWrite:
- url: http://vmsingle-victoria-metrics-single.observability.svc:8428/api/v1/write
config:
scrape_interval: 10s
scrape_configs:
- job_name: 'movienight-backend'
metrics_path: /actuator/prometheus
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
action: keep
regex: backend
- source_labels: [__meta_kubernetes_pod_container_port_name]
action: keep
regex: management
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: pod
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
+35
View File
@@ -0,0 +1,35 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-alert
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-alert
targetRevision: 0.12.0
helm:
releaseName: vmalert
valuesObject:
server:
datasource:
url: http://vmsingle-victoria-metrics-single.observability.svc:8428
notifier:
config: |
route:
group_by: ['alertname']
receiver: 'blackhole'
receivers:
- name: 'blackhole'
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
@@ -0,0 +1,32 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-operator
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-operator
targetRevision: 0.38.0
helm:
releaseName: victoriametrics-operator
valuesObject:
admissionWebhooks:
enabled: false
createCRD: true
operator:
# This enables the operator to watch for CRs in all namespaces
# or we can specify namespaces.
disableNamespaceRestriction: true
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
+30
View File
@@ -0,0 +1,30 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-single
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-single
targetRevision: 0.20.0
helm:
releaseName: vmsingle
valuesObject:
server:
retentionPeriod: 30d
persistentVolume:
enabled: true
size: 20Gi
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
+1 -17
View File
@@ -136,22 +136,6 @@ app.kubernetes.io/component: {{ $component }}
{{- if .Values.observability.grafana.datasource.url -}}
{{- .Values.observability.grafana.datasource.url -}}
{{- else -}}
{{- printf "http://%s.%s.svc:%v" (include "movienight.victoriaMetricsName" .) .Release.Namespace .Values.observability.victoriaMetrics.service.port -}}
{{- end -}}
{{- end -}}
{{- define "movienight.vmagentRemoteWriteURL" -}}
{{- if .Values.observability.vmagent.remoteWriteUrl -}}
{{- .Values.observability.vmagent.remoteWriteUrl -}}
{{- else -}}
{{- printf "http://%s:%v/api/v1/write" (include "movienight.victoriaMetricsName" .) .Values.observability.victoriaMetrics.service.port -}}
{{- end -}}
{{- end -}}
{{- define "movienight.backendMetricsTarget" -}}
{{- if .Values.backend.management.enabled -}}
{{- printf "%s-backend:%v" (include "movienight.fullname" .) .Values.backend.management.port -}}
{{- else -}}
{{- printf "%s-backend:%v" (include "movienight.fullname" .) .Values.backend.service.port -}}
{{- .Values.observability.victoriaMetrics.url -}}
{{- end -}}
{{- end -}}
@@ -1,142 +0,0 @@
{{- if and .Values.observability.enabled .Values.observability.victoriaMetrics.enabled }}
{{- if .Values.observability.victoriaMetrics.persistence.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ include "movienight.victoriaMetricsName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.observability.victoriaMetrics.persistence.size }}
{{- with .Values.observability.victoriaMetrics.persistence.storageClass }}
storageClassName: {{ . | quote }}
{{- end }}
---
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "movienight.victoriaMetricsName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 6 }}
template:
metadata:
labels:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 8 }}
{{- with .Values.observability.victoriaMetrics.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "movienight.serviceAccountName" . }}
{{- with .Values.global.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: victoriametrics
image: "{{ .Values.observability.victoriaMetrics.image.repository }}:{{ .Values.observability.victoriaMetrics.image.tag }}"
imagePullPolicy: {{ .Values.observability.victoriaMetrics.image.pullPolicy }}
args:
- -storageDataPath=/var/lib/victoriametrics
- -retentionPeriod={{ .Values.observability.victoriaMetrics.retentionPeriod }}
- -httpListenAddr=:{{ .Values.observability.victoriaMetrics.service.port }}
{{- with .Values.observability.victoriaMetrics.extraArgs }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.observability.victoriaMetrics.service.port }}
protocol: TCP
readinessProbe:
httpGet:
path: /-/ready
port: http
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
httpGet:
path: /-/healthy
port: http
initialDelaySeconds: 10
periodSeconds: 30
timeoutSeconds: 5
{{- with .Values.observability.victoriaMetrics.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
volumeMounts:
- name: data
mountPath: /var/lib/victoriametrics
volumes:
- name: data
{{- if .Values.observability.victoriaMetrics.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ include "movienight.victoriaMetricsName" . }}
{{- else }}
emptyDir: {}
{{- end }}
{{- with .Values.observability.victoriaMetrics.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "movienight.victoriaMetricsName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.observability.victoriaMetrics.service.type }}
ports:
- name: http
port: {{ .Values.observability.victoriaMetrics.service.port }}
targetPort: http
protocol: TCP
selector:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
{{- end }}
@@ -1,169 +0,0 @@
{{- if and .Values.observability.enabled .Values.observability.vmagent.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "movienight.vmagentName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
data:
scrape.yaml: |
global:
scrape_interval: {{ .Values.observability.vmagent.scrapeInterval }}
scrape_timeout: {{ .Values.observability.vmagent.scrapeTimeout }}
scrape_configs:
{{- if .Values.backend.enabled }}
- job_name: {{ .Values.observability.vmagent.backendJobName | quote }}
metrics_path: /actuator/prometheus
static_configs:
- targets:
- {{ include "movienight.backendMetricsTarget" . | quote }}
labels:
app: {{ include "movienight.name" . | quote }}
release: {{ .Release.Name | quote }}
namespace: {{ .Release.Namespace | quote }}
{{- end }}
- job_name: {{ printf "%s-vmagent" (include "movienight.name" .) | quote }}
static_configs:
- targets:
- {{ printf "%s:%v" (include "movienight.vmagentName" .) .Values.observability.vmagent.service.port | quote }}
labels:
app: {{ include "movienight.name" . | quote }}
release: {{ .Release.Name | quote }}
namespace: {{ .Release.Namespace | quote }}
{{- if .Values.observability.victoriaMetrics.enabled }}
- job_name: {{ printf "%s-victoriametrics" (include "movienight.name" .) | quote }}
static_configs:
- targets:
- {{ printf "%s:%v" (include "movienight.victoriaMetricsName" .) .Values.observability.victoriaMetrics.service.port | quote }}
labels:
app: {{ include "movienight.name" . | quote }}
release: {{ .Release.Name | quote }}
namespace: {{ .Release.Namespace | quote }}
{{- end }}
{{- with .Values.observability.vmagent.extraScrapeConfigs }}
{{- toYaml . | nindent 6 }}
{{- end }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "movienight.vmagentName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
replicas: 1
selector:
matchLabels:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 6 }}
template:
metadata:
labels:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 8 }}
{{- with .Values.observability.vmagent.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.vmagent.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "movienight.serviceAccountName" . }}
{{- with .Values.global.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.vmagent.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: vmagent
image: "{{ .Values.observability.vmagent.image.repository }}:{{ .Values.observability.vmagent.image.tag }}"
imagePullPolicy: {{ .Values.observability.vmagent.image.pullPolicy }}
args:
- -promscrape.config=/etc/vmagent/scrape.yaml
- -remoteWrite.url={{ include "movienight.vmagentRemoteWriteURL" . }}
- -httpListenAddr=:{{ .Values.observability.vmagent.service.port }}
{{- with .Values.observability.vmagent.extraArgs }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.observability.vmagent.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.observability.vmagent.service.port }}
protocol: TCP
readinessProbe:
httpGet:
path: /-/ready
port: http
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
httpGet:
path: /-/healthy
port: http
initialDelaySeconds: 10
periodSeconds: 30
timeoutSeconds: 5
{{- with .Values.observability.vmagent.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
volumeMounts:
- name: config
mountPath: /etc/vmagent
readOnly: true
- name: tmp
mountPath: /tmp
volumes:
- name: config
configMap:
name: {{ include "movienight.vmagentName" . }}
- name: tmp
emptyDir: {}
{{- with .Values.observability.vmagent.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.vmagent.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.vmagent.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "movienight.vmagentName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.observability.vmagent.service.type }}
ports:
- name: http
port: {{ .Values.observability.vmagent.service.port }}
targetPort: http
protocol: TCP
selector:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
{{- end }}
+5 -47
View File
@@ -93,10 +93,10 @@ backend:
httpGet:
path: /actuator/health
port: management
initialDelaySeconds: 5
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 24
failureThreshold: 12
gateway:
enabled: false
@@ -121,56 +121,14 @@ routes:
pathPrefix: /
observability:
enabled: false
enabled: true
metrics:
applicationTag: movienight
httpServerRequestsHistogram: true
victoriaMetrics:
enabled: true
image:
repository: docker.io/victoriametrics/victoria-metrics
tag: v1.134.0
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 8428
retentionPeriod: 30d
extraArgs: []
persistence:
url: "http://vmsingle-victoria-metrics-single.observability.svc:8428"
operator:
enabled: true
size: 10Gi
storageClass: ""
podAnnotations: {}
podLabels: {}
resources: {}
securityContext: {}
podSecurityContext: {}
nodeSelector: {}
tolerations: []
affinity: {}
vmagent:
enabled: true
image:
repository: docker.io/victoriametrics/vmagent
tag: v1.135.0
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 8429
scrapeInterval: 5s
scrapeTimeout: 5s
remoteWriteUrl: ""
backendJobName: movienight-backend
extraArgs: []
extraScrapeConfigs: []
podAnnotations: {}
podLabels: {}
resources: {}
securityContext: {}
podSecurityContext: {}
nodeSelector: {}
tolerations: []
affinity: {}
grafana:
datasource:
enabled: true
+3
View File
@@ -58,9 +58,12 @@ spring:
user-name-attribute: response
server:
port: ${SERVER_PORT:8080}
shutdown: graceful
management:
server:
port: ${MANAGEMENT_SERVER_PORT:8081}
endpoints:
web:
base-path: /actuator