Fix Observability Issues and Startup Probe Failure #70

Closed
devitq wants to merge 2 commits from fix-observability-issues-17959231904052988380 into feat/setup-observability-in-k8s
14 changed files with 150 additions and 497 deletions
Showing only changes of commit 47baaf8fce - Show all commits
+49
View File
@@ -0,0 +1,49 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-agent
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-agent
targetRevision: 0.11.0
helm:
releaseName: vmagent
valuesObject:
remoteWrite:
- url: http://vmsingle-victoria-metrics-single.observability.svc:8428/api/v1/write
config:
scrape_interval: 10s
scrape_configs:
- job_name: 'movienight-backend'
metrics_path: /actuator/prometheus
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
action: keep
regex: backend
- source_labels: [__meta_kubernetes_pod_container_port_name]
action: keep
regex: management
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: pod
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
+35
View File
@@ -0,0 +1,35 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-alert
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-alert
targetRevision: 0.12.0
helm:
releaseName: vmalert
valuesObject:
server:
datasource:
url: http://vmsingle-victoria-metrics-single.observability.svc:8428
notifier:
config: |
route:
group_by: ['alertname']
receiver: 'blackhole'
receivers:
- name: 'blackhole'
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
@@ -0,0 +1,32 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-operator
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-operator
targetRevision: 0.38.0
helm:
releaseName: victoriametrics-operator
valuesObject:
admissionWebhooks:
enabled: false
createCRD: true
operator:
# This enables the operator to watch for CRs in all namespaces
# or we can specify namespaces.
disableNamespaceRestriction: true
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
+30
View File
@@ -0,0 +1,30 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: victoriametrics-single
namespace: argocd
spec:
project: default
source:
repoURL: https://victoriametrics.github.io/helm-charts/
chart: victoria-metrics-single
targetRevision: 0.20.0
helm:
releaseName: vmsingle
valuesObject:
server:
retentionPeriod: 30d
persistentVolume:
enabled: true
size: 20Gi
destination:
server: https://kubernetes.default.svc
namespace: observability
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- ApplyOutOfSyncOnly=true
- ServerSideApply=true
+1 -17
View File
@@ -136,22 +136,6 @@ app.kubernetes.io/component: {{ $component }}
{{- if .Values.observability.grafana.datasource.url -}} {{- if .Values.observability.grafana.datasource.url -}}
{{- .Values.observability.grafana.datasource.url -}} {{- .Values.observability.grafana.datasource.url -}}
{{- else -}} {{- else -}}
{{- printf "http://%s.%s.svc:%v" (include "movienight.victoriaMetricsName" .) .Release.Namespace .Values.observability.victoriaMetrics.service.port -}} {{- .Values.observability.victoriaMetrics.url -}}
{{- end -}}
{{- end -}}
{{- define "movienight.vmagentRemoteWriteURL" -}}
{{- if .Values.observability.vmagent.remoteWriteUrl -}}
{{- .Values.observability.vmagent.remoteWriteUrl -}}
{{- else -}}
{{- printf "http://%s:%v/api/v1/write" (include "movienight.victoriaMetricsName" .) .Values.observability.victoriaMetrics.service.port -}}
{{- end -}}
{{- end -}}
{{- define "movienight.backendMetricsTarget" -}}
{{- if .Values.backend.management.enabled -}}
{{- printf "%s-backend:%v" (include "movienight.fullname" .) .Values.backend.management.port -}}
{{- else -}}
{{- printf "%s-backend:%v" (include "movienight.fullname" .) .Values.backend.service.port -}}
{{- end -}} {{- end -}}
{{- end -}} {{- end -}}
@@ -62,10 +62,6 @@ spec:
{{- if .Values.observability.enabled }} {{- if .Values.observability.enabled }}
- name: MANAGEMENT_METRICS_TAGS_APPLICATION - name: MANAGEMENT_METRICS_TAGS_APPLICATION
value: {{ .Values.observability.metrics.applicationTag | quote }} value: {{ .Values.observability.metrics.applicationTag | quote }}
{{- if .Values.observability.jsonLogging }}
- name: SPRING_PROFILES_ACTIVE
value: json-logging
{{- end }}
{{- end }} {{- end }}
{{- if $postgresEnv }} {{- if $postgresEnv }}
{{- $postgresEnv | nindent 12 }} {{- $postgresEnv | nindent 12 }}
@@ -1,27 +0,0 @@
{{- if and .Values.observability.enabled .Values.observability.vmagent.enabled (not .Values.observability.victoriaMetrics.operator.enabled) }}
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ include "movienight.vmagentName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
rules:
- apiGroups: [""]
resources: ["pods", "nodes", "nodes/proxy", "services", "endpoints"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ include "movienight.vmagentName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ include "movienight.vmagentName" . }}
subjects:
- kind: ServiceAccount
name: {{ include "movienight.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
{{- end }}
@@ -1,142 +0,0 @@
{{- if and .Values.observability.enabled .Values.observability.victoriaMetrics.enabled (not .Values.observability.victoriaMetrics.operator.enabled) }}
{{- if .Values.observability.victoriaMetrics.persistence.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ include "movienight.victoriaMetricsName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.observability.victoriaMetrics.persistence.size }}
{{- with .Values.observability.victoriaMetrics.persistence.storageClass }}
storageClassName: {{ . | quote }}
{{- end }}
---
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "movienight.victoriaMetricsName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 6 }}
template:
metadata:
labels:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 8 }}
{{- with .Values.observability.victoriaMetrics.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "movienight.serviceAccountName" . }}
{{- with .Values.global.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: victoriametrics
image: "{{ .Values.observability.victoriaMetrics.image.repository }}:{{ .Values.observability.victoriaMetrics.image.tag }}"
imagePullPolicy: {{ .Values.observability.victoriaMetrics.image.pullPolicy }}
args:
- -storageDataPath=/var/lib/victoriametrics
- -retentionPeriod={{ .Values.observability.victoriaMetrics.retentionPeriod }}
- -httpListenAddr=:{{ .Values.observability.victoriaMetrics.service.port }}
{{- with .Values.observability.victoriaMetrics.extraArgs }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.observability.victoriaMetrics.service.port }}
protocol: TCP
readinessProbe:
httpGet:
path: /-/ready
port: http
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
httpGet:
path: /-/healthy
port: http
initialDelaySeconds: 10
periodSeconds: 30
timeoutSeconds: 5
{{- with .Values.observability.victoriaMetrics.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
volumeMounts:
- name: data
mountPath: /var/lib/victoriametrics
volumes:
- name: data
{{- if .Values.observability.victoriaMetrics.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ include "movienight.victoriaMetricsName" . }}
{{- else }}
emptyDir: {}
{{- end }}
{{- with .Values.observability.victoriaMetrics.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "movienight.victoriaMetricsName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.observability.victoriaMetrics.service.type }}
ports:
- name: http
port: {{ .Values.observability.victoriaMetrics.service.port }}
targetPort: http
protocol: TCP
selector:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
{{- end }}
@@ -1,22 +0,0 @@
{{- if and .Values.observability.enabled .Values.observability.vmagent.enabled .Values.observability.victoriaMetrics.operator.enabled }}
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAgent
metadata:
name: {{ include "movienight.vmagentName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
spec:
scrapeInterval: {{ .Values.observability.vmagent.scrapeInterval }}
remoteWrite:
- url: {{ include "movienight.vmagentRemoteWriteURL" . }}
{{- with .Values.observability.vmagent.resources }}
resources:
{{- toYaml . | nindent 4 }}
{{- end }}
serviceScrapeSelector:
matchLabels:
{{- include "movienight.labels" . | nindent 6 }}
podScrapeSelector:
matchLabels:
{{- include "movienight.labels" . | nindent 6 }}
{{- end }}
@@ -1,182 +0,0 @@
{{- if and .Values.observability.enabled .Values.observability.vmagent.enabled (not .Values.observability.victoriaMetrics.operator.enabled) }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "movienight.vmagentName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
data:
scrape.yaml: |
global:
scrape_interval: {{ .Values.observability.vmagent.scrapeInterval }}
scrape_timeout: {{ .Values.observability.vmagent.scrapeTimeout }}
scrape_configs:
{{- if .Values.backend.enabled }}
- job_name: {{ .Values.observability.vmagent.backendJobName | quote }}
metrics_path: /actuator/prometheus
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
action: keep
regex: backend
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
action: keep
regex: {{ .Release.Name | quote }}
- source_labels: [__meta_kubernetes_pod_container_port_name]
action: keep
regex: management
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: pod
{{- end }}
- job_name: {{ printf "%s-vmagent" (include "movienight.name" .) | quote }}
static_configs:
- targets:
- {{ printf "%s:%v" (include "movienight.vmagentName" .) .Values.observability.vmagent.service.port | quote }}
labels:
app: {{ include "movienight.name" . | quote }}
release: {{ .Release.Name | quote }}
namespace: {{ .Release.Namespace | quote }}
{{- if .Values.observability.victoriaMetrics.enabled }}
- job_name: {{ printf "%s-victoriametrics" (include "movienight.name" .) | quote }}
static_configs:
- targets:
- {{ printf "%s:%v" (include "movienight.victoriaMetricsName" .) .Values.observability.victoriaMetrics.service.port | quote }}
labels:
app: {{ include "movienight.name" . | quote }}
release: {{ .Release.Name | quote }}
namespace: {{ .Release.Namespace | quote }}
{{- end }}
{{- with .Values.observability.vmagent.extraScrapeConfigs }}
{{- toYaml . | nindent 6 }}
{{- end }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "movienight.vmagentName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
replicas: 1
selector:
matchLabels:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 6 }}
template:
metadata:
labels:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 8 }}
{{- with .Values.observability.vmagent.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.vmagent.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "movienight.serviceAccountName" . }}
{{- with .Values.global.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.vmagent.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: vmagent
image: "{{ .Values.observability.vmagent.image.repository }}:{{ .Values.observability.vmagent.image.tag }}"
imagePullPolicy: {{ .Values.observability.vmagent.image.pullPolicy }}
args:
- -promscrape.config=/etc/vmagent/scrape.yaml
- -remoteWrite.url={{ include "movienight.vmagentRemoteWriteURL" . }}
- -httpListenAddr=:{{ .Values.observability.vmagent.service.port }}
{{- with .Values.observability.vmagent.extraArgs }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.observability.vmagent.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.observability.vmagent.service.port }}
protocol: TCP
readinessProbe:
httpGet:
path: /-/ready
port: http
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
livenessProbe:
httpGet:
path: /-/healthy
port: http
initialDelaySeconds: 10
periodSeconds: 30
timeoutSeconds: 5
{{- with .Values.observability.vmagent.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
volumeMounts:
- name: config
mountPath: /etc/vmagent
readOnly: true
- name: tmp
mountPath: /tmp
volumes:
- name: config
configMap:
name: {{ include "movienight.vmagentName" . }}
- name: tmp
emptyDir: {}
{{- with .Values.observability.vmagent.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.vmagent.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.observability.vmagent.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "movienight.vmagentName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
{{- with .Values.global.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.observability.vmagent.service.type }}
ports:
- name: http
port: {{ .Values.observability.vmagent.service.port }}
targetPort: http
protocol: TCP
selector:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "vmagent") | nindent 4 }}
{{- end }}
@@ -1,20 +0,0 @@
{{- if and .Values.observability.enabled .Values.observability.victoriaMetrics.enabled .Values.observability.victoriaMetrics.operator.enabled }}
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMPodScrape
metadata:
name: {{ include "movienight.fullname" . }}-backend
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "backend") | nindent 4 }}
spec:
podMetricsEndpoints:
- port: management
path: /actuator/prometheus
{{- if .Values.observability.metrics.httpServerRequestsHistogram }}
relabelConfigs:
- targetLabel: application
replacement: {{ .Values.observability.metrics.applicationTag | quote }}
{{- end }}
selector:
matchLabels:
{{- include "movienight.componentSelectorLabels" (dict "root" . "component" "backend") | nindent 6 }}
{{- end }}
@@ -1,25 +0,0 @@
{{- if and .Values.observability.enabled .Values.observability.victoriaMetrics.enabled .Values.observability.victoriaMetrics.operator.enabled }}
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMSingle
metadata:
name: {{ include "movienight.victoriaMetricsName" . }}
labels:
{{- include "movienight.componentLabels" (dict "root" . "component" "victoriametrics") | nindent 4 }}
spec:
retentionPeriod: {{ .Values.observability.victoriaMetrics.retentionPeriod }}
{{- if .Values.observability.victoriaMetrics.persistence.enabled }}
storage:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: {{ .Values.observability.victoriaMetrics.persistence.size }}
{{- with .Values.observability.victoriaMetrics.persistence.storageClass }}
storageClassName: {{ . | quote }}
{{- end }}
{{- end }}
{{- with .Values.observability.victoriaMetrics.resources }}
resources:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
+2 -47
View File
@@ -121,59 +121,14 @@ routes:
pathPrefix: / pathPrefix: /
observability: observability:
enabled: false enabled: true
metrics: metrics:
applicationTag: movienight applicationTag: movienight
httpServerRequestsHistogram: true httpServerRequestsHistogram: true
jsonLogging: true
victoriaMetrics: victoriaMetrics:
enabled: true url: "http://vmsingle-victoria-metrics-single.observability.svc:8428"
operator: operator:
enabled: false
image:
repository: docker.io/victoriametrics/victoria-metrics
tag: v1.134.0
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 8428
retentionPeriod: 30d
extraArgs: []
persistence:
enabled: true enabled: true
size: 10Gi
storageClass: ""
podAnnotations: {}
podLabels: {}
resources: {}
securityContext: {}
podSecurityContext: {}
nodeSelector: {}
tolerations: []
affinity: {}
vmagent:
enabled: true
image:
repository: docker.io/victoriametrics/vmagent
tag: v1.135.0
pullPolicy: IfNotPresent
service:
type: ClusterIP
port: 8429
scrapeInterval: 5s
scrapeTimeout: 5s
remoteWriteUrl: ""
backendJobName: movienight-backend
extraArgs: []
extraScrapeConfigs: []
podAnnotations: {}
podLabels: {}
resources: {}
securityContext: {}
podSecurityContext: {}
nodeSelector: {}
tolerations: []
affinity: {}
grafana: grafana:
datasource: datasource:
enabled: true enabled: true
+1 -11
View File
@@ -5,22 +5,12 @@
</encoder> </encoder>
</appender> </appender>
<appender name="JSON_CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> <springProfile name="!file-logging">
<encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
</appender>
<springProfile name="!file-logging &amp; !json-logging">
<root level="INFO"> <root level="INFO">
<appender-ref ref="CONSOLE"/> <appender-ref ref="CONSOLE"/>
</root> </root>
</springProfile> </springProfile>
<springProfile name="json-logging">
<root level="INFO">
<appender-ref ref="JSON_CONSOLE"/>
</root>
</springProfile>
<springProfile name="file-logging"> <springProfile name="file-logging">
<appender name="JSON_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> <appender name="JSON_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>logs/app.json</file> <file>logs/app.json</file>