feat(monitoring): refine unhealthy ring alert

intergral · Oct 19, 2023 · 0687855 · 0687855
1 parent c9fdf69
commit 0687855
Show file tree

Hide file tree

Showing 7 changed files with 21 additions and 6 deletions.
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/charts/deep-distributed/Chart.yaml b/charts/deep-distributed/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.0.21
+version: 0.0.22
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/deep-distributed/templates/prometheusrule.yaml b/charts/deep-distributed/templates/prometheusrule.yaml
@@ -18,6 +18,6 @@ spec:
     - name: deep.rules
       rules:
         {{- with .Values.monitoring.rules.rules }}
-        {{- tpl . $ | nindent 8 -}}
+        {{- toYaml . | nindent 8 -}}
         {{- end}}
 {{- end -}}
diff --git a/charts/deep-distributed/values.yaml b/charts/deep-distributed/values.yaml
@@ -1017,10 +1017,16 @@ monitoring:
     # The namespace to deploy the rules in
     namespace: null
     # The rules to add
-    rules: |
+    rules:
       - alert: Unhealthy Ring Member
         expr: |
           deep_ring_members{state="Unhealthy"} > 0
+        labels:
+          severity: critical
+        annotations:
+          description: "{{ $labels.service }} has {{ $value }} unhealthy nodes in ring {{ $labels.name }}."
+          runbook_url: https://intergral.github.io/deep/runbooks/unhealthy_ring_node/
+          summary: One or more ring members are unhealthy
 
 
 # Configuration for the gateway

diff --git a/charts/deep/Chart.yaml b/charts/deep/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.0.10
+version: 1.0.11
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/deep/templates/prometheusrule.yaml b/charts/deep/templates/prometheusrule.yaml
@@ -18,6 +18,6 @@ spec:
     - name: deep.rules
       rules:
         {{- with .Values.monitoring.rules.rules }}
-        {{- tpl . $ | nindent 8 -}}
+        {{- toYaml . | nindent 8 -}}
         {{- end}}
 {{- end -}}
diff --git a/charts/deep/values.yaml b/charts/deep/values.yaml
@@ -462,7 +462,13 @@ monitoring:
     # The namespace to deploy the rules in
     namespace: null
     # The rules to add
-    rules: |
+    rules:
       - alert: Unhealthy Ring Member
         expr: |
           deep_ring_members{state="Unhealthy"} > 0
+        labels:
+          severity: critical 
+        annotations:
+          description: "{{ $labels.service }} has {{ $value }} unhealthy nodes in ring {{ $labels.name }}."
+          runbook_url: https://intergral.github.io/deep/runbooks/unhealthy_ring_node/
+          summary: One or more ring members are unhealthy