- name: Celery alerts
rules:
# An alert for 0.9 quantile of task duration, warns that analyses are running slowly
- alert: Quality Analysis is slow!
expr: histogram_quantile(0.9, sum without (handler) (rate(flower_task_runtime_seconds_bucket{task="oz_core.tasks.tfss.process_analyse_quality"}[5m]))) > 5
for: 10m
labels:
severity: warning
annotations:
summary: "Too long duration of celery worker for TFSS task"
description: "The duration of 90% Quality analyses is longer than {{ $value }} seconds for last 10 minutes. NAMESPACE: {{ $labels.namespace }} POD: {{ $labels.pod }} TASK: {{ $labels.task }} WORKER: {{ $labels.worker }}"
# An alert for failed tasks, if the number is growing, something goes wrong
- alert: Celery tasks failed!
expr: sum by(type,task) (rate(flower_events_total{type="task-failed", task!=""}[1m])) > 0
for: 1m
labels:
severity: warning
annotations:
summary: "Celery tasks {{ $labels.task }} failed"
description: 'Failed celery tasks rate: {{ printf "%.2f" $value }} rps'
# A critical alert that warns that all the tasks are failed; it means that the system has stopped processing requests
- alert: Celery zero success tasks!
expr: sum by(type,task) (rate(flower_events_total{type="task-succeeded", task!=""}[1m])) == 0 and on (task) sum by(type,task) (rate(flower_events_total{type="task-received", task!=""}[1m])) > 0
for: 5m
labels:
severity: critical
annotations:
summary: "Celery has zero success tasks: {{ $labels.task }}!"
description: "Critical! Check if bio is alive!"
- name: Redis alerts
rules:
# A critical alert that warns about Redis being down
- alert: Redis is down
expr: redis_up != 1
for: 30s
labels:
severity: critical
annotations:
summary: "Redis is down for more than 30 seconds!"
description: "Critical: REDIS service is down in namespace: {{ $labels.namespace }}\nPod: {{ $labels.pod }}!"
# Displays if Redis rejects connections
- alert: Redis rejected connections
expr: rate(redis_rejected_connections_total[1m]) > 0
for: 1m
labels:
severity: warning
annotations:
summary: "Redis rejects connections for more than 1 minute in namespace: {{ $labels.namespace }}!"
description: "Some connections to Redis have been rejected!\nPod: {{ $labels.pod }}\nValue = {{ $value }}"
# Displays that Redis commands are being executed too slow
- alert: Redis command duration is slow!
expr: max by(namespace) (rate(redis_commands_duration_seconds_total[1m])) > 0.0004
for: 1m
labels:
severity: warning
annotations:
summary: "Redis max command duration is too high for more than 1 minute in namespace: {{ $labels.namespace }}!"
description: "Maximum command duration is longer than average!\nValue = {{ $value }} seconds"
# Warns that the Redis queue is too long
- alert: Redis queue length
expr: sum by (instance)(redis_key_size) > 50
for: 1m
labels:
severity: warning
annotations:
summary: "Redis queue size is too large!"
description: "Warning: Redis queue size : {{ $value }} for the last 1 min!"
# Warns that there are more than 10 processing (unacked) messages in the Redis queue
- alert: Redis unacked masseges
expr: sum by (key)(redis_key_size{key="unacked"}) > 10
for: 1m
labels:
severity: warning
annotations:
summary: "Redis has unacked messages!"
description: "Warning: Redis has {{ $value }} unacked messages!"
- name: TFSS alerts
rules:
# Critical alert that warns about blackbox detecting incorrect model behavior
- alert: TFSS models probe service alert!
expr: probe_success{job="blackbox-tfss-service"} != 1
for: 3m
labels:
severity: critical
annotations:
summary: "TFSS model in namespace {{ $labels.namespace }} is unavailable!"
description: "!!!ALERT!!! TFSS model or server doesn't work:\nMODEL:{{ $labels.model }}\nModel probe has been returning failed state for 3 min!"
- alert: TFSS models probe pod alert!
expr: probe_success{job="blackbox-tfss-models"} != 1
for: 3m
labels:
severity: critical
annotations:
summary: "TFSS model in namespace {{ $labels.namespace }} is unavailable!"
description: "TFSS in pod {{ $labels.pod }} doesn't work:\nMODEL:{{ $labels.model }}\nModel probe has been returning failed state for 3 min!"
- alert: TFSS predict probe pod alert!
expr: probe_success{job="blackbox-tfss-probe"} != 1
for: 3m
labels:
severity: critical
annotations:
summary: "TFSS model predict in namespace {{ $labels.namespace }} is unavailable!"
description: "TFSS in pod {{ $labels.pod }} doesn't work:\nMODEL:{{ $labels.model }}\nPredict probe has been returning failed state for 3 min!"
# Critical alert that warns about TFSS not processing requests
- alert: TFSS empty request rate!
expr: absent(:tensorflow:serving:request_count{namespace="api-prod"}) == 1
for: 1m
labels:
severity: critical
annotations:
summary: "TFSS request rate is empty!!!"
description: "Critical! Requests are not processed, check bio!!!"
groups:
- name: NGINX alerts
rules:
# A critical alert that displays that nginx hasn't handled some of the accepted connections
- alert: nginx not all connections are handled
expr: rate (nginx_connections_handled[5m]) / rate (nginx_connections_accepted[5m]) <1
for: 2m
labels:
severity: critical
annotations:
summary: "nginx issue with handling connections"
description: "Critical: nginx doesn't handle some accepted connections on the host {{ $labels.instance }} for more than 3 minutes!"
groups:
- name: API alerts
rules:
# A critical alert that warns that there are no ready API containers, so requests are not being processed
- alert: Absent ready api containers!
expr: absent(kube_pod_container_status_ready{container="oz-api", namespace="api-prod"} == 1)
for: 1m
labels:
severity: critical
annotations:
summary: "Absent ready api containers!!!"
description: "Critical! Check api containers!!!"
