Monitoring

prometheus_exporters:
# Global parameter. If true, add prometheus exporters for apps 
# with the same parameter also set to true in app section
  enable: true 
# Global parameter. if true - expose metrics for apps on ingress 
# Check the same parameters in apps' configs
  addToIngress: false 
# Base path for metrics, e.g., <https://host.local/metricsBasePath/metricsEndPath>
  metricsBasePath: /metrics/ 
  auth:
  # Set HTTP Basic AUTH for metrics in ingress. Usable only if 'addToIngress' is true
    enable: true 
    username: flower_user
    password: flowerpass
# Enable serviceMonitor (Service Discovery for Prometheus)
  serviceMonitor: 
    enable: true
    interval: 15s

spec:
  serviceMonitorNamespaceSelector:
    matchNames:
    - default
    - monitoring

spec:
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}

- name: Celery alerts
  rules:
  
  # An alert for 0.9 quantile of task duration, warns that analyses are running slowly
    - alert: Quality Analysis is slow!
      expr: histogram_quantile(0.9, sum without (handler) (rate(flower_task_runtime_seconds_bucket{task="oz_core.tasks.tfss.process_analyse_quality"}[5m]))) > 5
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: "Too long duration of celery worker for TFSS task"
        description: "The duration of 90% Quality analyses is longer than {{ $value }} seconds for last 10 minutes. NAMESPACE: {{ $labels.namespace }} POD: {{ $labels.pod }} TASK: {{ $labels.task }} WORKER: {{ $labels.worker }}"
  
  # An alert for failed tasks, if the number is growing, something goes wrong
    - alert: Celery tasks failed!
      expr: sum by(type,task) (rate(flower_events_total{type="task-failed", task!=""}[1m])) > 0
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: "Celery tasks {{ $labels.task }} failed"
        description: 'Failed celery tasks rate: {{ printf "%.2f" $value }} rps'
  
  # A critical alert that warns that all the tasks are failed; it means that the system has stopped processing requests
    - alert: Celery zero success tasks!
      expr: sum by(type,task) (rate(flower_events_total{type="task-succeeded", task!=""}[1m])) == 0 and on (task) sum by(type,task) (rate(flower_events_total{type="task-received", task!=""}[1m])) > 0
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: "Celery has zero success tasks: {{ $labels.task }}!"
        description: "Critical! Check if bio is alive!"

- name: Redis alerts
  rules:
  # A critical alert that warns about Redis being down
    - alert: Redis is down
      expr: redis_up != 1
      for: 30s
      labels:
        severity: critical
      annotations:
        summary: "Redis is down for more than 30 seconds!"
        description: "Critical: REDIS service is down in namespace: {{ $labels.namespace }}\nPod: {{ $labels.pod }}!"
  
  # Displays if Redis rejects connections
    - alert: Redis rejected connections
      expr: rate(redis_rejected_connections_total[1m]) > 0
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: "Redis rejects connections for more than 1 minute in namespace: {{ $labels.namespace }}!"
        description: "Some connections to Redis have been rejected!\nPod: {{ $labels.pod }}\nValue = {{ $value }}"

  # Displays that Redis commands are being executed too slow 
    - alert: Redis command duration is slow!
      expr: max by(namespace) (rate(redis_commands_duration_seconds_total[1m])) > 0.0004
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: "Redis max command duration is too high for more than 1 minute in namespace: {{ $labels.namespace }}!"
        description: "Maximum command duration is longer than average!\nValue = {{ $value }} seconds"
 
  # Warns that the Redis queue is too long
    - alert: Redis queue length
      expr: sum by (instance)(redis_key_size) > 50
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: "Redis queue size is too large!"
        description: "Warning: Redis queue size : {{ $value }} for the last 1 min!"

  # Warns that there are more than 10 processing (unacked) messages in the Redis queue
    - alert: Redis unacked masseges
      expr: sum by (key)(redis_key_size{key="unacked"}) > 10
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: "Redis has unacked messages!"
        description: "Warning: Redis has {{ $value }} unacked messages!"

- name: TFSS alerts
  rules:
  # Critical alert that warns about blackbox detecting incorrect model behavior
    - alert: TFSS models probe service alert!
      expr: probe_success{job="blackbox-tfss-service"} != 1
      for: 3m
      labels:
        severity: critical
      annotations:
        summary: "TFSS model in namespace {{ $labels.namespace }} is unavailable!"
        description: "!!!ALERT!!! TFSS model or server doesn't work:\nMODEL:{{ $labels.model }}\nModel probe has been returning failed state for 3 min!"
    - alert: TFSS models probe pod alert!
      expr: probe_success{job="blackbox-tfss-models"} != 1
      for: 3m
      labels:
        severity: critical
      annotations:
        summary: "TFSS model in namespace {{ $labels.namespace }} is unavailable!"
        description: "TFSS in pod {{ $labels.pod }} doesn't work:\nMODEL:{{ $labels.model }}\nModel probe has been returning failed state for 3 min!"
    - alert: TFSS predict probe pod alert!
      expr: probe_success{job="blackbox-tfss-probe"} != 1
      for: 3m
      labels:
        severity: critical
      annotations:
        summary: "TFSS model predict in namespace {{ $labels.namespace }} is unavailable!"
        description: "TFSS in pod {{ $labels.pod }} doesn't work:\nMODEL:{{ $labels.model }}\nPredict probe has been returning failed state for 3 min!"

  # Critical alert that warns about TFSS not processing requests 
    - alert: TFSS empty request rate!
      expr: absent(:tensorflow:serving:request_count{namespace="api-prod"}) == 1
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "TFSS request rate is empty!!!"
        description: "Critical! Requests are not processed, check bio!!!"

groups:
  - name: NGINX alerts
    rules:
      # A critical alert that displays that Nginx hasn't handled some of the accepted connections
      - alert: Nginx not all connections are handled
        expr: rate (nginx_connections_handled[5m]) / rate (nginx_connections_accepted[5m]) <1
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Nginx issue with handling connections"
          description: "Critical: Nginx doesn't handle some accepted connections on the host {{ $labels.instance }} for more than 3 minutes!"

groups:
  - name: API alerts
    rules:
      # A critical alert that warns that there are no ready API containers, so requests are not being processed
      - alert: Absent ready api containers!
        expr: absent(kube_pod_container_status_ready{container="oz-api", namespace="api-prod"} == 1)
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Absent ready api containers!!!"
          description: "Critical! Check api containers!!!"