method: POST
body: '{"inputs": {"images_bytes": [{"b64": "aaa"}]}}'
probe_success in TFSS is not working properly
Last updated
Was this helpful?
Was this helpful?
groups:
- name: API alerts
rules:
- alert: Absent ready API containers!
expr: absent(kube_pod_container_status_ready{container="oz-api", namespace="api-prod"} == 1)
for: 1m
labels:
severity: critical
annotations:
summary: "Absent ready API containers!!!"
description: "Critical: check API containers!!!"
- alert: Absent API metrics!
expr: absent(oz_api_versions_total)
for: 1m
labels:
severity: critical
annotations:
summary: "Absent API metrics!!!"
description: "Critical: check API containers, API might be not working properly!!!"
- alert: High frequency of API analysis requests
expr: sum (rate(oz_api_analyses_total{namespace="api-prod"}[1m])) > 1
labels:
severity: warning
annotations:
summary: "Too many API analysis requests per second"
description: "API analysis request rate is {{ $value }} rps."
- alert: High failed API analysis rate
expr: sum by (analysis_result)(rate(oz_api_analyses_total{namespace="api-prod", analysis_result="FAILED"}[1m])) > 0
for: 1m
labels:
severity: warning
annotations:
summary: "Failed API analyses detected"
description: "API analysis failure rate is {{ $value }} rps in the last minute."
- alert: High API analysis duration
expr: histogram_quantile(0.95,sum by (le, analysis_type) (rate(oz_api_analyse_duration_seconds_bucket{namespace="api-prod"}[1m]))) > 7
for: 10m
labels:
severity: warning
annotations:
summary: "{{ $labels.analysis_type }} analyses are going slow"
description: "The duration of 95% {{ $labels.analysis_type }} analyses is longer than {{ $value }} seconds in the last 10 minutes."
- alert: API exception errors
expr: sum by (exception_type)(rate(django_http_exceptions_total_by_type_total{namespace="api-prod", exception_type!~"AuthCredentialsExpired|AuthCredentialsInvalid|AuthCredentialsIsNotProvided|AuthForbidden|InvalidBodyStructureError|InvalidParamValueErrorNotExistError|OSError|InvalidParamValueError|NotExistError"}[1m])) > 0
for: 5m
labels:
severity: warning
annotations:
summary: "{{ $labels.exception_type }} in API exception errors"
description: "{{ $value }} rps in API returned {{ $labels.exception_type }} in the last 5 minutes."
- alert: High API latency
expr: (sum by(namespace) (rate(gunicorn_request_duration_sum{}[1m])) / on(namespace) sum by(namespace) (rate(gunicorn_request_duration_count{}[1m]))) > 5
for: 5m
labels:
severity: warning
annotations:
summary: "High API latency"
description: "API latency is {{ printf "%.2f" $value }} seconds in the last 5 minutes."
- alert: High API 5xx error rate!
expr: sum by(namespace) (max by(status) (rate(gunicorn_response_code{status=~"5.*"}[1m]))) > 1
for: 1m
labels:
severity: warning
annotations:
summary: "High 5XX API error percent"
description: 'High error percent: {{ printf "%.2f" $value }}%'- name: TFSS alerts
rules:
# Blackbox alerts: probes check whether TFSS models are working correctly
- alert: TFSS models probe service alert!
expr: probe_success{job="blackbox-tfss-service"} != 1
for: 3m
labels:
severity: critical
annotations:
summary: "TFSS model in namespace {{ $labels.namespace }} is unavailable!"
description: "!!!ALERT!!! TFSS model or server doesn't work:\nMODEL:{{ $labels.model }}\nModel probe has been returning failed state for 3 min!"
- alert: TFSS models probe pod alert!
expr: probe_success{job="blackbox-tfss-models"} != 1
for: 3m
labels:
severity: critical
annotations:
summary: "TFSS model in namespace {{ $labels.namespace }} is unavailable!"
description: "TFSS in pod {{ $labels.pod }} doesn't work:\nMODEL:{{ $labels.model }}\nModel probe has been returning failed state for 3 min!"
- alert: TFSS predict probe pod alert!
expr: probe_success{job="blackbox-tfss-probe"} != 1
for: 3m
labels:
severity: critical
annotations:
summary: "TFSS model predict in namespace {{ $labels.namespace }} is unavailable!"
description: "TFSS in pod {{ $labels.pod }} doesn't work:\nMODEL:{{ $labels.model }}\nPredict probe has been returning failed state for 3 min!"
# Critical: indicates the metric is absent, meaning TFSS is not processing requests
- alert: TFSS empty request rate!
expr: absent(:tensorflow:serving:request_count{namespace="api-prod"}) == 1
for: 1m
labels:
severity: critical
annotations:
summary: "TFSS request rate is empty!!!"
description: "Critical! Requests are not processed, check bio!!!"groups:
- name: NGINX alerts
rules:
- alert: nginx is down
expr: nginx_up != 1
for: 30s
labels:
severity: critical
annotations:
summary: "nginx has been down for more than 30 seconds!"
description: "Critical: nginx service is down on the host {{ $labels.instance }}!"
- alert: nginx not all connections are handled
expr: rate (nginx_connections_handled[5m]) / rate (nginx_connections_accepted[5m]) <1
for: 2m
labels:
severity: critical
annotations:
summary: "nginx has issues with handling connections"
description: "Critical: not all accepted connections have been handled by nginx on the host {{ $labels.instance }} for more than 3 minutes!"
- alert: High number of active connections in nginx
expr: nginx_connections_active > 3000
for: 3m
labels:
severity: critical
annotations:
summary: "High number of active connections in nginx"
description: "Critical: nginx has had too many active connections on the host {{ $labels.instance }} for more than 3 minutes!"