prometheus: serverFiles: alerts: groups: - name: KafkaAlerts rules: - alert: BrokerOverLoaded expr: avg(sum by(brokerId, kafka_cr, kubernetes_namespace) (rate(kafka_network_requestmetrics_requests_total[15m]))) > 30 for: 5m labels: severity: alert annotations: description: 'broker {{ $labels.brokerId }} overloaded (current value is: {{ $value }})' summary: 'broker overloaded' # If brokerConfigGroup is defined it will override scaled broker config # brokerConfigGroup: 'default_group' storageClass: 'standard' mountPath: '/kafkalog' diskSize: '2G' image: 'wurstmeister/kafka:2.12-2.3.0' command: 'upScale' - alert: BrokerUnderReplicated expr: kafka_server_replicamanager_underreplicatedpartitions > 0 for: 5m labels: severity: alert annotations: description: 'broker {{ $labels.brokerId }} underreplicated' summary: 'broker underreplicated' storageClass: 'standard' mountPath: '/kafkalog' diskSize: '2G' image: 'wurstmeister/kafka:2.12-2.3.0' command: 'upScale' - alert: PartitionCountHigh expr: max(kafka_server_replicamanager_partitioncount) by (kubernetes_namespace, kafka_cr) > 100 for: 3m labels: severity: alert annotations: description: 'broker {{ $labels.brokerId }} has high partition count' summary: 'high partition count' storageClass: 'standard' mountPath: '/kafkalog' diskSize: '2G' image: 'wurstmeister/kafka:2.12-2.3.0' command: 'upScale' - alert: PartitionCountLow expr: min(kafka_server_replicamanager_partitioncount) by (kubernetes_namespace, kafka_cr) < 40 for: 3m labels: severity: alert annotations: description: 'broker {{ $labels.brokerId }} has low partition count' summary: 'low partition count' command: 'downScale' - alert: RemainingDiskSpaceLow expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"kafka-.*"} < 1 * 1000 * 1000 * 1000 for: 2m labels: severity: alert annotations: description: 'broker {{ $labels.brokerId }} has low disk space' summary: 'low diskspace' storageClass: 'standard' mountPath: '/kafkalog' diskSize: '2G' command: 'addPVC'