apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-dashboards-config
  namespace: calico-monitoring
data:
  calico-dashboards.yaml: |-
    apiVersion: 1
    providers:
    - name: 'calico-dashboards'
      orgId: 1
      folder: ''
      folderUid: ''
      type: file
      disableDeletion: false
      editable: true
      updateIntervalSeconds: 10
      allowUiUpdates: false
      options:
        path: /etc/grafana/provisioning/dashboards
  felix-dashboard.json: |-
    {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"description":"Dashboard for monitoring Project Calico's per-host agent, Felix.  Project Calico provides CNI networking and security for Kubernetes.  Felix is responsible for calculating and applying the per-node dataplane state.  It also collects flow logs and other metrics.","editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":3,"links":[],"panels":[{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":14,"panels":[],"title":"Cluster-wide metrics","type":"row"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Total number of Node resources that Felix \"sees\" in the datamodel.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":1},"id":15,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"max(felix_cluster_num_hosts)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"Nodes","range":true,"refId":"A","useBackend":false}],"title":"Nodes in datamodel","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Total number of network Policies and Profiles (an internal policy-like resource used to represent namespaces/service accounts).","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":1},"id":17,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"max(felix_cluster_num_policies)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"Policies","range":true,"refId":"A","useBackend":false},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"max(felix_cluster_num_profiles)","fullMetaSearch":false,"hide":false,"includeNullMetadata":true,"instant":false,"legendFormat":"Profiles","range":true,"refId":"B","useBackend":false},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"max(felix_cluster_num_tiers)","fullMetaSearch":false,"hide":false,"includeNullMetadata":true,"instant":false,"legendFormat":"Tiers","range":true,"refId":"C","useBackend":false}],"title":"Number of policies/profiles/tiers","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Total number of Endpoints (Pods/VMs/host endpoints) in the datamodel as seen by Felix.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"series","axisLabel":"Workload Endpoints","axisPlacement":"left","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"Host"},"properties":[{"id":"custom.axisPlacement","value":"right"},{"id":"custom.axisLabel","value":"Host Endpoints"}]}]},"gridPos":{"h":7,"w":6,"x":12,"y":1},"id":16,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"max(felix_cluster_num_workload_endpoints)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"Workload","range":true,"refId":"A","useBackend":false},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"max(felix_cluster_num_host_endpoints)","fullMetaSearch":false,"hide":false,"includeNullMetadata":true,"instant":false,"legendFormat":"Host","range":true,"refId":"B","useBackend":false}],"title":"Endpoints in datamodel","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":8},"id":3,"panels":[],"title":"Process metrics","type":"row"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of times the Felix process restarted in 1 hour moving window.  Note that Felix restarts (within the calico-node pod) when its configuration is changed so some restarts are expected when adjusting the FelixConfiguration resource.  In addition, Felix is restarted when the pod is restarted or upgraded.","fieldConfig":{"defaults":{"color":{"fixedColor":"dark-red","mode":"thresholds"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":1,"drawStyle":"bars","fillOpacity":39,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":0,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":0},{"color":"dark-orange","value":2},{"color":"red","value":5},{"color":"dark-red","value":10}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":9},"id":19,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, changes(process_start_time_seconds[1h])) > 0","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Felix restarts/hr (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of times Felix resynced with the datastore (for example reconnecting with Typha).  Resyncs are normal if Typha is rebalancing load or restarting during upgrade etc.","fieldConfig":{"defaults":{"color":{"fixedColor":"dark-red","mode":"thresholds"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":1,"drawStyle":"bars","fillOpacity":39,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":0,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"normal"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"green","value":2},{"color":"light-orange","value":5},{"color":"dark-red","value":10}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":9},"id":27,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, changes(felix_resyncs_started[1h])) > 0","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Felix datastore resyncs/hr (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"CPU usage of the Felix process only; does not include sub-processes such as iptables.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":12,"y":9},"id":1,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, rate(process_cpu_seconds_total[$__rate_interval]))","fullMetaSearch":false,"includeNullMetadata":false,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"CPU Usage (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of open file descriptors.  If this trends upwards over time, indicates a leak.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":18,"y":9},"id":7,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, process_open_fds)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Open file descriptors(top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of garbage collections performed over time.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"opm"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":16},"id":23,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","exemplar":false,"expr":"topk($top_n, rate(go_gc_duration_seconds_count[$__rate_interval]))*60","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"interval":"1m","legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Runtime GC rate/min (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"The Go runtime's current target heap size.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":16},"id":24,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","exemplar":false,"expr":"topk($top_n, avg_over_time(go_memstats_next_gc_bytes[5m]))","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"interval":"1m","legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Runtime GC target heap size (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"The Go runtime's current target heap size.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":12,"y":16},"id":25,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","exemplar":false,"expr":"avg(go_memstats_next_gc_bytes)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"interval":"1m","legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Runtime GC target heap size (average)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Felix process memory usage (resident set size).  does not include sub-processes/other components of calico-node.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":-2,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":18,"y":16},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, process_resident_memory_bytes)","fullMetaSearch":false,"includeNullMetadata":false,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Process memory (RSS) (top $top_n)","type":"timeseries"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":23},"id":4,"panels":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of workload and host endpoints on each node.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":0,"y":26},"id":5,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":false,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, felix_active_local_endpoints)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Active local endpoints (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of policies that are active on each node.  Policies are active when they apply to a local endpoint.  Only active policies consume dataplane resources.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":6,"y":26},"id":6,"options":{"legend":{"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":false,"sortBy":"Last *","sortDesc":true},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, felix_active_local_policies)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Active local policies (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of label selectors that are active on each node.  Label selectors become active when used in a rule of an active policy.  Each label selector has an ongoing CPU cost proportional to the churn rate of endpoints.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed"}},"decimals":0,"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":1000},{"color":"red","value":2000}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":12,"y":26},"id":9,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, felix_active_local_selectors)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Active local selectors (top $top_n)","type":"timeseries"}],"title":"Active resources on each node","type":"row"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":24},"id":21,"panels":[],"title":"Calculation graph","type":"row"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Percentage of the time that the calculation graph thread is actively updating the desired state of the local node.  Approaching 100% indicates that the calculation graph is overloaded.\n\nOverload can be caused by lack of a CPU reservation on the calico-node pod, or by a too-small CPU reservation, or by very high datastore churn rate.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"max":1,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":0.5},{"color":"red","value":0.95}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":25},"id":12,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, rate(felix_calc_graph_update_time_seconds_sum[$__rate_interval]))","fullMetaSearch":false,"includeNullMetadata":false,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Calculation graph load (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Rate of calculation graph updates","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":100},{"color":"red","value":1000}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":25},"id":18,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"rate(felix_calc_graph_update_time_seconds_count[$__rate_interval])","fullMetaSearch":false,"hide":false,"includeNullMetadata":false,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Calculation graph updates/s (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"The 99%ile time to update the calculated desired state of the dataplane","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":0.2},{"color":"red","value":1}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":12,"y":25},"id":11,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, felix_calc_graph_update_time_seconds{quantile=\"0.99\"}) ","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"99%ile calculation update time (top $top_n)","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":32},"id":8,"panels":[],"title":"Dataplane: common","type":"row"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Fraction of time that the dataplane thread is actively updating the dataplane.  Approaching 100% indicates the dataplane is overloaded.  Updates will be delayed and batched resulting in increased update latency.\n\nOverload can be caused by high churn; very large rule sets; lack of a CPU reservation on the calico-node pod, or by a too-small CPU reservation on calico-node.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"max":1,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":0.5},{"color":"red","value":0.95}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":33},"id":13,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, rate(felix_int_dataplane_apply_time_seconds_sum[$__rate_interval]))","fullMetaSearch":false,"includeNullMetadata":false,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Dataplane update load (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Rate of dataplane operations, including updates to the dataplane and periodic resyncs.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":33},"id":20,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","exemplar":false,"expr":"topk($top_n, 60*rate(felix_int_dataplane_apply_time_seconds_count[1m]))","fullMetaSearch":false,"hide":false,"includeNullMetadata":false,"instant":false,"interval":"","legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Dataplane operations/min (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"The 99%ile tiem to update the dataplane in felix's main dataplane update loop.  Longer times indicate increasing latency to update the dataplane with new updates from the datastore.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":2},{"color":"red","value":5}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":12,"y":33},"id":10,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, felix_int_dataplane_apply_time_seconds{quantile=\"0.99\"})","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"99%ile dataplane update time (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Rate of errors returned from subprocesses such as iptables/ipset/etc.  Some background errors are expected due to resource contention in the kernel but high error rates should be investigated.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":1},{"color":"red","value":10}]},"unit":"none"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":18,"y":33},"id":22,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, rate(felix_ipset_errors[$__rate_interval]) + rate(felix_iptables_restore_errors[$__rate_interval]) + rate(felix_iptables_save_errors[$__rate_interval]))","fullMetaSearch":false,"hide":false,"includeNullMetadata":false,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Dataplane subprocess errors (top $top_n)","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":40},"id":42,"panels":[],"title":"Dataplane: iptables","type":"row"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of Calico-owned iptables rules","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":41},"id":43,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, sum by(instance) (felix_iptables_rules))","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Iptables rules (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of Calico-owned IP sets","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":41},"id":44,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, felix_ipsets_calico)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"IP sets (top $top_n)","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":48},"id":28,"panels":[],"title":"Dataplane: eBPF","type":"row"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Rate of events sent from BPF programs to Felix.  Events are used to tell Felix about new flows.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":49},"id":37,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, rate(felix_bpf_events{type!=\"lost\",type!=\"no_sink\"}[$__rate_interval])) ","fullMetaSearch":false,"includeNullMetadata":false,"legendFormat":"$instance_label type={{type}}","range":true,"refId":"A","useBackend":false}],"title":"Events/s (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of kernel-to-Felix events lost due to buffer overruns (Felix not keeping up with the event stream).","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","axisSoftMax":10,"barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"stepBefore","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":0},{"color":"red","value":1}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":49},"id":38,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, increase(felix_bpf_events{type=\"lost\"}[$__rate_interval])) ","fullMetaSearch":false,"includeNullMetadata":false,"legendFormat":"$instance_label type={{type}}","range":true,"refId":"A","useBackend":false}],"title":"Lost events (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of conntrack table entries seen at the last conntrack table garbage collection.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":12,"y":49},"id":30,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, felix_bpf_conntrack_used)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Conntrack table entries (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Time taken to scan the eBPF conntrack table","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":1000000000},{"color":"red","value":5000000000}]},"unit":"ns"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":18,"y":49},"id":32,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, felix_bpf_conntrack_sweep_duration)","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Conntrack sweep duration (top $top_n)","type":"timeseries"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":56},"id":31,"panels":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":57},"id":33,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, rate(felix_nflog_logs_received[$__rate_interval]))","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"NFLOG events/s (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of times the kernel-to-Felix buffer overran resulting in lost events (and hence missing flow log information)","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"stepBefore","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":0},{"color":"red","value":1}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":57},"id":34,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, increase(felix_nflog_buffer_overruns[$__rate_interval]))","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"NFLOG buffer overruns (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of buffered connection information objects waiting to be sent from the NFLOG reader to the log collector.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"#EAB839","value":1000}]}},"overrides":[]},"gridPos":{"h":7,"w":6,"x":12,"y":57},"id":35,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, clamp_min(felix_nflog_aggregates_created-felix_nflog_aggregates_flushed, 0))","fullMetaSearch":false,"includeNullMetadata":true,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"NFLOG backlog size (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Fraction of the time that the NFLOG reader thread spends waiting to send logs downstream.  Time spent waiting cannot be spent reading logs.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"#EAB839","value":0.01}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":18,"y":57},"id":36,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, rate(felix_nflog_block_time_seconds[$__rate_interval]))","fullMetaSearch":false,"includeNullMetadata":false,"legendFormat":"$instance_label group={{groupNum}}","range":true,"refId":"A","useBackend":false}],"title":"NFLOG flush wait time (top $top_n)","type":"timeseries"}],"title":"Flow logs dataplane: iptables/nftables","type":"row"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":57},"id":40,"panels":[],"title":"Flow logs collector","type":"row"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"dashed+area"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"#EAB839","value":0.05},{"color":"red","value":10}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":58},"id":39,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"builder","expr":"topk($top_n, rate(felix_collector_conntrack_processing_latency_seconds_sum[$__rate_interval]))","fullMetaSearch":false,"includeNullMetadata":false,"legendFormat":"$instance_label group={{groupNum}}","range":true,"refId":"A","useBackend":false}],"title":"Flow logs collector conntrack load (top $top_n)","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"description":"Number of pre-aggregation flows that the collector is tracking information about.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"barWidthFactor":0.6,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green"}]},"unit":"none"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":58},"id":41,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"hideZeros":false,"mode":"single","sort":"none"}},"pluginVersion":"12.0.0","targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"disableTextWrap":false,"editorMode":"code","expr":"topk($top_n, felix_collector_epstats)","fullMetaSearch":false,"includeNullMetadata":false,"legendFormat":"$instance_label","range":true,"refId":"A","useBackend":false}],"title":"Cached flows (top $top_n)","type":"timeseries"}],"preload":false,"refresh":"10s","schemaVersion":41,"tags":["kubernetes","calico","felix","bpf","dataplane"],"templating":{"list":[{"current":{"text":"50","value":"50"},"description":"","label":"Top N limit","name":"top_n","options":[{"selected":false,"text":"10","value":"10"},{"selected":false,"text":"1","value":"1"},{"selected":false,"text":"5","value":"5"},{"selected":false,"text":"20","value":"20"},{"selected":true,"text":"50","value":"50"},{"selected":false,"text":"100","value":"100"},{"selected":false,"text":"1000","value":"1000"},{"selected":false,"text":"10000","value":"10000"}],"query":"10,1,5,20,50,100,1000,10000","type":"custom"},{"current":{"text":"{{instance}}","value":"{{instance}}"},"description":"The label to apply to each instance of a metric.  For example, if your prometheurs metrics are labeled with pod name as a \"pod\", label, set this to {{pod}}","hide":2,"label":"Instance label","name":"instance_label","query":"{{instance}}","skipUrlSync":true,"type":"constant"}]},"time":{"from":"now-15m","to":"now"},"timepicker":{},"timezone":"browser","title":"Calico: Felix","uid":"beft70ojnzsw0e","version":3}
  typha-dashboard.json: |-
    {"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":null,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":23,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"refId":"A"}],"title":"Top-level metrics","type":"row"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":1},"id":10,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"max by (instance) (typha_cache_size)","format":"time_series","interval":"2s","intervalFactor":2,"legendFormat":"{{pod}}","metric":"typha_connections_active","range":true,"refId":"A","step":5}],"title":"Datastore cache size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":1},"hideTimeOverride":false,"id":4,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"rate(process_cpu_seconds_total{job=\"typha_metrics\"}[30s]) ","format":"time_series","interval":"100ms","intervalFactor":2,"legendFormat":"{{instance}}","metric":"process_cpu_seconds_total","range":true,"refId":"A","step":5}],"title":"CPU Usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":12,"y":1},"id":5,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"process_resident_memory_bytes{job=\"typha_metrics\"}\n","format":"time_series","interval":"200ms","intervalFactor":2,"legendFormat":"{{instance}}","metric":"go_memstats_alloc_bytes","range":true,"refId":"A","step":5}],"title":"Memory Usage","type":"timeseries"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"gridPos":{"h":1,"w":24,"x":0,"y":8},"id":25,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"refId":"A"}],"title":"Clients","type":"row"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":4,"x":0,"y":9},"id":17,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"rate(typha_connections_accepted[1m])","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A","step":10}],"title":"Total connections accepted","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":4,"x":4,"y":9},"id":1,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"sum by (instance) (typha_connections_streaming)","format":"time_series","interval":"2s","intervalFactor":2,"legendFormat":"{{pod}}","metric":"typha_connections_active","range":true,"refId":"A","step":5}],"title":"Client connections actively streaming","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"description":"Number of client connections dropped to rebalance load.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Number of dropped connections","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":4,"x":8,"y":9},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"rate(typha_connections_dropped[$__rate_interval])","format":"time_series","interval":"2s","intervalFactor":2,"legendFormat":"{{instance}}","metric":"typha_connections_dropped","range":true,"refId":"A","step":5}],"title":"Rebalanced client connenctions","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"description":"This metric measures how far behind Typha's clients are at reading updates.\nIt goes up if the client is slow or if Typha is overloaded and it cannot keep up with writes to all its clients.\nIt is normal for this to spike when new clients connect, they must download and process the snapshot, during which time they get slightly behind.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line+area"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":null},{"color":"#EAB839","value":10},{"color":"red","value":60}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":4,"x":12,"y":9},"id":3,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"max by (instance) (typha_client_latency_secs{quantile='0.99'})","format":"time_series","interval":"2s","intervalFactor":2,"legendFormat":"{{instance}}","metric":"typha_client_latency_secs_sum","range":true,"refId":"A","step":5}],"title":"99%ile Client fall-behind","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":4,"x":16,"y":9},"id":22,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"max by (instance) (typha_client_write_latency_secs)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A","step":10}],"title":"99%ile write latency","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"description":"This metric tracks how long it takes for Typha's clients to respond to pings over the Typha protocol.\nAn increase in this metric indicates that the clients, network or Typha are more heavily loaded.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line+area"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":null},{"color":"red","value":1}]},"unit":"s"},"overrides":[]},"gridPos":{"h":7,"w":4,"x":20,"y":9},"id":13,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"max by (instance) (typha_ping_latency{quantile=\"0.99\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A","step":10}],"title":"99%ile client ping latency","type":"timeseries"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"gridPos":{"h":1,"w":24,"x":0,"y":16},"id":26,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"refId":"A"}],"title":"Cache internals","type":"row"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"description":"Internally, Typha stores datastore changes as a series of blocks called breadcrumbs.\nThe default maximum size of a block is 100 so, if Typha is consistently generating size-100 blocks it may be overloaded.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line+area"}},"mappings":[],"max":110,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":null},{"color":"red","value":95}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":6,"x":0,"y":17},"id":9,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"max by (instance) (typha_breadcrumb_size{quantile=\"0.99\"})","format":"time_series","interval":"1s","intervalFactor":2,"legendFormat":"{{instance}} ","range":true,"refId":"A","step":5}],"title":"99%ile Breadcrumb Size","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"description":"Internally, Typha stores datastore changes as a series of blocks called \"breadcrumbs\".\nEach client \"follows the breadcrumbs\" either by blocking and waiting or skipping straight to the next one (non-blocking) if it is already available.\nAs load on Typha increases, the ratio of skip ahead non-blocking reads increases.  If it approaches 100% then Typha may be overloaded (since clients only do non-blocking reads when they're behind).","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"line+area"}},"mappings":[],"max":1.1,"thresholds":{"mode":"absolute","steps":[{"color":"transparent","value":null},{"color":"#EAB839","value":0.9},{"color":"red","value":1}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":10,"w":6,"x":6,"y":17},"id":8,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"(sum by (instance) (rate(typha_breadcrumb_non_block[30s])))/((sum by (instance) (rate(typha_breadcrumb_non_block[30s])))+(sum by (instance) (rate(typha_breadcrumb_block[30s]))))","format":"time_series","interval":"1s","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A","step":5}],"title":"Non-blocking breadcrumb fraction","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"description":"This panel shows the rate of updates from the datastore(s).\nFor example, updates to Pods/Nodes/Policies/etc.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":6,"x":12,"y":17},"id":15,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"sum by (instance) (rate(typha_updates_total[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A","step":10}],"title":"Datastore updates total","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"description":"This panel shows the number of updates from the datastore that Typha detected were no-ops.\nFor example, an update to a Kubernetes Node resource that didn't touch any values that Calico cares about.\nSuch updates are not propagated to clients, saving resources.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"#EAB839","value":""}]},"unit":"short"},"overrides":[]},"gridPos":{"h":10,"w":6,"x":18,"y":17},"id":16,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"sum by (instance) (rate(typha_updates_skipped[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"B","step":10}],"title":"Datastore updates skipped as no-ops","type":"timeseries"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"gridPos":{"h":1,"w":24,"x":0,"y":27},"id":27,"panels":[],"targets":[{"datasource":{"type":"prometheus","uid":"P11C5FC3F3B681947"},"refId":"A"}],"title":"Snapshot details","type":"row"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"description":"This panel shows the 99%ile time to stream the initial datastore snapshot to each clients.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"s"},"overrides":[]},"gridPos":{"h":9,"w":6,"x":0,"y":28},"id":12,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"max by (instance) (typha_client_snapshot_send_secs{quantile=\"0.5\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A","step":10}],"title":"Median snapshot send time","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"description":"After sending the snapshot to the client, Typha allows a grace period for the client to catch up to the most recent data.\nThis metric tracks how many clients required the grace period.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":9,"w":6,"x":6,"y":28},"id":21,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"sum by (instance) (typha_connections_grace_used)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","range":true,"refId":"A","step":10}],"title":"Clients requiring grace period","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":12,"y":28},"id":18,"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"max(typha_snapshot_raw_bytes)","format":"time_series","intervalFactor":2,"legendFormat":"","range":true,"refId":"A","step":40}],"title":"Max snapshot size (raw)","type":"stat"},{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":18,"y":28},"id":19,"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"10.4.0","targets":[{"datasource":{"type":"prometheus","uid":"calico-demo-prometheus"},"editorMode":"code","expr":"max(typha_snapshot_compressed_bytes)","format":"time_series","intervalFactor":2,"legendFormat":"","range":true,"refId":"A","step":40}],"title":"Max snapshot size (compressed)","type":"stat"}],"refresh":"","schemaVersion":39,"tags":[],"templating":{"list":[]},"time":{"from":"now-30m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"Calico Typha","uid":"calico-typha-dashboard","version":7,"weekStart":""}