apiVersion: v1 kind: Namespace metadata: name: flow-visibility --- allowVolumeExpansion: true apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: grafana-storage provisioner: kubernetes.io/no-provisioner reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer --- apiVersion: v1 kind: ServiceAccount metadata: name: grafana namespace: flow-visibility --- apiVersion: v1 kind: ServiceAccount metadata: labels: app: theia-cli name: theia-cli namespace: flow-visibility --- apiVersion: v1 kind: ServiceAccount metadata: labels: app: theia-manager name: theia-manager namespace: flow-visibility --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: labels: app: flow-visibility name: grafana-role namespace: flow-visibility rules: - apiGroups: - "" resources: - services verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app: theia-cli name: theia-cli rules: - apiGroups: - intelligence.theia.antrea.io resources: - networkpolicyrecommendations - throughputanomalydetectors verbs: - get - list - create - delete - apiGroups: - stats.theia.antrea.io resources: - clickhouse verbs: - get - apiGroups: - system.theia.antrea.io resources: - supportbundles verbs: - get - create - delete - apiGroups: - system.theia.antrea.io resources: - supportbundles/download verbs: - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app: theia-manager name: theia-manager-role rules: - apiGroups: - authentication.k8s.io resources: - tokenreviews verbs: - create - apiGroups: - authorization.k8s.io resources: - subjectaccessreviews verbs: - create - apiGroups: - "" resourceNames: - theia-ca resources: - configmaps verbs: - get - update - apiGroups: - "" resources: - configmaps verbs: - create - apiGroups: - "" resourceNames: - extension-apiserver-authentication resources: - configmaps verbs: - get - list - watch - apiGroups: - crd.theia.antrea.io resources: - networkpolicyrecommendations - recommendednetworkpolicies - throughputanomalydetectors verbs: - get - list - watch - create - delete - apiGroups: - crd.theia.antrea.io resources: - networkpolicyrecommendations/status - throughputanomalydetectors/status verbs: - update - apiGroups: - "" resources: - pods verbs: - get - list - apiGroups: - "" resources: - pods/exec verbs: - get - create - apiGroups: - "" resources: - pods/log verbs: - get - apiGroups: - "" resources: - services - secrets verbs: - get - apiGroups: - sparkoperator.k8s.io resources: - sparkapplications verbs: - create - delete - get - list --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: app: flow-visibility name: grafana-role-binding namespace: flow-visibility roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: grafana-role subjects: - kind: ServiceAccount name: grafana namespace: flow-visibility --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app: theia-cli name: theia-cli roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: theia-cli subjects: - kind: ServiceAccount name: theia-cli namespace: flow-visibility --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: app: theia-manager name: theia-manager-cluster-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: theia-manager-role subjects: - kind: ServiceAccount name: theia-manager namespace: flow-visibility --- apiVersion: v1 data: 000001_0-1-0.down.sql: "" 000001_0-1-0.up.sql: | --Create a table to store records CREATE TABLE IF NOT EXISTS flows_local ( timeInserted DateTime DEFAULT now(), flowStartSeconds DateTime, flowEndSeconds DateTime, flowEndSecondsFromSourceNode DateTime, flowEndSecondsFromDestinationNode DateTime, flowEndReason UInt8, sourceIP String, destinationIP String, sourceTransportPort UInt16, destinationTransportPort UInt16, protocolIdentifier UInt8, packetTotalCount UInt64, octetTotalCount UInt64, packetDeltaCount UInt64, octetDeltaCount UInt64, reversePacketTotalCount UInt64, reverseOctetTotalCount UInt64, reversePacketDeltaCount UInt64, reverseOctetDeltaCount UInt64, sourcePodName String, sourcePodNamespace String, sourceNodeName String, destinationPodName String, destinationPodNamespace String, destinationNodeName String, destinationClusterIP String, destinationServicePort UInt16, destinationServicePortName String, ingressNetworkPolicyName String, ingressNetworkPolicyNamespace String, ingressNetworkPolicyRuleName String, ingressNetworkPolicyRuleAction UInt8, ingressNetworkPolicyType UInt8, egressNetworkPolicyName String, egressNetworkPolicyNamespace String, egressNetworkPolicyRuleName String, egressNetworkPolicyRuleAction UInt8, egressNetworkPolicyType UInt8, tcpState String, flowType UInt8, sourcePodLabels String, destinationPodLabels String, throughput UInt64, reverseThroughput UInt64, throughputFromSourceNode UInt64, throughputFromDestinationNode UInt64, reverseThroughputFromSourceNode UInt64, reverseThroughputFromDestinationNode UInt64, trusted UInt8 DEFAULT 0 ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY (timeInserted, flowEndSeconds); --Move data from old table and drop old tables INSERT INTO flows_local SELECT * FROM flows; DROP TABLE flows; DROP VIEW flows_pod_view; DROP VIEW flows_node_view; DROP VIEW flows_policy_view; --Create a table to store the network policy recommendation results CREATE TABLE IF NOT EXISTS recommendations_local ( id String, type String, timeCreated DateTime, yamls String ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY (timeCreated); --Move data from old table and drop the old table INSERT INTO recommendations_local SELECT * FROM recommendations; DROP TABLE recommendations; CREATE TABLE IF NOT EXISTS flows AS flows_local engine=Distributed('{cluster}', default, flows_local, rand()); CREATE TABLE IF NOT EXISTS recommendations AS recommendations_local engine=Distributed('{cluster}', default, recommendations_local, rand()); 000002_0-2-0.down.sql: | --Alter table to drop new columns ALTER TABLE flows DROP clusterUUID String; ALTER TABLE flows_local DROP clusterUUID String; 000002_0-2-0.up.sql: | --Alter table to add new columns ALTER TABLE flows ADD COLUMN clusterUUID String; ALTER TABLE flows_local ADD COLUMN clusterUUID String; 000003_0-3-0.down.sql: | --Remove structured recommendations --Add yamls column ALTER TABLE recommendations ADD COLUMN yamls String; ALTER TABLE recommendations_local ADD COLUMN yamls String; --Copy old data and replace policy and kind by yamls INSERT INTO recommendations_local (id, type, timeCreated, yamls) SELECT id, type, timeCreated, arrayStringConcat(groupArray(policy), '\n---\n') AS yamls FROM recommendations_local GROUP BY id, type, timeCreated; --Delete old data ALTER TABLE recommendations_local DELETE WHERE yamls=''; --Drop yamls column ALTER TABLE recommendations DROP COLUMN kind; ALTER TABLE recommendations_local DROP COLUMN kind; ALTER TABLE recommendations DROP COLUMN policy; ALTER TABLE recommendations_local DROP COLUMN policy; 000003_0-3-0.up.sql: | --Support structured recommendations --Add policy and kind column ALTER TABLE recommendations ADD COLUMN kind String; ALTER TABLE recommendations_local ADD COLUMN kind String; ALTER TABLE recommendations ADD COLUMN policy String; ALTER TABLE recommendations_local ADD COLUMN policy String; --Copy old data and replace yamls column by policy and kind INSERT INTO recommendations_local (id, type, timeCreated, policy) SELECT id, type, timeCreated, arrayJoin(splitByString('---\n', yamls)) AS policy FROM recommendations_local WHERE kind=''; ALTER TABLE recommendations_local UPDATE kind='knp' WHERE policy LIKE '%networking.k8s.io/v1%NetworkPolicy%'; ALTER TABLE recommendations_local UPDATE kind='anp' WHERE policy LIKE '%crd.antrea.io/v1alpha1%NetworkPolicy%'; ALTER TABLE recommendations_local UPDATE kind='acnp' WHERE policy LIKE '%ClusterNetworkPolicy%'; ALTER TABLE recommendations_local UPDATE kind='acg' WHERE policy LIKE '%ClusterGroup%'; --Delete old data ALTER TABLE recommendations_local DELETE WHERE policy=''; --Drop yamls column ALTER TABLE recommendations DROP COLUMN yamls; ALTER TABLE recommendations_local DROP COLUMN yamls; 000004_0-4-0.down.sql: | --Drop table DROP tadetector_local 000004_0-4-0.up.sql: | --Create a table to store the Throughput Anomaly Detector results CREATE TABLE IF NOT EXISTS tadetector_local ( sourceIP String, sourceTransportPort UInt16, destinationIP String, destinationTransportPort UInt16, protocolIdentifier UInt16, flowStartSeconds DateTime, flowEndSeconds DateTime, throughputStandardDeviation Float64, algoType String, algoCalc Float64, throughput Float64, anomaly String, id String ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY (flowStartSeconds); CREATE TABLE IF NOT EXISTS tadetector AS tadetector_local engine=Distributed('{cluster}', default, tadetector_local, rand()); 000005_0-6-0.down.sql: | DROP VIEW flows_pod_view_local; DROP VIEW flows_node_view_local; DROP VIEW flows_policy_view_local; --Create a Materialized View to aggregate data for pods CREATE MATERIALIZED VIEW IF NOT EXISTS flows_pod_view_local ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY ( timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourcePodName, destinationPodName, destinationIP, destinationServicePort, destinationServicePortName, flowType, sourcePodNamespace, destinationPodNamespace, sourceTransportPort, destinationTransportPort, clusterUUID) POPULATE AS SELECT timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourcePodName, destinationPodName, destinationIP, destinationServicePort, destinationServicePortName, flowType, sourcePodNamespace, destinationPodNamespace, sourceTransportPort, destinationTransportPort, sum(octetDeltaCount) AS octetDeltaCount, sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount, sum(throughput) AS throughput, sum(reverseThroughput) AS reverseThroughput, sum(throughputFromSourceNode) AS throughputFromSourceNode, sum(throughputFromDestinationNode) AS throughputFromDestinationNode, clusterUUID FROM flows_local GROUP BY timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourcePodName, destinationPodName, destinationIP, destinationServicePort, destinationServicePortName, flowType, sourcePodNamespace, destinationPodNamespace, sourceTransportPort, destinationTransportPort, clusterUUID; --Create a Materialized View to aggregate data for nodes CREATE MATERIALIZED VIEW IF NOT EXISTS flows_node_view_local ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY ( timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourceNodeName, destinationNodeName, sourcePodNamespace, destinationPodNamespace, clusterUUID) POPULATE AS SELECT timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourceNodeName, destinationNodeName, sourcePodNamespace, destinationPodNamespace, sum(octetDeltaCount) AS octetDeltaCount, sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount, sum(throughput) AS throughput, sum(reverseThroughput) AS reverseThroughput, sum(throughputFromSourceNode) AS throughputFromSourceNode, sum(reverseThroughputFromSourceNode) AS reverseThroughputFromSourceNode, sum(throughputFromDestinationNode) AS throughputFromDestinationNode, sum(reverseThroughputFromDestinationNode) AS reverseThroughputFromDestinationNode, clusterUUID FROM flows_local GROUP BY timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourceNodeName, destinationNodeName, sourcePodNamespace, destinationPodNamespace, clusterUUID; --Create a Materialized View to aggregate data for network policies CREATE MATERIALIZED VIEW IF NOT EXISTS flows_policy_view_local ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY ( timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, egressNetworkPolicyName, egressNetworkPolicyNamespace, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyNamespace, ingressNetworkPolicyRuleAction, sourcePodName, sourceTransportPort, sourcePodNamespace, destinationPodName, destinationTransportPort, destinationPodNamespace, destinationServicePort, destinationServicePortName, destinationIP, clusterUUID) POPULATE AS SELECT timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, egressNetworkPolicyName, egressNetworkPolicyNamespace, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyNamespace, ingressNetworkPolicyRuleAction, sourcePodName, sourceTransportPort, sourcePodNamespace, destinationPodName, destinationTransportPort, destinationPodNamespace, destinationServicePort, destinationServicePortName, destinationIP, sum(octetDeltaCount) AS octetDeltaCount, sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount, sum(throughput) AS throughput, sum(reverseThroughput) AS reverseThroughput, sum(throughputFromSourceNode) AS throughputFromSourceNode, sum(reverseThroughputFromSourceNode) AS reverseThroughputFromSourceNode, sum(throughputFromDestinationNode) AS throughputFromDestinationNode, sum(reverseThroughputFromDestinationNode) AS reverseThroughputFromDestinationNode, clusterUUID FROM flows_local GROUP BY timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, egressNetworkPolicyName, egressNetworkPolicyNamespace, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyNamespace, ingressNetworkPolicyRuleAction, sourcePodName, sourceTransportPort, sourcePodNamespace, destinationPodName, destinationTransportPort, destinationPodNamespace, destinationServicePort, destinationServicePortName, destinationIP, clusterUUID; INSERT INTO ".inner.flows_pod_view_local" SELECT * FROM pod_view_table_local; INSERT INTO ".inner.flows_node_view_local" SELECT * FROM node_view_table_local; INSERT INTO ".inner.flows_policy_view_local" SELECT * FROM policy_view_table_local; DROP TABLE pod_view_table_local; DROP TABLE node_view_table_local; DROP TABLE policy_view_table_local; --Alter table to drop new columns ALTER TABLE flows DROP COLUMN egressName, DROP COLUMN egressIP; ALTER TABLE flows_local DROP COLUMN egressName, DROP COLUMN egressIP; ALTER TABLE tadetector DROP COLUMN podNamespace, DROP COLUMN podLabels, DROP COLUMN destinationServicePortName, DROP COLUMN aggType, DROP COLUMN direction, DROP COLUMN podName; ALTER TABLE tadetector_local DROP COLUMN podNamespace, DROP COLUMN podLabels, DROP COLUMN destinationServicePortName, DROP COLUMN aggType, DROP COLUMN direction, DROP COLUMN podName; 000005_0-6-0.up.sql: | -- Create underlying tables for Materialized Views to attach data CREATE TABLE IF NOT EXISTS pod_view_table_local ( timeInserted DateTime DEFAULT now(), flowEndSeconds DateTime, flowEndSecondsFromSourceNode DateTime, flowEndSecondsFromDestinationNode DateTime, sourcePodName String, destinationPodName String, destinationIP String, destinationServicePort UInt16, destinationServicePortName String, flowType UInt8, sourcePodNamespace String, destinationPodNamespace String, sourceTransportPort UInt16, destinationTransportPort UInt16, octetDeltaCount UInt64, reverseOctetDeltaCount UInt64, throughput UInt64, reverseThroughput UInt64, throughputFromSourceNode UInt64, throughputFromDestinationNode UInt64, clusterUUID String ) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY ( timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourcePodName, destinationPodName, destinationIP, destinationServicePort, destinationServicePortName, flowType, sourcePodNamespace, destinationPodNamespace, sourceTransportPort, destinationTransportPort, clusterUUID); CREATE TABLE IF NOT EXISTS node_view_table_local ( timeInserted DateTime DEFAULT now(), flowEndSeconds DateTime, flowEndSecondsFromSourceNode DateTime, flowEndSecondsFromDestinationNode DateTime, sourceNodeName String, destinationNodeName String, sourcePodNamespace String, destinationPodNamespace String, octetDeltaCount UInt64, reverseOctetDeltaCount UInt64, throughput UInt64, reverseThroughput UInt64, throughputFromSourceNode UInt64, reverseThroughputFromSourceNode UInt64, throughputFromDestinationNode UInt64, reverseThroughputFromDestinationNode UInt64, clusterUUID String ) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY ( timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourceNodeName, destinationNodeName, sourcePodNamespace, destinationPodNamespace, clusterUUID); CREATE TABLE IF NOT EXISTS policy_view_table_local ( timeInserted DateTime DEFAULT now(), flowEndSeconds DateTime, flowEndSecondsFromSourceNode DateTime, flowEndSecondsFromDestinationNode DateTime, egressNetworkPolicyName String, egressNetworkPolicyNamespace String, egressNetworkPolicyRuleAction UInt8, ingressNetworkPolicyName String, ingressNetworkPolicyNamespace String, ingressNetworkPolicyRuleAction UInt8, sourcePodName String, sourceTransportPort UInt16, sourcePodNamespace String, destinationPodName String, destinationTransportPort UInt16, destinationPodNamespace String, destinationServicePort UInt16, destinationServicePortName String, destinationIP String, octetDeltaCount UInt64, reverseOctetDeltaCount UInt64, throughput UInt64, reverseThroughput UInt64, throughputFromSourceNode UInt64, reverseThroughputFromSourceNode UInt64, throughputFromDestinationNode UInt64, reverseThroughputFromDestinationNode UInt64, clusterUUID String ) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY ( timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, egressNetworkPolicyName, egressNetworkPolicyNamespace, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyNamespace, ingressNetworkPolicyRuleAction, sourcePodName, sourceTransportPort, sourcePodNamespace, destinationPodName, destinationTransportPort, destinationPodNamespace, destinationServicePort, destinationServicePortName, destinationIP, clusterUUID); --Move data from old mv underlying tables and drop old mvs INSERT INTO pod_view_table_local SELECT * FROM ".inner.flows_pod_view_local"; INSERT INTO node_view_table_local SELECT * FROM ".inner.flows_node_view_local"; INSERT INTO policy_view_table_local SELECT * FROM ".inner.flows_policy_view_local"; DROP VIEW flows_pod_view_local; DROP VIEW flows_node_view_local; DROP VIEW flows_policy_view_local; --Alter table to add new columns ALTER TABLE flows ADD COLUMN egressName String, ADD COLUMN egressIP String; ALTER TABLE flows_local ADD COLUMN egressName String, ADD COLUMN egressIP String; ALTER TABLE tadetector ADD COLUMN podNamespace String, ADD COLUMN podLabels String, ADD COLUMN destinationServicePortName String, ADD COLUMN aggType String, ADD COLUMN direction String, ADD COLUMN podName String; ALTER TABLE tadetector_local ADD COLUMN podNamespace String, ADD COLUMN podLabels String, ADD COLUMN destinationServicePortName String, ADD COLUMN aggType String, ADD COLUMN direction String, ADD COLUMN podName String; create_table.sh: | #!/usr/bin/env bash # Copyright 2022 Antrea Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ttlTimeout is calculated by the smaller value in its default value and TTL function createTable { clickhouse client -n -h 127.0.0.1 <<-EOSQL --Create a table to store records CREATE TABLE IF NOT EXISTS flows_local ( timeInserted DateTime DEFAULT now(), flowStartSeconds DateTime, flowEndSeconds DateTime, flowEndSecondsFromSourceNode DateTime, flowEndSecondsFromDestinationNode DateTime, flowEndReason UInt8, sourceIP String, destinationIP String, sourceTransportPort UInt16, destinationTransportPort UInt16, protocolIdentifier UInt8, packetTotalCount UInt64, octetTotalCount UInt64, packetDeltaCount UInt64, octetDeltaCount UInt64, reversePacketTotalCount UInt64, reverseOctetTotalCount UInt64, reversePacketDeltaCount UInt64, reverseOctetDeltaCount UInt64, sourcePodName String, sourcePodNamespace String, sourceNodeName String, destinationPodName String, destinationPodNamespace String, destinationNodeName String, destinationClusterIP String, destinationServicePort UInt16, destinationServicePortName String, ingressNetworkPolicyName String, ingressNetworkPolicyNamespace String, ingressNetworkPolicyRuleName String, ingressNetworkPolicyRuleAction UInt8, ingressNetworkPolicyType UInt8, egressNetworkPolicyName String, egressNetworkPolicyNamespace String, egressNetworkPolicyRuleName String, egressNetworkPolicyRuleAction UInt8, egressNetworkPolicyType UInt8, tcpState String, flowType UInt8, sourcePodLabels String, destinationPodLabels String, throughput UInt64, reverseThroughput UInt64, throughputFromSourceNode UInt64, throughputFromDestinationNode UInt64, reverseThroughputFromSourceNode UInt64, reverseThroughputFromDestinationNode UInt64, clusterUUID String, egressName String, egressIP String, trusted UInt8 DEFAULT 0 ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY (timeInserted, flowEndSeconds); ALTER TABLE flows_local MODIFY TTL timeInserted + INTERVAL 12 HOUR; ALTER TABLE flows_local MODIFY SETTING merge_with_ttl_timeout=14400; --Create a Materialized View to aggregate data for Pods and save the data --to default.pod_view_table_local CREATE TABLE IF NOT EXISTS pod_view_table_local ( timeInserted DateTime DEFAULT now(), flowEndSeconds DateTime, flowEndSecondsFromSourceNode DateTime, flowEndSecondsFromDestinationNode DateTime, sourcePodName String, destinationPodName String, destinationIP String, destinationServicePort UInt16, destinationServicePortName String, flowType UInt8, sourcePodNamespace String, destinationPodNamespace String, sourceTransportPort UInt16, destinationTransportPort UInt16, octetDeltaCount UInt64, reverseOctetDeltaCount UInt64, throughput UInt64, reverseThroughput UInt64, throughputFromSourceNode UInt64, throughputFromDestinationNode UInt64, clusterUUID String ) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY ( timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourcePodName, destinationPodName, destinationIP, destinationServicePort, destinationServicePortName, flowType, sourcePodNamespace, destinationPodNamespace, sourceTransportPort, destinationTransportPort, clusterUUID); ALTER TABLE "pod_view_table_local" MODIFY TTL timeInserted + INTERVAL 12 HOUR; ALTER TABLE "pod_view_table_local" MODIFY SETTING merge_with_ttl_timeout=14400; CREATE MATERIALIZED VIEW IF NOT EXISTS flows_pod_view_local TO pod_view_table_local AS SELECT timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourcePodName, destinationPodName, destinationIP, destinationServicePort, destinationServicePortName, flowType, sourcePodNamespace, destinationPodNamespace, sourceTransportPort, destinationTransportPort, sum(octetDeltaCount) AS octetDeltaCount, sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount, sum(throughput) AS throughput, sum(reverseThroughput) AS reverseThroughput, sum(throughputFromSourceNode) AS throughputFromSourceNode, sum(throughputFromDestinationNode) AS throughputFromDestinationNode, clusterUUID FROM flows_local GROUP BY timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourcePodName, destinationPodName, destinationIP, destinationServicePort, destinationServicePortName, flowType, sourcePodNamespace, destinationPodNamespace, sourceTransportPort, destinationTransportPort, clusterUUID; --Create a Materialized View to aggregate data for Nodes and save the data --to default.node_view_table_local CREATE TABLE IF NOT EXISTS node_view_table_local ( timeInserted DateTime DEFAULT now(), flowEndSeconds DateTime, flowEndSecondsFromSourceNode DateTime, flowEndSecondsFromDestinationNode DateTime, sourceNodeName String, destinationNodeName String, sourcePodNamespace String, destinationPodNamespace String, octetDeltaCount UInt64, reverseOctetDeltaCount UInt64, throughput UInt64, reverseThroughput UInt64, throughputFromSourceNode UInt64, reverseThroughputFromSourceNode UInt64, throughputFromDestinationNode UInt64, reverseThroughputFromDestinationNode UInt64, clusterUUID String ) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY ( timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourceNodeName, destinationNodeName, sourcePodNamespace, destinationPodNamespace, clusterUUID); ALTER TABLE "node_view_table_local" MODIFY TTL timeInserted + INTERVAL 12 HOUR; ALTER TABLE "node_view_table_local" MODIFY SETTING merge_with_ttl_timeout=14400; CREATE MATERIALIZED VIEW IF NOT EXISTS flows_node_view_local TO node_view_table_local AS SELECT timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourceNodeName, destinationNodeName, sourcePodNamespace, destinationPodNamespace, sum(octetDeltaCount) AS octetDeltaCount, sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount, sum(throughput) AS throughput, sum(reverseThroughput) AS reverseThroughput, sum(throughputFromSourceNode) AS throughputFromSourceNode, sum(reverseThroughputFromSourceNode) AS reverseThroughputFromSourceNode, sum(throughputFromDestinationNode) AS throughputFromDestinationNode, sum(reverseThroughputFromDestinationNode) AS reverseThroughputFromDestinationNode, clusterUUID FROM flows_local GROUP BY timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, sourceNodeName, destinationNodeName, sourcePodNamespace, destinationPodNamespace, clusterUUID; --Create a Materialized View to aggregate data for network policies and --save the data to default.policy_view_table_local CREATE TABLE IF NOT EXISTS policy_view_table_local ( timeInserted DateTime DEFAULT now(), flowEndSeconds DateTime, flowEndSecondsFromSourceNode DateTime, flowEndSecondsFromDestinationNode DateTime, egressNetworkPolicyName String, egressNetworkPolicyNamespace String, egressNetworkPolicyRuleAction UInt8, ingressNetworkPolicyName String, ingressNetworkPolicyNamespace String, ingressNetworkPolicyRuleAction UInt8, sourcePodName String, sourceTransportPort UInt16, sourcePodNamespace String, destinationPodName String, destinationTransportPort UInt16, destinationPodNamespace String, destinationServicePort UInt16, destinationServicePortName String, destinationIP String, octetDeltaCount UInt64, reverseOctetDeltaCount UInt64, throughput UInt64, reverseThroughput UInt64, throughputFromSourceNode UInt64, reverseThroughputFromSourceNode UInt64, throughputFromDestinationNode UInt64, reverseThroughputFromDestinationNode UInt64, clusterUUID String ) ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY ( timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, egressNetworkPolicyName, egressNetworkPolicyNamespace, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyNamespace, ingressNetworkPolicyRuleAction, sourcePodName, sourceTransportPort, sourcePodNamespace, destinationPodName, destinationTransportPort, destinationPodNamespace, destinationServicePort, destinationServicePortName, destinationIP, clusterUUID); ALTER TABLE "policy_view_table_local" MODIFY TTL timeInserted + INTERVAL 12 HOUR; ALTER TABLE "policy_view_table_local" MODIFY SETTING merge_with_ttl_timeout=14400; CREATE MATERIALIZED VIEW IF NOT EXISTS flows_policy_view_local to policy_view_table_local AS SELECT timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, egressNetworkPolicyName, egressNetworkPolicyNamespace, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyNamespace, ingressNetworkPolicyRuleAction, sourcePodName, sourceTransportPort, sourcePodNamespace, destinationPodName, destinationTransportPort, destinationPodNamespace, destinationServicePort, destinationServicePortName, destinationIP, sum(octetDeltaCount) AS octetDeltaCount, sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount, sum(throughput) AS throughput, sum(reverseThroughput) AS reverseThroughput, sum(throughputFromSourceNode) AS throughputFromSourceNode, sum(reverseThroughputFromSourceNode) AS reverseThroughputFromSourceNode, sum(throughputFromDestinationNode) AS throughputFromDestinationNode, sum(reverseThroughputFromDestinationNode) AS reverseThroughputFromDestinationNode, clusterUUID FROM flows_local GROUP BY timeInserted, flowEndSeconds, flowEndSecondsFromSourceNode, flowEndSecondsFromDestinationNode, egressNetworkPolicyName, egressNetworkPolicyNamespace, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyNamespace, ingressNetworkPolicyRuleAction, sourcePodName, sourceTransportPort, sourcePodNamespace, destinationPodName, destinationTransportPort, destinationPodNamespace, destinationServicePort, destinationServicePortName, destinationIP, clusterUUID; --Create a table to store the network policy recommendation results CREATE TABLE IF NOT EXISTS recommendations_local ( id String, type String, timeCreated DateTime, policy String, kind String ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY (timeCreated); --Create a table to store the Throughput Anomaly Detector results CREATE TABLE IF NOT EXISTS tadetector_local ( sourceIP String, sourceTransportPort UInt16, destinationIP String, destinationTransportPort UInt16, protocolIdentifier UInt16, flowStartSeconds DateTime, podNamespace String, podLabels String, podName String, destinationServicePortName String, direction String, flowEndSeconds DateTime, throughputStandardDeviation Float64, aggType String, algoType String, algoCalc Float64, throughput Float64, anomaly String, id String ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY (flowStartSeconds); --Create distributed tables for cluster CREATE TABLE IF NOT EXISTS flows AS flows_local engine=Distributed('{cluster}', default, flows_local, rand()); CREATE TABLE IF NOT EXISTS flows_pod_view AS flows_pod_view_local engine=Distributed('{cluster}', default, flows_pod_view_local, rand()); CREATE TABLE IF NOT EXISTS flows_node_view AS flows_node_view_local engine=Distributed('{cluster}', default, flows_node_view_local, rand()); CREATE TABLE IF NOT EXISTS flows_policy_view AS flows_policy_view_local engine=Distributed('{cluster}', default, flows_policy_view_local, rand()); CREATE TABLE IF NOT EXISTS recommendations AS recommendations_local engine=Distributed('{cluster}', default, recommendations_local, rand()); CREATE TABLE IF NOT EXISTS tadetector AS tadetector_local engine=Distributed('{cluster}', default, tadetector_local, rand()); EOSQL } init.sh: | #!/usr/bin/env bash # Copyright 2022 Antrea Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source $THIS_DIR/create_table.sh ../clickhouse-schema-management createTable kind: ConfigMap metadata: name: clickhouse-mounted-configmap namespace: flow-visibility --- apiVersion: v1 data: flow_records_dashboard.json: |- { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 4, "iteration": 1659392847247, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 5, "x": 0, "y": 0 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT count(*) as count\nFROM flows\nWHERE $__timeFilter(flowEndSeconds)", "refId": "A" } ], "title": "Flow Records Count", "transparent": true, "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 19, "x": 5, "y": 0 }, "id": 6, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT count() as count, $__timeInterval(flowEndSeconds) as time\nFROM flows\nWHERE $__timeFilter(flowEndSeconds)\nGROUP BY time\nORDER BY time", "refId": "A" } ], "title": "Flow Records Count", "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "filterable": true }, "mappings": [], "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "flowType" }, "properties": [ { "id": "mappings", "value": [ { "options": { "1": { "index": 0, "text": "Intra-Node" }, "2": { "index": 1, "text": "Inter-Node" }, "3": { "index": 2, "text": "To External" }, "4": { "index": 3, "text": "From External" } }, "type": "value" } ] } ] }, { "matcher": { "id": "byName", "options": "sourcePodNamespace" }, "properties": [ { "id": "custom.width", "value": 199 } ] }, { "matcher": { "id": "byName", "options": "destinationPodName" }, "properties": [ { "id": "custom.width", "value": 183 } ] }, { "matcher": { "id": "byName", "options": "destinationPodNamespace" }, "properties": [ { "id": "custom.width", "value": 214 } ] }, { "matcher": { "id": "byName", "options": "destinationNodeName" }, "properties": [ { "id": "custom.width", "value": 183 } ] }, { "matcher": { "id": "byName", "options": "sourceNodeName" }, "properties": [ { "id": "custom.width", "value": 161 } ] }, { "matcher": { "id": "byName", "options": "sourcePodName" }, "properties": [ { "id": "custom.width", "value": 149 } ] }, { "matcher": { "id": "byName", "options": "destinationServicePort" }, "properties": [ { "id": "custom.width", "value": 184 } ] }, { "matcher": { "id": "byName", "options": "destinationServicePortName" }, "properties": [ { "id": "custom.width", "value": 215 } ] }, { "matcher": { "id": "byName", "options": "ingressNetworkPolicyName" }, "properties": [ { "id": "custom.width", "value": 213 } ] }, { "matcher": { "id": "byName", "options": "ingressNetworkPolicyNamespace" }, "properties": [ { "id": "custom.width", "value": 256 } ] }, { "matcher": { "id": "byName", "options": "ingressNetworkPolicyRuleName" }, "properties": [ { "id": "custom.width", "value": 243 } ] }, { "matcher": { "id": "byName", "options": "ingressNetworkPolicyRuleAction" }, "properties": [ { "id": "custom.width", "value": 249 }, { "id": "mappings", "value": [ { "options": { "0": { "index": 0, "text": "No Action" }, "1": { "index": 1, "text": "Allow" }, "2": { "index": 2, "text": "Drop" }, "3": { "index": 3, "text": "Reject" } }, "type": "value" } ] } ] }, { "matcher": { "id": "byName", "options": "ingressNetworkPolicyType" }, "properties": [ { "id": "custom.width", "value": 209 }, { "id": "mappings", "value": [ { "options": { "0": { "index": 3, "text": "N/A" }, "1": { "index": 0, "text": "K8s NetworkPolicy" }, "2": { "index": 1, "text": "Antrea NetworkPolicy" }, "3": { "index": 2, "text": "Antrea ClusterNetworkPolicy" } }, "type": "value" } ] } ] }, { "matcher": { "id": "byName", "options": "egressNetworkPolicyName" }, "properties": [ { "id": "custom.width", "value": 215 } ] }, { "matcher": { "id": "byName", "options": "egressNetworkPolicyRuleName" }, "properties": [ { "id": "custom.width", "value": 238 } ] }, { "matcher": { "id": "byName", "options": "egressNetworkPolicyNamespace" }, "properties": [ { "id": "custom.width", "value": 248 } ] }, { "matcher": { "id": "byName", "options": "egressNetworkPolicyType" }, "properties": [ { "id": "custom.width", "value": 207 }, { "id": "mappings", "value": [ { "options": { "0": { "index": 3, "text": "N/A" }, "1": { "index": 0, "text": "K8s NetworkPolicy" }, "2": { "index": 1, "text": "Antrea NetworkPolicy" }, "3": { "index": 2, "text": "Antrea ClusterNetworkPolicy" } }, "type": "value" } ] } ] }, { "matcher": { "id": "byName", "options": "destinationPodLabels" }, "properties": [ { "id": "custom.width", "value": 173 } ] }, { "matcher": { "id": "byName", "options": "throughputFromSourceNode" }, "properties": [ { "id": "custom.width", "value": 230 }, { "id": "displayName", "value": "throughput reported by source" } ] }, { "matcher": { "id": "byName", "options": "throughputFromDestinationNode" }, "properties": [ { "id": "custom.width", "value": 260 }, { "id": "displayName", "value": "throughput reported by destination" } ] }, { "matcher": { "id": "byName", "options": "reverseThroughputFromSourceNode" }, "properties": [ { "id": "custom.width", "value": 280 }, { "id": "displayName", "value": "reverse throughput reported by source" } ] }, { "matcher": { "id": "byName", "options": "reverseThroughputFromDestinationNode" }, "properties": [ { "id": "custom.width", "value": 310 }, { "id": "displayName", "value": "reverse throughput reported by destination" } ] }, { "matcher": { "id": "byName", "options": "reverseOctetDeltaCount" }, "properties": [ { "id": "custom.width", "value": 186 } ] }, { "matcher": { "id": "byName", "options": "reversePacketDeltaCount" }, "properties": [ { "id": "custom.width", "value": 204 } ] }, { "matcher": { "id": "byName", "options": "reverseOctetTotalCount" }, "properties": [ { "id": "custom.width", "value": 191 } ] }, { "matcher": { "id": "byName", "options": "reversePacketTotalCount" }, "properties": [ { "id": "custom.width", "value": 199 } ] }, { "matcher": { "id": "byName", "options": "flowEndSecondsFromDestinationNode" }, "properties": [ { "id": "custom.width", "value": 290 }, { "id": "displayName", "value": "end time of flow reported by destination" } ] }, { "matcher": { "id": "byName", "options": "flowEndSecondsFromSourceNode" }, "properties": [ { "id": "custom.width", "value": 260 }, { "id": "displayName", "value": "end time of flow reported by source" } ] }, { "matcher": { "id": "byName", "options": "protocolIdentifier" }, "properties": [ { "id": "mappings", "value": [ { "options": { "6": { "index": 0, "text": "TCP" }, "17": { "index": 1, "text": "UDP" } }, "type": "value" } ] } ] }, { "matcher": { "id": "byName", "options": "sourceTransportPort" }, "properties": [ { "id": "custom.width", "value": 170 } ] }, { "matcher": { "id": "byName", "options": "destinationTransportPort" }, "properties": [ { "id": "custom.width", "value": 196 } ] }, { "matcher": { "id": "byName", "options": "destinationClusterIP" }, "properties": [ { "id": "custom.width", "value": 168 } ] }, { "matcher": { "id": "byName", "options": "egressNetworkPolicyRuleAction" }, "properties": [ { "id": "custom.width", "value": 244 }, { "id": "mappings", "value": [ { "options": { "0": { "index": 0, "text": "No Action" }, "1": { "index": 1, "text": "Allow" }, "2": { "index": 2, "text": "Drop" }, "3": { "index": 3, "text": "Reject" } }, "type": "value" } ] } ] }, { "matcher": { "id": "byName", "options": "reverseThroughput" }, "properties": [ { "id": "custom.width", "value": 161 } ] }, { "matcher": { "id": "byName", "options": "flowEndSeconds" }, "properties": [ { "id": "custom.width", "value": 185 }, { "id": "displayName", "value": "end time of flow" } ] }, { "matcher": { "id": "byName", "options": "flowStartSeconds" }, "properties": [ { "id": "custom.width", "value": 185 }, { "id": "displayName", "value": "start time of flow" } ] }, { "matcher": { "id": "byName", "options": "timeInserted" }, "properties": [ { "id": "custom.width", "value": 187 }, { "id": "displayName", "value": "insert time of flow" } ] }, { "matcher": { "id": "byName", "options": "flowEndReason" }, "properties": [ { "id": "mappings", "value": [ { "options": { "1": { "index": 0, "text": "Idle Timeout" }, "2": { "index": 1, "text": "Active Timeout" }, "3": { "index": 2, "text": "End Of Flow" } }, "type": "value" } ] }, { "id": "custom.width", "value": 142 } ] }, { "matcher": { "id": "byName", "options": "octetDeltaCount" }, "properties": [ { "id": "displayName", "value": "delta bytes from the last record" }, { "id": "custom.width", "value": 235 } ] }, { "matcher": { "id": "byName", "options": "packetDeltaCount" }, "properties": [ { "id": "custom.width", "value": 250 }, { "id": "displayName", "value": "delta packets from the last record" } ] } ] }, "gridPos": { "h": 15, "w": 24, "x": 0, "y": 7 }, "id": 2, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT * \nFROM flows\nWHERE $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC\nLIMIT 10000", "refId": "A" } ], "title": "Flow Records Table", "transformations": [ { "id": "seriesToColumns", "options": {} } ], "transparent": true, "type": "table" } ], "refresh": "", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "filters": [], "hide": 0, "name": "Filter", "skipUrlSync": false, "type": "adhoc" }, { "hide": 2, "name": "clickhouse_adhoc_query", "query": "default.flows", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "flow_records_dashboard", "uid": "t1UGX7t7k", "version": 2, "weekStart": "" } homepage.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 5, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 32, "panels": [], "title": "Cluster Overview", "type": "row" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "displayName": "Number of Pods", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 12, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT COUNT(derivedtable.pod) as Number_of_Pods\nFROM\n(\n SELECT DISTINCT CONCAT(sourcePodName, sourcePodNamespace) AS pod FROM default.flows WHERE pod != '' AND $__timeFilter(flowEndSeconds)\n UNION ALL\n SELECT DISTINCT CONCAT(destinationPodName, destinationPodNamespace) AS pod FROM default.flows WHERE pod != '' AND $__timeFilter(flowEndSeconds)\n) derivedtable\nWHERE derivedtable.pod != ''", "refId": "A" } ], "transformations": [ { "id": "labelsToFields", "options": {} } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "displayName": "Number of Services", "mappings": [], "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "id": 14, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT COUNT(DISTINCT destinationServicePortName) as Number_of_Services \nFROM default.flows \nWHERE destinationServicePortName != '' AND $__timeFilter(flowEndSeconds)", "refId": "A" } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "displayName": "Number of Nodes", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "id": 13, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT COUNT(DISTINCT derivedtable.node) as Number_of_Nodes\nFROM\n(\n SELECT DISTINCT sourceNodeName AS node FROM default.flows WHERE node != '' AND $__timeFilter(flowEndSeconds)\n UNION ALL\n SELECT DISTINCT destinationNodeName AS node FROM default.flows WHERE node != '' AND $__timeFilter(flowEndSeconds)\n) derivedtable\nWHERE derivedtable.node IS NOT NULL", "refId": "A" } ], "type": "stat" }, { "gridPos": { "h": 5, "w": 12, "x": 12, "y": 1 }, "id": 6, "options": { "content": "
\n \n

Theia is a network observability and\nanalytics platform for Kubernetes, built on top of\nAntrea. It supports network flow\nvisualization and monitoring with Grafana. Follow this\ndocument\nto know more about the Grafana Flow Collector and network flow visualization\nfunctionality of Theia.

\n
\n\n", "mode": "html" }, "pluginVersion": "8.3.3", "title": "Overview of Project Theia", "type": "text" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "purple", "mode": "fixed" }, "displayName": "Number of Active Connections", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 5 }, "id": 15, "interval": "2s", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "value_and_name" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_Active_Connections\nfrom default.flows\nWHERE flowEndReason == 2 AND $__timeFilter(flowEndSeconds)", "refId": "A" } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "purple", "mode": "fixed" }, "displayName": "Number of Terminated Connections", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 5 }, "id": 16, "interval": "2s", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_Stopped_Connections\nfrom default.flows WHERE flowEndReason != 2 AND $__timeFilter(flowEndSeconds)", "refId": "A" } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "purple", "mode": "fixed" }, "displayName": "Number of Denied Connections", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 5 }, "id": 23, "interval": "2s", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_Denied_Connections\nfrom default.flows\nWHERE (ingressNetworkPolicyRuleAction in (2,3) OR egressNetworkPolicyRuleAction in (2,3))\nAND $__timeFilter(flowEndSeconds)", "refId": "A" } ], "type": "stat" }, { "gridPos": { "h": 11, "w": 12, "x": 12, "y": 6 }, "id": 27, "options": { "content": "The following seven dashboards are pre-built and are recommended for network flow\nvisualization.\n\nFlow Records Dashboard displays the flow records being captured in the selected\ntime range.\n\nPod-to-Pod Flows Dashboard shows cumulative bytes and throughput of Pod-to-Pod\ntraffic.\n\nPod-to-External Flows Dashboard shows cumulative bytes and throughput of\ntraffic from in-cluster to out-of-cluster. The destination of such a traffic flow\nis represented by the destination IP address.\n\nPod-to-Service Flows Dashboard shows cumulative bytes and throughput of\nPod-to-Service flows. The destination of such a traffic is represented by the\ndestination Service metadata.\n\nNode-to-Node Flows Dashboard visualizes the Node-to-Node traffic, including intra-Node\nand inter-Node flows.\n\nNetwork-Policy Flows Dashboard visualizes both the traffic with NetworkPolicies enforced,\nand unprotected traffic.\n\nNetwork Topology Dashboard visualizes both Pod-to-Pod and Pod-to-Service traffic via a graph.", "mode": "markdown" }, "pluginVersion": "8.3.3", "title": "Introduction of Pre-built Dashboards", "type": "text" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "displayName": "Data Transmitted", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 9 }, "id": 21, "interval": "2s", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount) as Data_Transmitted\nfrom default.flows_pod_view WHERE $__timeFilter(flowEndSeconds)", "refId": "A" } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "displayName": "Overall Throughput", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 9 }, "id": 22, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT (SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount))/60 as Overall_Throughput\nfrom default.flows_pod_view WHERE (now() - flowEndSeconds) < 60", "refId": "A" } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "displayName": "Number of NetworkPolicies", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 9 }, "id": 26, "interval": "2s", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "value_and_name" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT (COUNT(DISTINCT ingressNetworkPolicyName) + COUNT(DISTINCT egressNetworkPolicyName)) as Number_of_NetworkPolicies\nfrom default.flows_policy_view\nWHERE CONCAT(ingressNetworkPolicyName, egressNetworkPolicyName) != ''\nAND $__timeFilter(flowEndSeconds)", "refId": "A" } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "orange", "mode": "fixed" }, "displayName": "Data Transmitted with External", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 13 }, "id": 28, "interval": "2s", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount) as Data_Transmitted_With_External\nFROM default.flows_pod_view \nWHERE $__timeFilter(flowEndSeconds)\nAND flowType == 3", "refId": "A" } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "orange", "mode": "fixed" }, "displayName": "Overall Throughput with External", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 13 }, "id": 29, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT (SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount))/60 as Overall_Throughput_With_External\nfrom default.flows_pod_view WHERE (now() - flowEndSeconds) < 60\nAND flowType == 3", "refId": "A" } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "orange", "mode": "fixed" }, "displayName": "Number of ToExternal Connections", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 13 }, "id": 33, "interval": "2s", "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.3.3", "targets": [ { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "database": "default", "fields": [], "filters": [], "limit": 100, "metrics": [], "mode": "list", "orderBy": [], "table": "flows" } }, "queryType": "sql", "rawSql": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_ToExternal_Connections\nfrom default.flows\nWHERE flowType == 3\nAND $__timeFilter(flowEndSeconds)", "refId": "A" } ], "type": "stat" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 17 }, "id": 18, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": true }, "showUnfilled": true }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT CONCAT(sourcePodNamespace, '/', sourcePodName) as pod,\nSUM(octetDeltaCount) as bytes\nFROM default.flows_pod_view\nWHERE $__timeFilter(flowEndSeconds)\nAND pod != '/'\nGROUP BY pod\nORDER BY bytes DESC LIMIT 10", "refId": "A" } ], "title": "Top 10 Active Source Pods", "type": "bargauge" }, { "datasource": { "type": "grafana", "uid": "grafana" }, "gridPos": { "h": 18, "w": 12, "x": 12, "y": 17 }, "id": 4, "links": [], "options": { "folderId": 0, "maxItems": 10, "query": "", "showHeadings": true, "showRecentlyViewed": true, "showSearch": true, "showStarred": true, "tags": [] }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "datasource", "uid": "grafana" }, "queryType": "list", "refId": "A" } ], "title": "Dashboard Links", "type": "dashlist" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 80, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 26 }, "id": 25, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\ncount(*) as count\nFROM default.flows\nWHERE $__timeFilter(time)\nGROUP BY time\nORDER BY time", "refId": "A" } ], "title": "Number of Flow Records Per Minute", "type": "timeseries" } ], "refresh": "", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "homepage", "uid": "Yw6zwRkVk", "version": 6, "weekStart": "" } network_topology_dashboard.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 8, "iteration": 1682533463233, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 35, "w": 23, "x": 0, "y": 0 }, "id": 2, "options": { "color": "yellow", "groupByLabel": false, "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodName != ''\nAND sourcePodName != ''\nAND octetDeltaCount != 0\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC", "refId": "A" } ], "title": "Network Topology", "type": "theia-grafana-dependency-plugin" } ], "refresh": "", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "filters": [], "hide": 0, "name": "Filter", "skipUrlSync": false, "type": "adhoc" }, { "hide": 2, "name": "clickhouse_adhoc_query", "query": "default.flows", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "network_topology_dashboard", "uid": "yRVDEad4k", "version": 2, "weekStart": "" } networkpolicy_dashboard.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 2, "iteration": 1659133512870, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 20, "w": 15, "x": 0, "y": 0 }, "id": 2, "interval": "1s", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT CONCAT(sourcePodNamespace, '/', sourcePodName) as srcPod,\nCONCAT(destinationPodNamespace, '/', destinationPodName) as dstPod,\nsourceTransportPort as srcPort,\ndestinationTransportPort as dstPort,\ndestinationServicePort as dstSvcPort,\ndestinationServicePortName as dstSvc,\ndestinationIP as dstIP,\nSUM(octetDeltaCount) as bytes,\nSUM(reverseOctetDeltaCount) as revBytes,\negressNetworkPolicyName,\negressNetworkPolicyRuleAction,\ningressNetworkPolicyName,\ningressNetworkPolicyRuleAction\nfrom flows_policy_view\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\n AND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\n AND $__timeFilter(flowEndSeconds)\nGROUP BY srcPod, dstPod, srcPort, dstPort, dstSvcPort, dstSvc, dstIP, egressNetworkPolicyName, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyRuleAction\nHAVING bytes > 0\norder by bytes DESC\n", "refId": "A" } ], "title": "Cumulative Bytes of Flows with NetworkPolicy Information", "transparent": true, "type": "theia-grafana-chord-plugin" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 0 }, "id": 10, "options": { "legend": { "displayMode": "table", "placement": "bottom", "values": [ "percent", "value" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 25, "values": true }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(octetDeltaCount) as bytes, \nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace, '/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np\nFROM flows_policy_view\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY np\nHAVING SUM(octetDeltaCount) != 0\nORDER BY bytes DESC", "refId": "A" } ], "title": "Cumulative Bytes of Ingress Network Policy", "transparent": true, "type": "piechart" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 8, "x": 16, "y": 10 }, "id": 9, "options": { "displayLabels": [], "legend": { "displayMode": "table", "placement": "bottom", "values": [ "percent", "value" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 25, "values": true }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(octetDeltaCount) as bytes,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace, '/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND AS np\nFROM flows_policy_view\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY np\nHAVING SUM(octetDeltaCount) > 0\nORDER BY bytes DESC", "refId": "A" } ], "title": "Cumulative Bytes of Egress Network Policy", "transparent": true, "type": "piechart" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, "id": 4, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND AS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction == 1\nAND egressNetworkPolicyRuleAction NOT IN (2, 3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time\n", "refId": "A" } ], "title": "Throughput of Ingress Allow NetworkPolicy", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, "id": 5, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction == 1\nAND ingressNetworkPolicyRuleAction not in (2, 3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Egress Allow NetworkPolicy", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, "id": 6, "interval": "1s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "expand": false, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction in (2,3)\nGROUP BY time, src, dst, np\nHAVING SUM(octetDeltaCount) > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Ingress Deny NetworkPolicy", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, "id": 7, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction in (2,3)\nGROUP BY time, src, dst, np\nHAVING SUM(octetDeltaCount) > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Egress Deny NetworkPolicy", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" } ], "refresh": "", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "filters": [], "hide": 0, "name": "Filter", "skipUrlSync": false, "type": "adhoc" }, { "hide": 2, "name": "clickhouse_adhoc_query", "query": "default.flows_policy_view", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "networkpolicy_dashboard", "uid": "KJNMOwQnk", "version": 2, "weekStart": "" } node_to_node_dashboard.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 3, "iteration": 1661796451586, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 18, "w": 12, "x": 0, "y": 0 }, "id": 11, "interval": "60", "options": { "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" }, "pluginVersion": "7.5.2", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(octetDeltaCount) as bytes, sourceNodeName as source, destinationNodeName as destination\nFrom flows_node_view\nWHERE source != '' AND destination != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nORDER BY bytes DESC\nLIMIT 50", "refId": "A" } ], "title": "Cumulative Bytes of Node-to-Node", "transformations": [ { "id": "labelsToFields", "options": {} } ], "transparent": true, "type": "theia-grafana-sankey-plugin" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 18, "w": 12, "x": 12, "y": 0 }, "id": 14, "interval": "60s", "options": { "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" }, "pluginVersion": "7.5.2", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(reverseOctetDeltaCount) as bytes, sourceNodeName as source, destinationNodeName as destination\nFrom flows_node_view\nWHERE source != '' AND destination != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nORDER BY bytes DESC\nLIMIT 50", "refId": "A" } ], "title": "Cumulative Reverse Bytes of Node-to-Node", "transparent": true, "type": "theia-grafana-sankey-plugin" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 18 }, "id": 18, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourceNodeName, '->', destinationNodeName) as pair, SUM(octetDeltaCount)*8000/$__interval_ms as Node\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nORDER BY time", "refId": "A" } ], "title": "Throughput of Node-to-Node", "transformations": [ { "id": "groupBy", "options": { "fields": { "Time": { "aggregations": [], "operation": "aggregate" } } } }, { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 18 }, "id": 24, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourceNodeName, '->', destinationNodeName) as pair, SUM(reverseOctetDeltaCount)*8000/$__interval_ms as Node\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nORDER BY time", "refId": "A" } ], "title": "Reverse Throughput of Node-to-Node", "transformations": [ { "id": "groupBy", "options": { "fields": { "Time": { "aggregations": [], "operation": "aggregate" } } } }, { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 28 }, "id": 25, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, sourceNodeName, SUM(octetDeltaCount)*8000/$__interval_ms\nFROM flows_node_view\nWHERE sourceNodeName != '' \nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, sourceNodeName\nORDER BY time", "refId": "A" } ], "title": "Throughput of Node as Source", "transformations": [ { "id": "groupBy", "options": { "fields": { "Time": { "aggregations": [], "operation": "aggregate" } } } }, { "id": "labelsToFields", "options": { "valueLabel": "sourceNodeName" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 28 }, "id": 21, "interval": "60s", "options": { "displayLabels": [], "legend": { "displayMode": "table", "placement": "right", "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 25, "values": true }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "rawSql": "SELECT SUM(octetDeltaCount) as bytes, sourceNodeName\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY sourceNodeName\nORDER BY bytes DESC", "refId": "A" } ], "title": "Cumulative Bytes of Node as Source", "transparent": true, "type": "piechart" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 38 }, "id": 26, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, destinationNodeName, SUM(octetDeltaCount)*8000/$__interval_ms\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, destinationNodeName\nORDER BY time", "refId": "A" } ], "title": "Throughput of Node as Destination", "transformations": [ { "id": "groupBy", "options": { "fields": { "Time": { "aggregations": [], "operation": "aggregate" } } } }, { "id": "labelsToFields", "options": { "valueLabel": "destinationNodeName" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 38 }, "id": 22, "options": { "legend": { "displayMode": "table", "placement": "right", "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 25, "values": true }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "rawSql": "SELECT SUM(octetDeltaCount) as bytes, destinationNodeName\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY destinationNodeName\nORDER BY bytes DESC", "refId": "A" } ], "title": "Cumulative Bytes of Node as Destination", "transparent": true, "type": "piechart" } ], "refresh": "", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "filters": [], "hide": 0, "name": "Filter", "skipUrlSync": false, "type": "adhoc" }, { "hide": 2, "name": "clickhouse_adhoc_query", "query": "default.flows_node_view", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "node_to_node_dashboard", "uid": "1F56RJh7z", "version": 2, "weekStart": "" } pod_to_external_dashboard.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 4, "iteration": 1659133592592, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 18, "w": 12, "x": 0, "y": 0 }, "id": 13, "options": { "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" }, "pluginVersion": "7.5.2", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(octetDeltaCount) as bytes, \nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR)) as source, \ndestinationIP as destination\nFrom flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nHAVING bytes > 0\nORDER BY bytes DESC\nLIMIT 50", "refId": "A" } ], "title": "Cumulative Bytes of Pod-to-External", "transparent": true, "type": "theia-grafana-sankey-plugin" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 18, "w": 12, "x": 12, "y": 0 }, "id": 12, "options": { "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" }, "pluginVersion": "7.5.2", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(reverseOctetDeltaCount) as bytes,\nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR)) as source, \ndestinationIP as destination\nFrom flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nHAVING bytes > 0\nORDER BY bytes DESC\nLIMIT 50", "refId": "A" } ], "title": "Cumulative Reverse Bytes of Pod-to-External", "transparent": true, "type": "theia-grafana-sankey-plugin" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 18 }, "id": 2, "interval": "1s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, \nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR), '->', destinationIP) as pair,\nAVG(throughput)\nFROM flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING AVG(throughput) > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Pod-to-External", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 18 }, "id": 7, "interval": "1s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR), '->', destinationIP) as pair,\nAVG(reverseThroughput)\nFROM flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING AVG(reverseThroughput) > 0\nORDER BY time", "refId": "A" } ], "title": "Reverse Throughput of Pod-to-External", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" } ], "refresh": "", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "filters": [], "hide": 0, "name": "Filter", "skipUrlSync": false, "type": "adhoc" }, { "hide": 2, "name": "clickhouse_adhoc_query", "query": "default.flows_pod_view", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "pod_to_external_dashboard", "uid": "K9SPrnJ7k", "version": 2, "weekStart": "" } pod_to_pod_dashboard.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 5, "iteration": 1661806433864, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 18, "w": 12, "x": 0, "y": 0 }, "id": 13, "options": { "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" }, "pluginVersion": "7.5.2", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(octetDeltaCount) as bytes, \nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR)) as source, \nCONCAT(destinationPodNamespace, '/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR)) as destination\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nORDER BY bytes DESC\nLIMIT 50", "refId": "A" } ], "title": "Cumulative Bytes of Pod-to-Pod", "transparent": true, "type": "theia-grafana-sankey-plugin" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 18, "w": 12, "x": 12, "y": 0 }, "id": 12, "options": { "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" }, "pluginVersion": "7.5.2", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(reverseOctetDeltaCount) as bytes, \nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR)) as source, \nCONCAT(destinationPodNamespace, '/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR)) as destination\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nORDER BY bytes DESC\nLIMIT 50", "refId": "A" } ], "title": "Cumulative Reverse Bytes of Pod-to-Pod", "transparent": true, "type": "theia-grafana-sankey-plugin" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 18 }, "id": 21, "interval": "1s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, \nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR), ' -> ', destinationPodNamespace, '/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR)) as pair, \nAVG(throughput)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING AVG(throughput) > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Pod-to-Pod", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 18 }, "id": 22, "interval": "1s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, \nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR), ' -> ', destinationPodNamespace, '/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR)) as pair, \nAVG(reverseThroughput)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING AVG(reverseThroughput) > 0\nORDER BY time", "refId": "A" } ], "title": "Reverse Throughput of Pod-to-Pod", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 28 }, "id": 15, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR)) as src,\nSUM(octetDeltaCount)*8000/$__interval_ms as throughput\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, src\nHAVING throughput > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Pod as Source", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "src" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 28 }, "id": 18, "options": { "legend": { "displayMode": "table", "placement": "right", "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 25, "values": true }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(octetDeltaCount) as bytes, sourcePodNamespace\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY sourcePodNamespace\nHAVING bytes > 0\nORDER BY bytes DESC", "refId": "A" } ], "title": "Cumulative Bytes of Source Pod Namespace", "transparent": true, "type": "piechart" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 38 }, "id": 16, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCONCAT(destinationPodNamespace, '/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR)) as dst,\nSUM(octetDeltaCount)*8000/$__interval_ms as throughput\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, dst\nHAVING throughput > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Pod as Destination", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "dst" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "decbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "destinationPodNamespace" }, "properties": [ { "id": "noValue", "value": "N/A" } ] } ] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 38 }, "id": 19, "options": { "legend": { "displayMode": "table", "placement": "right", "values": [ "percent", "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 25, "values": true }, "tooltip": { "mode": "single" } }, "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "rawSql": "SELECT SUM(octetDeltaCount) as bytes, destinationPodNamespace\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY destinationPodNamespace\nORDER BY bytes DESC", "refId": "A" } ], "title": "Cumulative Bytes of Destination Pod Namespace", "transparent": true, "type": "piechart" } ], "refresh": "", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "filters": [], "hide": 0, "name": "Filter", "skipUrlSync": false, "type": "adhoc" }, { "hide": 2, "name": "clickhouse_adhoc_query", "query": "default.flows_pod_view", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "pod_to_pod_dashboard", "uid": "Yxn0Ghh7k", "version": 3, "weekStart": "" } pod_to_service_dashboard.json: | { "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 6, "iteration": 1661806778467, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 18, "w": 12, "x": 0, "y": 0 }, "id": 13, "options": { "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" }, "pluginVersion": "7.5. 2", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(octetDeltaCount) as bytes, \nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR)) as source, \nCONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR)) as destination\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nHAVING bytes > 0\nORDER BY bytes DESC\nLIMIT 50", "refId": "A" } ], "title": "Cumulative Bytes Pod-to-Service", "transparent": true, "type": "theia-grafana-sankey-plugin" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "gridPos": { "h": 18, "w": 12, "x": 12, "y": 0 }, "id": 12, "options": { "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" }, "pluginVersion": "7.5. 2", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 1, "queryType": "sql", "rawSql": "SELECT SUM(reverseOctetDeltaCount) as bytes,\nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR)) as source, \nCONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR)) as destination\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nHAVING bytes > 0\nORDER BY bytes DESC\nLIMIT 50", "refId": "A" } ], "title": "Cumulative Reverse Bytes Pod-to-Service", "transparent": true, "type": "theia-grafana-sankey-plugin" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 18 }, "id": 18, "interval": "1s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, \nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR), ' -> ', destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR)) as pair, \nAVG(throughput)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY time, pair\nHAVING AVG(throughput) > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Pod-to-Service", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 18 }, "id": 19, "interval": "1s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, \nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR), ' -> ', destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR)) as pair, \nAVG(reverseThroughput)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING AVG(reverseThroughput) > 0\nORDER BY time", "refId": "A" } ], "title": "Reverse Throughput of Pod-to-Service", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "pair" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 28 }, "id": 20, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR)) as src,\nSUM(octetDeltaCount)*8000/$__interval_ms as throughput\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(time)\nGROUP BY time, src\nHAVING throughput > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Pod as Source", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "src" } } ], "transparent": true, "type": "timeseries" }, { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": 90000, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 28 }, "id": 21, "interval": "60s", "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.3.3", "targets": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "format": 2, "meta": { "builderOptions": { "fields": [], "limit": 100, "mode": "list" } }, "queryType": "sql", "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR)) as dst,\nSUM(octetDeltaCount)*8000/$__interval_ms as throughput\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(time)\nGROUP BY time, dst\nHAVING throughput > 0\nORDER BY time", "refId": "A" } ], "title": "Throughput of Service as Destination", "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "dst" } } ], "transparent": true, "type": "timeseries" } ], "refresh": "", "schemaVersion": 34, "style": "dark", "tags": [], "templating": { "list": [ { "datasource": { "type": "grafana-clickhouse-datasource", "uid": "PDEE91DDB90597936" }, "filters": [], "hide": 0, "name": "Filter", "skipUrlSync": false, "type": "adhoc" }, { "hide": 2, "name": "clickhouse_adhoc_query", "query": "default.flows_pod_view", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "pod_to_service_dashboard", "uid": "LGdxbW17z", "version": 3, "weekStart": "" } kind: ConfigMap metadata: name: grafana-dashboard-config namespace: flow-visibility --- apiVersion: v1 data: dashboard_provider.yaml: |- apiVersion: 1 providers: - name: grafana-dashboards folder: '' type: file allowUiUpdates: true options: path: /opt/grafana/dashboards kind: ConfigMap metadata: name: grafana-dashboard-provider namespace: flow-visibility --- apiVersion: v1 data: datasource_provider.yaml: |- apiVersion: 1 datasources: - name: ClickHouse type: grafana-clickhouse-datasource access: proxy url: http://clickhouse-clickhouse.flow-visibility.svc:8123 editable: true jsonData: server: clickhouse-clickhouse.flow-visibility.svc port: 9000 username: $CLICKHOUSE_USERNAME secureJsonData: password: $CLICKHOUSE_PASSWORD kind: ConfigMap metadata: name: grafana-datasource-provider namespace: flow-visibility --- apiVersion: v1 data: theia-manager.conf: | # apiServer contains APIServer related configuration options. apiServer: # The port for the theia-manager APIServer to serve on. apiPort: 11347 # Indicates whether to use auto-generated self-signed TLS certificate. # If false, a Secret named "theia-manager-tls" must be provided with the following keys: # ca.crt: # tls.crt: # tls.key: selfSignedCert: true # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. # https://golang.org/pkg/crypto/tls/#pkg-constants # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always # prefer TLS1.3 Cipher Suites whenever possible. tlsCipherSuites: "" # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. tlsMinVersion: "" kind: ConfigMap metadata: labels: app: theia-manager name: theia-manager-configmap namespace: flow-visibility --- apiVersion: v1 kind: Secret metadata: name: clickhouse-secret namespace: flow-visibility stringData: password: clickhouse_operator_password readOnlyPassword: readonly_password readOnlyUsername: readonly username: clickhouse_operator type: Opaque --- apiVersion: v1 kind: Secret metadata: name: grafana-secret namespace: flow-visibility stringData: admin-password: admin admin-username: admin type: Opaque --- apiVersion: v1 kind: Secret metadata: annotations: kubernetes.io/service-account.name: theia-cli name: theia-cli-account-token namespace: flow-visibility type: kubernetes.io/service-account-token --- apiVersion: v1 kind: Service metadata: name: grafana namespace: flow-visibility spec: ports: - port: 3000 protocol: TCP targetPort: http-grafana selector: app: grafana sessionAffinity: None type: NodePort --- apiVersion: v1 kind: Service metadata: labels: app: theia-manager name: theia-manager namespace: flow-visibility spec: ports: - port: 11347 protocol: TCP targetPort: theia-api-http selector: app: theia-manager --- apiVersion: v1 kind: Service metadata: labels: app: zookeeper name: zookeeper namespace: flow-visibility spec: ports: - name: client port: 2181 - name: prometheus port: 7000 selector: app: zookeeper what: node --- apiVersion: v1 kind: Service metadata: labels: app: zookeeper name: zookeepers namespace: flow-visibility spec: clusterIP: None ports: - name: server port: 2888 - name: leader-election port: 3888 selector: app: zookeeper what: node --- apiVersion: v1 kind: PersistentVolume metadata: name: grafana-pv spec: accessModes: - ReadWriteOnce capacity: storage: 1Gi hostPath: path: /data/grafana storageClassName: grafana-storage --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: grafana-pvc namespace: flow-visibility spec: accessModes: - ReadWriteOnce resources: requests: storage: 1Gi storageClassName: grafana-storage --- apiVersion: apps/v1 kind: Deployment metadata: labels: app: grafana name: grafana namespace: flow-visibility spec: selector: matchLabels: app: grafana template: metadata: labels: app: grafana spec: containers: - env: - name: GF_INSTALL_PLUGINS value: https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-sankey-plugin-1.0.3.zip;theia-grafana-sankey-plugin,https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-chord-plugin-1.0.2.zip;theia-grafana-chord-plugin,https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-dependency-plugin-1.0.2.zip;theia-grafana-dependency-plugin,grafana-clickhouse-datasource 2.2.0 - name: CLICKHOUSE_USERNAME valueFrom: secretKeyRef: key: username name: clickhouse-secret - name: CLICKHOUSE_PASSWORD valueFrom: secretKeyRef: key: password name: clickhouse-secret - name: GF_AUTH_BASIC_ENABLED value: "true" - name: GF_AUTH_ANONYMOUS_ENABLED value: "false" - name: GF_SECURITY_ADMIN_USER valueFrom: secretKeyRef: key: admin-username name: grafana-secret - name: GF_SECURITY_ADMIN_PASSWORD valueFrom: secretKeyRef: key: admin-password name: grafana-secret - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH value: /opt/grafana/dashboards/homepage.json - name: GF_LOG_MODE value: console file - name: GF_LOG_LEVEL value: info - name: GF_LOG_FILE_LOG_ROTATE value: "true" - name: GF_LOG_FILE_MAX_LINES value: "1000000" - name: GF_LOG_FILE_MAX_SIZE_SHIFT value: "27" - name: GF_LOG_FILE_DAILY_ROTATE value: "true" - name: GF_LOG_FILE_MAX_DAYS value: "7" image: projects.registry.vmware.com/antrea/theia-grafana:9.1.6 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 10 successThreshold: 1 tcpSocket: port: 3000 timeoutSeconds: 1 name: grafana ports: - containerPort: 3000 name: http-grafana protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: /robots.txt port: 3000 scheme: HTTP initialDelaySeconds: 10 periodSeconds: 30 successThreshold: 1 timeoutSeconds: 2 resources: requests: cpu: 250m memory: 750Mi volumeMounts: - mountPath: /var/lib/grafana name: grafana-pv - mountPath: /etc/grafana/provisioning/datasources name: grafana-datasource-provider - mountPath: /etc/grafana/provisioning/dashboards name: grafana-dashboard-provider - mountPath: /opt/grafana/dashboards name: grafana-dashboard-config initContainers: - command: - chown - -R - 472:472 - /data image: projects.registry.vmware.com/antrea/busybox imagePullPolicy: IfNotPresent name: init-pv volumeMounts: - mountPath: /data name: grafana-pv securityContext: fsGroup: 472 supplementalGroups: - 0 serviceAccountName: grafana volumes: - name: grafana-pv persistentVolumeClaim: claimName: grafana-pvc - configMap: name: grafana-datasource-provider name: grafana-datasource-provider - configMap: name: grafana-dashboard-provider name: grafana-dashboard-provider - configMap: name: grafana-dashboard-config name: grafana-dashboard-config --- apiVersion: apps/v1 kind: Deployment metadata: labels: app: theia-manager name: theia-manager namespace: flow-visibility spec: replicas: 1 selector: matchLabels: app: theia-manager template: metadata: labels: app: theia-manager spec: containers: - args: - --config - /etc/theia-manager/theia-manager.conf - --logtostderr=false - --log_dir=/var/log/antrea/theia-manager - --alsologtostderr - --log_file_max_size=100 - --log_file_max_num=4 env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: CLICKHOUSE_USERNAME valueFrom: secretKeyRef: key: username name: clickhouse-secret - name: CLICKHOUSE_PASSWORD valueFrom: secretKeyRef: key: password name: clickhouse-secret - name: CLICKHOUSE_URL value: tcp://clickhouse-clickhouse.flow-visibility.svc:9000 - name: GOCOVERDIR value: /theia-manager-coverage image: projects.registry.vmware.com/antrea/theia-manager:latest imagePullPolicy: IfNotPresent name: theia-manager ports: - containerPort: 11347 name: theia-api-http volumeMounts: - mountPath: /etc/theia-manager name: theia-manager-config readOnly: true - mountPath: /var/run/theia/theia-manager-tls name: theia-manager-tls - mountPath: /var/log/antrea/theia-manager name: host-var-log-antrea-theia-manager - mountPath: /theia-manager-coverage name: theia-manager-coverage nodeSelector: kubernetes.io/arch: amd64 kubernetes.io/os: linux serviceAccountName: theia-manager volumes: - configMap: name: theia-manager-configmap name: theia-manager-config - name: theia-manager-tls secret: defaultMode: 256 optional: true secretName: theia-manager-tls - hostPath: path: /var/log/antrea/theia-manager type: DirectoryOrCreate name: host-var-log-antrea-theia-manager - hostPath: path: /var/log/tm-coverage type: DirectoryOrCreate name: theia-manager-coverage --- apiVersion: apps/v1 kind: StatefulSet metadata: labels: app: zookeeper name: zookeeper namespace: flow-visibility spec: podManagementPolicy: Parallel replicas: 1 selector: matchLabels: app: zookeeper serviceName: zookeepers template: metadata: annotations: prometheus.io/port: "7000" prometheus.io/scrape: "true" labels: app: zookeeper what: node spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: app operator: In values: - zookeeper topologyKey: kubernetes.io/hostname containers: - command: - bash - -x - -c - | SERVERS=1 && HOST=`hostname -s` && DOMAIN=`hostname -d` && CLIENT_PORT=2181 && SERVER_PORT=2888 && ELECTION_PORT=3888 && PROMETHEUS_PORT=7000 && ZOO_DATA_DIR=/var/lib/zookeeper/data && ZOO_DATA_LOG_DIR=/var/lib/zookeeper/datalog && { echo "clientPort=${CLIENT_PORT}" echo 'tickTime=2000' echo 'initLimit=300' echo 'syncLimit=10' echo 'maxClientCnxns=2000' echo 'maxSessionTimeout=60000000' echo "dataDir=${ZOO_DATA_DIR}" echo "dataLogDir=${ZOO_DATA_LOG_DIR}" echo 'autopurge.snapRetainCount=10' echo 'autopurge.purgeInterval=1' echo 'preAllocSize=131072' echo 'snapCount=3000000' echo 'leaderServes=yes' echo 'standaloneEnabled=false' echo '4lw.commands.whitelist=*' echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider' echo "metricsProvider.httpPort=${PROMETHEUS_PORT}" } > /conf/zoo.cfg && { echo "zookeeper.root.logger=CONSOLE" echo "zookeeper.console.threshold=INFO" echo "log4j.rootLogger=\${zookeeper.root.logger}" echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" } > /conf/log4j.properties && echo 'JVMFLAGS="-Xms128M -Xmx4G -XX:+UseG1GC -XX:+CMSParallelRemarkEnabled -XX:ActiveProcessorCount=8"' > /conf/java.env && if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then NAME=${BASH_REMATCH[1]} ORD=${BASH_REMATCH[2]} else echo "Failed to parse name and ordinal of Pod" exit 1 fi && mkdir -p ${ZOO_DATA_DIR} && mkdir -p ${ZOO_DATA_LOG_DIR} && export MY_ID=$((ORD+1)) && echo $MY_ID > $ZOO_DATA_DIR/myid && for (( i=1; i<=$SERVERS; i++ )); do echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg; done && if [[ $SERVERS -eq 1 ]]; then echo "group.1=1" >> /conf/zoo.cfg; else groupstr="group.1=1" for (( i=2; i<=$SERVERS; i++ )); do groupstr+=":$i" done echo $groupstr >> /conf/zoo.cfg; fi && for (( i=1; i<=$SERVERS; i++ )); do WEIGHT=1 if [[ $i == 1 ]]; then WEIGHT=10 fi echo "weight.$i=$WEIGHT" >> /conf/zoo.cfg; done && chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" "$ZOO_LOG_DIR" "$ZOO_CONF_DIR" && zkServer.sh start-foreground image: projects.registry.vmware.com/antrea/theia-zookeeper:3.8.0 imagePullPolicy: IfNotPresent livenessProbe: exec: command: - bash - -c - OK=$(echo ruok | nc 127.0.0.1 2181); if [[ "$OK" == "imok" ]]; then exit 0; else exit 1; fi initialDelaySeconds: 10 timeoutSeconds: 5 name: kubernetes-zookeeper ports: - containerPort: 2181 name: client - containerPort: 2888 name: server - containerPort: 3888 name: leader-election - containerPort: 7000 name: prometheus readinessProbe: exec: command: - bash - -c - OK=$(echo ruok | nc 127.0.0.1 2181); if [[ "$OK" == "imok" ]]; then STATE=$(echo mntr | nc 127.0.0.1 2181 | grep zk_server_state | cut -d " " -f 2); if [[ "$STATE" == "leader" ]]; then SYNCED_FOLLOWERS=$(echo mntr | nc 127.0.0.1 2181 | grep zk_synced_followers | cut -d " " -f 2 | cut -d "." -f 1); if [[ $SYNCED_FOLLOWERS == $(( $SERVERS - 1 )) ]]; then ./bin/zkCli.sh ls /; exit $?; else exit 1; fi; elif [[ "$STATE" == "follower" ]]; then PEER_STATE=$(echo mntr | nc 127.0.0.1 2181 | grep zk_peer_state); if [[ "$PEER_STATE" == "following - broadcast" ]]; then ./bin/zkCli.sh ls /; exit $?; else exit 1; fi; fi; else exit 1; fi initialDelaySeconds: 10 timeoutSeconds: 5 resources: limits: cpu: "2" memory: 4Gi requests: cpu: "1" memory: 512M volumeMounts: - mountPath: /var/lib/zookeeper name: datadir-volume securityContext: fsGroup: 1000 runAsUser: 1000 volumes: - emptyDir: medium: "" sizeLimit: 5Gi name: datadir-volume updateStrategy: type: RollingUpdate --- apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: zookeeper-pod-disruption-budget namespace: flow-visibility spec: maxUnavailable: 1 selector: matchLabels: app: zookeeper --- apiVersion: clickhouse.altinity.com/v1 kind: ClickHouseInstallation metadata: labels: app: clickhouse name: clickhouse namespace: flow-visibility spec: configuration: clusters: - layout: replicasCount: 1 shardsCount: 1 name: clickhouse profiles: readonly/readonly: 1 settings: logger/count: 4 logger/level: information logger/size: 100M users: clickhouse_operator/k8s_secret_password: flow-visibility/clickhouse-secret/password clickhouse_operator/networks/ip: ::/0 readonly/k8s_secret_password: flow-visibility/clickhouse-secret/readOnlyPassword readonly/networks/ip: ::/0 readonly/profile: readonly zookeeper: nodes: - host: zookeeper.flow-visibility defaults: templates: podTemplate: pod-template serviceTemplate: service-template templates: podTemplates: - name: pod-template podDistribution: - number: 1 topologyKey: kubernetes.io/hostname type: MaxNumberPerNode spec: containers: - command: - /bin/sh - -c - chown -R clickhouse:clickhouse /var/lib/clickhouse && /entrypoint.sh env: - name: THEIA_VERSION value: 0.9.0 - name: CLICKHOUSE_INIT_TIMEOUT value: "60" - name: DB_URL value: localhost:9000 - name: MIGRATE_USERNAME valueFrom: secretKeyRef: key: username name: clickhouse-secret - name: MIGRATE_PASSWORD valueFrom: secretKeyRef: key: password name: clickhouse-secret image: projects.registry.vmware.com/antrea/theia-clickhouse-server:latest imagePullPolicy: IfNotPresent name: clickhouse volumeMounts: - mountPath: /docker-entrypoint-initdb.d name: clickhouse-configmap-volume - mountPath: /var/lib/clickhouse name: clickhouse-storage-volume - env: - name: CLICKHOUSE_USERNAME valueFrom: secretKeyRef: key: username name: clickhouse-secret - name: CLICKHOUSE_PASSWORD valueFrom: secretKeyRef: key: password name: clickhouse-secret - name: DB_URL value: tcp://localhost:9000 - name: TABLE_NAME value: default.flows_local - name: MV_NAMES value: default.pod_view_table_local default.node_view_table_local default.policy_view_table_local - name: STORAGE_SIZE value: 8Gi - name: THRESHOLD value: "0.5" - name: DELETE_PERCENTAGE value: "0.5" - name: EXEC_INTERVAL value: 1m - name: SKIP_ROUNDS_NUM value: "3" - name: GOCOVERDIR value: /clickhouse-monitor-coverage image: projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest imagePullPolicy: IfNotPresent name: clickhouse-monitor volumeMounts: - mountPath: /clickhouse-monitor-coverage name: clickhouse-monitor-coverage volumes: - configMap: items: - key: create_table.sh path: create_table.sh - key: init.sh path: init.sh - key: 000001_0-1-0.down.sql path: migrators/000001_0-1-0.down.sql - key: 000001_0-1-0.up.sql path: migrators/000001_0-1-0.up.sql - key: 000002_0-2-0.down.sql path: migrators/000002_0-2-0.down.sql - key: 000002_0-2-0.up.sql path: migrators/000002_0-2-0.up.sql - key: 000003_0-3-0.down.sql path: migrators/000003_0-3-0.down.sql - key: 000003_0-3-0.up.sql path: migrators/000003_0-3-0.up.sql - key: 000004_0-4-0.down.sql path: migrators/000004_0-4-0.down.sql - key: 000004_0-4-0.up.sql path: migrators/000004_0-4-0.up.sql - key: 000005_0-6-0.down.sql path: migrators/000005_0-6-0.down.sql - key: 000005_0-6-0.up.sql path: migrators/000005_0-6-0.up.sql name: clickhouse-mounted-configmap name: clickhouse-configmap-volume - emptyDir: medium: Memory sizeLimit: 8Gi name: clickhouse-storage-volume - hostPath: path: /var/log/cm-coverage type: DirectoryOrCreate name: clickhouse-monitor-coverage serviceTemplates: - name: service-template spec: ports: - name: http port: 8123 targetPort: 8123 - name: tcp port: 9000 targetPort: 9000 type: ClusterIP