# create amazon-cloudwatch namespace apiVersion: v1 kind: Namespace metadata: name: amazon-cloudwatch labels: name: amazon-cloudwatch --- # create configmap for prometheus cwagent config apiVersion: v1 data: # cwagent json config cwagentconfig.json: | { "logs": { "metrics_collected": { "prometheus": { "prometheus_config_path": "/etc/prometheusconfig/prometheus.yaml", "emf_processor": { "metric_declaration": [ { "source_labels": ["Service"], "label_matcher": ".*nginx.*", "dimensions": [["Service","Namespace","ClusterName"]], "metric_selectors": [ "^nginx_ingress_controller_(requests|success)$", "^nginx_ingress_controller_nginx_process_connections$", "^nginx_ingress_controller_nginx_process_connections_total$", "^nginx_ingress_controller_nginx_process_resident_memory_bytes$", "^nginx_ingress_controller_nginx_process_cpu_seconds_total$", "^nginx_ingress_controller_config_last_reload_successful$" ] }, { "source_labels": ["Service"], "label_matcher": ".*nginx.*", "dimensions": [["Service","Namespace","ClusterName","ingress"],["Service","Namespace","ClusterName","status"]], "metric_selectors": ["^nginx_ingress_controller_requests$"] }, { "source_labels": ["Service", "frontend"], "label_matcher": ".*haproxy-ingress-.*metrics;(httpfront-shared-frontend|httpfront-default-backend|httpsfront|_front_http)", "dimensions": [["Service","Namespace","ClusterName","frontend","code"]], "metric_selectors": [ "^haproxy_frontend_http_responses_total$" ] }, { "source_labels": ["Service", "backend"], "label_matcher": ".*haproxy-ingress-.*metrics", "dimensions": [["Service","Namespace","ClusterName","backend","code"]], "metric_selectors": [ "^haproxy_backend_http_responses_total$" ] }, { "source_labels": ["Service"], "label_matcher": ".*haproxy-ingress-.*metrics", "dimensions": [["Service","Namespace","ClusterName"]], "metric_selectors": [ "^haproxy_backend_up$", "^haproxy_backend_status$", "^haproxy_backend_bytes_(in|out)_total$", "^haproxy_backend_connections_total$", "^haproxy_backend_connection_errors_total$", "^haproxy_backend_current_sessions$", "^haproxy_frontend_bytes_(in|out)_total$", "^haproxy_frontend_connections_total$", "^haproxy_frontend_http_requests_total$", "^haproxy_frontend_request_errors_total$", "^haproxy_frontend_requests_denied_total$", "^haproxy_frontend_current_sessions$" ] }, { "source_labels": ["Service"], "label_matcher": ".*memcached.*", "dimensions": [["Service","Namespace","ClusterName"]], "metric_selectors": [ "^memcached_current_(bytes|items|connections)$", "^memcached_items_(reclaimed|evicted)_total$", "^memcached_(written|read)_bytes_total$", "^memcached_limit_bytes$", "^memcached_commands_total$" ] }, { "source_labels": ["Service", "status", "command"], "label_matcher": ".*memcached.*;hit;get", "dimensions": [["Service","Namespace","ClusterName","status","command"]], "metric_selectors": [ "^memcached_commands_total$" ] }, { "source_labels": ["Service", "command"], "label_matcher": ".*memcached.*;(get|set)", "dimensions": [["Service","Namespace","ClusterName","command"]], "metric_selectors": [ "^memcached_commands_total$" ] }, { "source_labels": ["container_name"], "label_matcher": "^envoy$", "dimensions": [["ClusterName","Namespace"]], "metric_selectors": [ "^envoy_http_downstream_rq_(total|xx)$", "^envoy_cluster_upstream_cx_(r|t)x_bytes_total$", "^envoy_cluster_membership_(healthy|total)$", "^envoy_server_memory_(allocated|heap_size)$", "^envoy_cluster_upstream_cx_(connect_timeout|destroy_local_with_active_rq)$", "^envoy_cluster_upstream_rq_(pending_failure_eject|pending_overflow|timeout|per_try_timeout|rx_reset|maintenance_mode)$", "^envoy_http_downstream_cx_destroy_remote_active_rq$", "^envoy_cluster_upstream_flow_control_(paused_reading_total|resumed_reading_total|backed_up_total|drained_total)$", "^envoy_cluster_upstream_rq_retry$", "^envoy_cluster_upstream_rq_retry_(success|overflow)$", "^envoy_server_(version|uptime|live)$" ] }, { "source_labels": ["container_name"], "label_matcher": "^envoy$", "dimensions": [["ClusterName","Namespace","envoy_http_conn_manager_prefix","envoy_response_code_class"]], "metric_selectors": [ "^envoy_http_downstream_rq_xx$" ] }, { "source_labels": ["container_name"], "label_matcher": "^fluent-bit.*$", "dimensions": [["ClusterName","Namespace","NodeName"]], "metric_selectors": [ "^fluentbit_output_errors_total$", "^fluentbit_input_bytes_total$", "^fluentbit_output_proc_bytes_total$", "^fluentbit_input_records_total$", "^fluentbit_output_proc_records_total$", "^fluentbit_output_retries_(total|failed_total)$" ] }, { "source_labels": ["job"], "label_matcher": "^kubernetes-pod-jmx$", "dimensions": [["ClusterName","Namespace"]], "metric_selectors": [ "^jvm_threads_(current|daemon)$", "^jvm_classes_loaded$", "^java_lang_operatingsystem_(freephysicalmemorysize|totalphysicalmemorysize|freeswapspacesize|totalswapspacesize|systemcpuload|processcpuload|availableprocessors|openfiledescriptorcount)$", "^catalina_manager_(rejectedsessions|activesessions)$", "^jvm_gc_collection_seconds_(count|sum)$", "^catalina_globalrequestprocessor_(bytesreceived|bytessent|requestcount|errorcount|processingtime)$" ] }, { "source_labels": ["job"], "label_matcher": "^kubernetes-pod-jmx$", "dimensions": [["ClusterName","Namespace","area"]], "metric_selectors": [ "^jvm_memory_bytes_used$" ] }, { "source_labels": ["job"], "label_matcher": "^kubernetes-pod-jmx$", "dimensions": [["ClusterName","Namespace","pool"]], "metric_selectors": [ "^jvm_memory_pool_bytes_used$" ] } ] } } }, "force_flush_interval": 5 } } kind: ConfigMap metadata: name: prometheus-cwagentconfig namespace: amazon-cloudwatch --- # create configmap for prometheus scrape config apiVersion: v1 data: # prometheus config prometheus.yaml: | global: scrape_interval: 1m scrape_timeout: 10s scrape_configs: - job_name: 'kubernetes-pod-appmesh-envoy' sample_limit: 10000 metrics_path: /stats/prometheus kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: [__meta_kubernetes_pod_container_name] action: keep regex: '^envoy$' - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace regex: ([^:]+)(?::\d+)?;(\d+) replacement: ${1}:9901 target_label: __address__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace source_labels: - __meta_kubernetes_namespace target_label: Namespace - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: pod_name - action: replace source_labels: - __meta_kubernetes_pod_container_name target_label: container_name - action: replace source_labels: - __meta_kubernetes_pod_controller_name target_label: pod_controller_name - action: replace source_labels: - __meta_kubernetes_pod_controller_kind target_label: pod_controller_kind - action: replace source_labels: - __meta_kubernetes_pod_phase target_label: pod_phase - job_name: 'kubernetes-pod-fluentbit-plugin' sample_limit: 10000 metrics_path: /api/v1/metrics/prometheus kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: [__meta_kubernetes_pod_container_name] action: keep regex: '^fluent-bit.*$' - source_labels: [__address__] action: replace regex: ([^:]+)(?::\d+)? replacement: ${1}:2020 target_label: __address__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace source_labels: - __meta_kubernetes_namespace target_label: Namespace - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: pod_name - action: replace source_labels: - __meta_kubernetes_pod_container_name target_label: container_name - action: replace source_labels: - __meta_kubernetes_pod_controller_name target_label: pod_controller_name - action: replace source_labels: - __meta_kubernetes_pod_controller_kind target_label: pod_controller_kind - action: replace source_labels: - __meta_kubernetes_pod_phase target_label: pod_phase - action: replace source_labels: - __meta_kubernetes_pod_node_name target_label: NodeName - job_name: kubernetes-service-endpoints sample_limit: 10000 kubernetes_sd_configs: - role: endpoints relabel_configs: - action: keep regex: true source_labels: - __meta_kubernetes_service_annotation_prometheus_io_scrape - action: replace regex: (https?) source_labels: - __meta_kubernetes_service_annotation_prometheus_io_scheme target_label: __scheme__ - action: replace regex: (.+) source_labels: - __meta_kubernetes_service_annotation_prometheus_io_path target_label: __metrics_path__ - action: replace regex: ([^:]+)(?::\d+)?;(\d+) replacement: $1:$2 source_labels: - __address__ - __meta_kubernetes_service_annotation_prometheus_io_port target_label: __address__ - action: labelmap regex: __meta_kubernetes_service_label_(.+) - action: replace source_labels: - __meta_kubernetes_namespace target_label: Namespace - action: replace source_labels: - __meta_kubernetes_service_name target_label: Service - action: replace source_labels: - __meta_kubernetes_pod_node_name target_label: kubernetes_node - action: replace source_labels: - __meta_kubernetes_pod_name target_label: pod_name - action: replace source_labels: - __meta_kubernetes_pod_container_name target_label: container_name metric_relabel_configs: - source_labels: [__name__] regex: 'go_gc_duration_seconds.*' action: drop - source_labels: [__name__, proxy] regex: "haproxy_frontend.+;(.+)" target_label: frontend replacement: "$1" - source_labels: [__name__, proxy] regex: "haproxy_server.+;(.+)" target_label: backend replacement: "$1" - source_labels: [__name__, proxy] regex: "haproxy_backend.+;(.+)" target_label: backend replacement: "$1" - regex: proxy action: labeldrop - job_name: 'kubernetes-pod-jmx' sample_limit: 10000 metrics_path: /metrics kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: [__address__] action: keep regex: '.*:9404$' - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace source_labels: - __meta_kubernetes_namespace target_label: Namespace - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: pod_name - action: replace source_labels: - __meta_kubernetes_pod_container_name target_label: container_name - action: replace source_labels: - __meta_kubernetes_pod_controller_name target_label: pod_controller_name - action: replace source_labels: - __meta_kubernetes_pod_controller_kind target_label: pod_controller_kind - action: replace source_labels: - __meta_kubernetes_pod_phase target_label: pod_phase metric_relabel_configs: - source_labels: [__name__] regex: 'jvm_gc_collection_seconds.*' action: drop kind: ConfigMap metadata: name: prometheus-config namespace: amazon-cloudwatch --- # create cwagent service account and role binding apiVersion: v1 kind: ServiceAccount metadata: name: cwagent-prometheus namespace: amazon-cloudwatch --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cwagent-prometheus-role rules: - apiGroups: [""] resources: - nodes - nodes/proxy - services - endpoints - pods verbs: ["get", "list", "watch"] - apiGroups: - extensions resources: - ingresses verbs: ["get", "list", "watch"] - nonResourceURLs: ["/metrics"] verbs: ["get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cwagent-prometheus-role-binding subjects: - kind: ServiceAccount name: cwagent-prometheus namespace: amazon-cloudwatch roleRef: kind: ClusterRole name: cwagent-prometheus-role apiGroup: rbac.authorization.k8s.io --- # Deployment apiVersion: apps/v1 kind: Deployment metadata: name: cwagent-prometheus namespace: amazon-cloudwatch spec: replicas: 1 selector: matchLabels: app: cwagent-prometheus template: metadata: labels: app: cwagent-prometheus spec: containers: - name: cloudwatch-agent image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.300036.0b573 imagePullPolicy: Always resources: limits: cpu: 1000m memory: 1000Mi requests: cpu: 200m memory: 200Mi # Please don't change below envs env: - name: CI_VERSION value: "k8s/1.3.20" # Please don't change the mountPath volumeMounts: - name: prometheus-cwagentconfig mountPath: /etc/cwagentconfig - name: prometheus-config mountPath: /etc/prometheusconfig volumes: - name: prometheus-cwagentconfig configMap: name: prometheus-cwagentconfig - name: prometheus-config configMap: name: prometheus-config terminationGracePeriodSeconds: 60 serviceAccountName: cwagent-prometheus