# Production Monitoring Stack - Prometheus + Grafana
# Phase 3: Production Observability Infrastructure

---
# Prometheus Configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: janua-production
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
      external_labels:
        cluster: 'janua-production'
        environment: 'production'

    rule_files:
      - "/etc/prometheus/rules/*.yml"

    alerting:
      alertmanagers:
        - static_configs:
            - targets:
              - alertmanager:9093

    scrape_configs:
      # API Monitoring
      - job_name: 'janua-api'
        kubernetes_sd_configs:
        - role: endpoints
          namespaces:
            names:
            - janua-production
        relabel_configs:
        - source_labels: [__meta_kubernetes_service_name]
          action: keep
          regex: janua-api-service
        - source_labels: [__meta_kubernetes_endpoint_port_name]
          action: keep
          regex: http
        metrics_path: '/metrics/performance'
        scrape_interval: 10s

      # Database Monitoring
      - job_name: 'postgres'
        kubernetes_sd_configs:
        - role: endpoints
          namespaces:
            names:
            - janua-production
        relabel_configs:
        - source_labels: [__meta_kubernetes_service_name]
          action: keep
          regex: postgres-exporter-service

      # Redis Monitoring
      - job_name: 'redis'
        kubernetes_sd_configs:
        - role: endpoints
          namespaces:
            names:
            - janua-production
        relabel_configs:
        - source_labels: [__meta_kubernetes_service_name]
          action: keep
          regex: redis-exporter-service

      # Kubernetes Monitoring
      - job_name: 'kubernetes-apiservers'
        kubernetes_sd_configs:
        - role: endpoints
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
          action: keep
          regex: default;kubernetes;https

      - job_name: 'kubernetes-nodes'
        kubernetes_sd_configs:
        - role: node
        scheme: https
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
        relabel_configs:
        - action: labelmap
          regex: __meta_kubernetes_node_label_(.+)

      - job_name: 'kubernetes-pods'
        kubernetes_sd_configs:
        - role: pod
          namespaces:
            names:
            - janua-production
        relabel_configs:
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)

  # Alerting Rules
  alert_rules.yml: |
    groups:
    - name: janua.rules
      rules:
      # API Performance Alerts
      - alert: HighResponseTime
        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket{job="janua-api"}[5m])) > 0.1
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High API response time detected"
          description: "95th percentile response time is {{ $value }}s (>100ms)"

      - alert: HighErrorRate
        expr: rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "High API error rate detected"
          description: "Error rate is {{ $value | humanizePercentage }}"

      # Resource Alerts
      - alert: HighCPUUsage
        expr: rate(cpu_usage_total[5m]) > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage detected"
          description: "CPU usage is {{ $value | humanizePercentage }}"

      - alert: HighMemoryUsage
        expr: memory_usage_ratio > 0.85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage detected"
          description: "Memory usage is {{ $value | humanizePercentage }}"

      # Database Alerts
      - alert: DatabaseConnectionHigh
        expr: pg_stat_database_numbackends / pg_settings_max_connections > 0.8
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High database connection usage"
          description: "Database connections at {{ $value | humanizePercentage }} of limit"

      - alert: SlowQueries
        expr: rate(pg_stat_statements_mean_time_ms[5m]) > 100
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Slow database queries detected"
          description: "Average query time is {{ $value }}ms"

      # Cache Alerts
      - alert: LowCacheHitRate
        expr: redis_keyspace_hits_total / (redis_keyspace_hits_total + redis_keyspace_misses_total) < 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Low Redis cache hit rate"
          description: "Cache hit rate is {{ $value | humanizePercentage }}"

---
# Prometheus Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: janua-production
  labels:
    app: prometheus
    component: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
        component: monitoring
    spec:
      serviceAccountName: prometheus-serviceaccount
      containers:
      - name: prometheus
        image: prom/prometheus:v2.45.0
        ports:
        - containerPort: 9090
          name: http
        args:
        - '--config.file=/etc/prometheus/prometheus.yml'
        - '--storage.tsdb.path=/prometheus'
        - '--web.console.libraries=/etc/prometheus/console_libraries'
        - '--web.console.templates=/etc/prometheus/consoles'
        - '--storage.tsdb.retention.time=30d'
        - '--web.enable-lifecycle'
        - '--web.enable-admin-api'
        resources:
          requests:
            cpu: 500m
            memory: 1Gi
          limits:
            cpu: 2000m
            memory: 4Gi
        volumeMounts:
        - name: config
          mountPath: /etc/prometheus
        - name: data
          mountPath: /prometheus
        livenessProbe:
          httpGet:
            path: /-/healthy
            port: 9090
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /-/ready
            port: 9090
          initialDelaySeconds: 5
          periodSeconds: 5
        securityContext:
          runAsNonRoot: true
          runAsUser: 65534
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
      volumes:
      - name: config
        configMap:
          name: prometheus-config
      - name: data
        persistentVolumeClaim:
          claimName: prometheus-data

---
# Prometheus Service Account
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus-serviceaccount
  namespace: janua-production

---
# Prometheus RBAC
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus-clusterrole
rules:
- apiGroups: [""]
  resources:
  - nodes
  - nodes/proxy
  - services
  - endpoints
  - pods
  verbs: ["get", "list", "watch"]
- apiGroups:
  - extensions
  resources:
  - ingresses
  verbs: ["get", "list", "watch"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus-clusterrolebinding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus-clusterrole
subjects:
- kind: ServiceAccount
  name: prometheus-serviceaccount
  namespace: janua-production

---
# Prometheus PVC
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-data
  namespace: janua-production
spec:
  accessModes:
  - ReadWriteOnce
  storageClassName: "fast-ssd"
  resources:
    requests:
      storage: 50Gi

---
# Prometheus Service
apiVersion: v1
kind: Service
metadata:
  name: prometheus-service
  namespace: janua-production
  labels:
    app: prometheus
    component: monitoring
spec:
  type: ClusterIP
  ports:
  - port: 9090
    targetPort: 9090
    protocol: TCP
    name: http
  selector:
    app: prometheus

---
# Grafana ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-config
  namespace: janua-production
data:
  grafana.ini: |
    [server]
    http_port = 3000
    
    [security]
    admin_user = admin
    disable_gravatar = true
    
    [users]
    allow_sign_up = false
    
    [auth.anonymous]
    enabled = false
    
    [dashboards]
    default_home_dashboard_path = /etc/grafana/provisioning/dashboards/janua-overview.json
    
    [log]
    mode = console
    level = info

  # Data source configuration
  datasources.yml: |
    apiVersion: 1
    datasources:
    - name: Prometheus
      type: prometheus
      access: proxy
      url: http://prometheus-service:9090
      isDefault: true

  # Dashboard configuration
  dashboards.yml: |
    apiVersion: 1
    providers:
    - name: 'default'
      orgId: 1
      folder: ''
      type: file
      disableDeletion: false
      updateIntervalSeconds: 10
      options:
        path: /etc/grafana/provisioning/dashboards

---
# Grafana Secret
apiVersion: v1
kind: Secret
metadata:
  name: grafana-secret
  namespace: janua-production
type: Opaque
stringData:
  GF_SECURITY_ADMIN_PASSWORD: "REPLACE_WITH_SECURE_GRAFANA_PASSWORD"

---
# Grafana Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: grafana
  namespace: janua-production
  labels:
    app: grafana
    component: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: grafana
  template:
    metadata:
      labels:
        app: grafana
        component: monitoring
    spec:
      containers:
      - name: grafana
        image: grafana/grafana:10.0.0
        ports:
        - containerPort: 3000
          name: http
        envFrom:
        - secretRef:
            name: grafana-secret
        env:
        - name: GF_USERS_ALLOW_SIGN_UP
          value: "false"
        - name: GF_INSTALL_PLUGINS
          value: "grafana-piechart-panel,grafana-clock-panel"
        resources:
          requests:
            cpu: 100m
            memory: 256Mi
          limits:
            cpu: 500m
            memory: 1Gi
        volumeMounts:
        - name: config
          mountPath: /etc/grafana/grafana.ini
          subPath: grafana.ini
        - name: config
          mountPath: /etc/grafana/provisioning/datasources/datasources.yml
          subPath: datasources.yml
        - name: config
          mountPath: /etc/grafana/provisioning/dashboards/dashboards.yml
          subPath: dashboards.yml
        - name: data
          mountPath: /var/lib/grafana
        livenessProbe:
          httpGet:
            path: /api/health
            port: 3000
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /api/health
            port: 3000
          initialDelaySeconds: 5
          periodSeconds: 5
        securityContext:
          runAsNonRoot: true
          runAsUser: 472
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
      volumes:
      - name: config
        configMap:
          name: grafana-config
      - name: data
        persistentVolumeClaim:
          claimName: grafana-data

---
# Grafana PVC
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: grafana-data
  namespace: janua-production
spec:
  accessModes:
  - ReadWriteOnce
  storageClassName: "fast-ssd"
  resources:
    requests:
      storage: 10Gi

---
# Grafana Service
apiVersion: v1
kind: Service
metadata:
  name: grafana-service
  namespace: janua-production
  labels:
    app: grafana
    component: monitoring
spec:
  type: ClusterIP
  ports:
  - port: 3000
    targetPort: 3000
    protocol: TCP
    name: http
  selector:
    app: grafana

---
# Grafana Ingress
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: grafana-ingress
  namespace: janua-production
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    cert-manager.io/cluster-issuer: "letsencrypt-prod"
    nginx.ingress.kubernetes.io/auth-type: basic
    nginx.ingress.kubernetes.io/auth-secret: grafana-basic-auth
    nginx.ingress.kubernetes.io/auth-realm: "Authentication Required - Janua Monitoring"
spec:
  tls:
  - hosts:
    - monitoring.janua.dev
    secretName: grafana-tls
  rules:
  - host: monitoring.janua.dev
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: grafana-service
            port:
              number: 3000