/* * Parseable Server (C) 2022 - 2025 Parseable, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . * */ use std::{ collections::HashMap, sync::{Arc, Mutex}, time::Duration, }; use async_trait::async_trait; use base64::Engine; use chrono::Utc; use http::header::AUTHORIZATION; use itertools::Itertools; use once_cell::sync::Lazy; use serde_json::{Value, json}; use tokio::sync::{Mutex as AsyncMutex, RwLock}; use tracing::{error, trace, warn}; use ulid::Ulid; use url::Url; use super::{ ALERTS, outbound_http_policy::{self, AlertTargetKind}, title_registry, }; use crate::{ alerts::{AlertError, AlertState, Context, alert_traits::CallableTarget}, metastore::metastore_traits::MetastoreObject, parseable::{DEFAULT_TENANT, PARSEABLE}, storage::object_storage::target_json_path, }; pub static TARGETS: Lazy = Lazy::new(|| TargetConfigs { target_configs: RwLock::new(HashMap::new()), mutation_lock: AsyncMutex::new(()), }); #[derive(Debug)] pub struct TargetConfigs { pub target_configs: RwLock>>, mutation_lock: AsyncMutex<()>, } impl TargetConfigs { /// Loads alerts from disk, blocks pub async fn load(&self) -> anyhow::Result<()> { let targets = PARSEABLE.metastore.get_targets().await?; let mut map = self.target_configs.write().await; for (tenant_id, targets) in targets { let inner = targets .into_iter() .map(|mut t| { t.tenant = Some(tenant_id.clone()); (t.id, t) }) .collect(); map.insert(tenant_id, inner); } Ok(()) } pub async fn create(&self, target: Target) -> Result<(), AlertError> { let tenant_id = target.tenant.as_deref().unwrap_or(DEFAULT_TENANT); let _mutation_guard = self.mutation_lock.lock().await; // Another node may have created a target since this node loaded its cache. let saved = PARSEABLE .metastore .get_targets_for_tenant(&target.tenant) .await?; if saved.iter().any(|item| item.name == target.name) { return Err(AlertError::DuplicateTargetName(target.name)); } if saved.iter().any(|item| item.id == target.id) { return Err(AlertError::InvalidTargetModification( "Target ID already exists".to_string(), )); } if !title_registry::reserve(&target.name, target.id, &target.tenant).await? { return Err(AlertError::DuplicateTargetName(target.name)); } if let Err(error) = PARSEABLE .metastore .put_target(&target, &target.tenant) .await { title_registry::release(&target.name, target.id, &target.tenant).await?; return Err(error.into()); } self.target_configs .write() .await .entry(tenant_id.to_owned()) .or_default() .insert(target.id, target); Ok(()) } pub async fn update(&self, target: Target) -> Result<(), AlertError> { PARSEABLE .metastore .put_target(&target, &target.tenant) .await?; let mut map = self.target_configs.write().await; let tenant_id = target.tenant.as_deref().unwrap_or(DEFAULT_TENANT); map.entry(tenant_id.to_owned()) .or_default() .insert(target.id, target); // map.insert(target.id, target.clone()); Ok(()) } pub async fn list(&self, tenant_id: &Option) -> Result, AlertError> { let tenant_id = tenant_id.as_deref().unwrap_or(DEFAULT_TENANT); let targets = if let Some(targets) = self.target_configs.read().await.get(tenant_id) { targets.values().cloned().collect_vec() } else { vec![] }; Ok(targets) } pub async fn get_target_by_id( &self, target_id: &Ulid, tenant_id: &Option, ) -> Result { let tenant_id = tenant_id.as_deref().unwrap_or(DEFAULT_TENANT); let target = if let Some(targets) = self.target_configs.read().await.get(tenant_id) { targets .get(target_id) .ok_or(AlertError::InvalidTargetID(target_id.to_string())) .cloned()? } else { return Err(AlertError::InvalidTargetID(target_id.to_string())); }; Ok(target) } pub async fn delete( &self, target_id: &Ulid, tenant_id: &Option, ) -> Result { let tenant = tenant_id.as_deref().unwrap_or(DEFAULT_TENANT); let _mutation_guard = self.mutation_lock.lock().await; // ensure that the target is not being used by any alert let guard = ALERTS.read().await; let alerts = if let Some(alerts) = guard.as_ref() { alerts } else { return Err(AlertError::CustomError("No AlertManager set".into())); }; for alert in alerts.get_all_alerts(tenant_id).await.values() { if alert.get_targets().contains(target_id) { return Err(AlertError::TargetInUse); } } let target = if let Some(targets) = self.target_configs.read().await.get(tenant) { targets .get(target_id) .cloned() .ok_or(AlertError::InvalidTargetID(target_id.to_string()))? } else { return Err(AlertError::InvalidTargetID(target_id.to_string())); }; title_registry::release(&target.name, target.id, tenant_id).await?; PARSEABLE .metastore .delete_target(&target, tenant_id) .await?; if let Some(targets) = self.target_configs.write().await.get_mut(tenant) { targets.remove(target_id); } Ok(target) } } #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] #[serde(untagged)] pub enum Retry { Infinite, Finite(usize), } impl Default for Retry { fn default() -> Self { Retry::Finite(1) } } #[derive(Debug, serde::Serialize, serde::Deserialize, Clone)] #[serde(rename_all = "camelCase")] #[serde(try_from = "TargetVerifier")] pub struct Target { pub name: String, #[serde(flatten)] pub target: TargetType, #[serde(default = "Ulid::new")] pub id: Ulid, pub tenant: Option, } impl Target { pub async fn validate_outbound_policy(&self) -> Result<(), AlertError> { let tenant_id = &self.tenant; match &self.target { TargetType::Slack(target) => { outbound_http_policy::prepare_alert_target( tenant_id, &target.endpoint, AlertTargetKind::Slack, false, None, ) .await?; } TargetType::Other(target) => { outbound_http_policy::prepare_alert_target( tenant_id, &target.endpoint, AlertTargetKind::Webhook, target.skip_tls_check, Some(&target.headers), ) .await?; } TargetType::AlertManager(target) => { let prepared = outbound_http_policy::prepare_alert_target( tenant_id, &target.endpoint, AlertTargetKind::AlertManager, target.skip_tls_check, None, ) .await?; if target.auth.is_some() && !prepared.authorization_allowed { return Err(outbound_http_policy::OutboundPolicyError::DeniedHeader( AUTHORIZATION.as_str().to_string(), ) .into()); } } } Ok(()) } pub fn mask(self) -> Value { match self.target { TargetType::Slack(slack_web_hook) => { json!({ "name":self.name, "type":"slack", "endpoint":format!("{}://********", slack_web_hook.endpoint.scheme()), "id":self.id }) } TargetType::Other(other_web_hook) => { let safe_headers: HashMap = other_web_hook .headers .into_keys() .map(|k| (k, "********".to_string())) .collect(); json!({ "name":self.name, "type":"webhook", "endpoint": format!("{}://********", other_web_hook.endpoint.scheme()), "headers":safe_headers, "skipTlsCheck":other_web_hook.skip_tls_check, "id":self.id }) } TargetType::AlertManager(alert_manager) => { let endpoint = format!("{}://********", alert_manager.endpoint.scheme()); if let Some(auth) = alert_manager.auth { let password = "********"; json!({ "name":self.name, "type":"webhook", "endpoint":endpoint, "username":auth.username, "password":password, "skipTlsCheck":alert_manager.skip_tls_check, "id":self.id }) } else { json!({ "name":self.name, "type":"webhook", "endpoint":endpoint, "username":Value::Null, "password":Value::Null, "skipTlsCheck":alert_manager.skip_tls_check, "id":self.id }) } } } } pub fn call(&self, context: Context) { trace!("target.call context- {context:?}"); let timeout = context.notification_config.clone(); let resolves = context.alert_info.alert_state; let mut state = timeout.state.lock().unwrap(); trace!("target.call state- {state:?}"); state.alert_state = resolves; match resolves { AlertState::Triggered => { if !state.timed_out { // call once and then start sleeping // reduce repeats by 1 call_target(self.target.clone(), self.tenant.clone(), context.clone()); // set state state.timed_out = true; state.awaiting_resolve = true; drop(state); self.spawn_timeout_task(&timeout, context.clone()); } } alert_state @ AlertState::NotTriggered => { state.alert_state = alert_state; if state.timed_out { // if in timeout and resolve came in, only process if it's the first one ( awaiting resolve ) if state.awaiting_resolve { state.awaiting_resolve = false; } else { // no further resolve will be considered in timeout period return; } } call_target(self.target.clone(), self.tenant.clone(), context); } // do not send out any notifs // (an eval should not have run!) AlertState::Disabled => {} } } fn spawn_timeout_task(&self, target_timeout: &NotificationConfig, alert_context: Context) { trace!("repeat-\n{target_timeout:?}"); let state = Arc::clone(&target_timeout.state); let retry = target_timeout.times; let timeout = target_timeout.interval; let target = self.target.clone(); let alert_id = alert_context.alert_info.alert_id; let tenant_id = self.tenant.clone(); let sleep_and_check_if_call = move |timeout_state: Arc>, current_state: AlertState| async move { tokio::time::sleep(Duration::from_secs(timeout * 60)).await; let mut state = timeout_state.lock().unwrap(); if current_state == AlertState::Triggered { state.awaiting_resolve = true; true } else { state.timed_out = false; false } }; trace!("Spawning retry task"); tokio::spawn(async move { // Get alerts manager reference once at the start let alerts = { let guard = ALERTS.read().await; if let Some(alerts) = guard.as_ref() { alerts.clone() } else { error!("No AlertManager set for alert_id: {alert_id}, stopping timeout task"); *state.lock().unwrap() = TimeoutState::default(); return; } }; // Lock released immediately match retry { Retry::Infinite => loop { let current_state = if let Ok(state) = alerts.get_state(alert_id, &tenant_id).await { state } else { *state.lock().unwrap() = TimeoutState::default(); warn!( "Unable to fetch state for given alert_id- {alert_id}, stopping target notifs" ); return; }; let should_call = sleep_and_check_if_call(Arc::clone(&state), current_state).await; if should_call { call_target(target.clone(), tenant_id.clone(), alert_context.clone()) } }, Retry::Finite(times) => { for _ in 0..(times - 1) { let current_state = if let Ok(state) = alerts.get_state(alert_id, &tenant_id).await { state } else { *state.lock().unwrap() = TimeoutState::default(); warn!( "Unable to fetch state for given alert_id- {alert_id}, stopping target notifs" ); return; }; let should_call = sleep_and_check_if_call(Arc::clone(&state), current_state).await; if should_call { call_target(target.clone(), tenant_id.clone(), alert_context.clone()) } } } } *state.lock().unwrap() = TimeoutState::default(); }); } } impl MetastoreObject for Target { fn get_object_path(&self) -> String { target_json_path(&self.id, &self.tenant).to_string() } fn get_object_id(&self) -> String { self.id.to_string() } } fn call_target(target: TargetType, tenant_id: Option, context: Context) { trace!("Calling target with context- {context:?}"); tokio::spawn(async move { target.call(&tenant_id, &context).await }); } #[derive(Debug, serde::Deserialize)] pub struct NotificationConfigVerifier { interval: Option, times: Option, } #[derive(Debug, serde::Deserialize)] #[serde(rename_all = "camelCase")] pub struct TargetVerifier { pub name: String, #[serde(flatten)] pub target: TargetType, #[serde(default)] pub notification_config: Option, #[serde(default = "Ulid::new")] pub id: Ulid, pub tenant_id: Option, } impl TryFrom for Target { type Error = String; fn try_from(value: TargetVerifier) -> Result { let mut timeout = NotificationConfig::default(); // Default is Infinite in case of alertmanager if matches!(value.target, TargetType::AlertManager(_)) { timeout.times = Retry::Infinite } if let Some(notification_config) = value.notification_config { let interval = notification_config.interval.map(|ref interval| *interval); if let Some(interval) = interval { timeout.interval = interval } if let Some(times) = notification_config.times { timeout.times = Retry::Finite(times) } } Ok(Target { name: value.name, target: value.target, id: value.id, tenant: value.tenant_id, }) } } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] #[serde(tag = "type")] #[serde(deny_unknown_fields)] pub enum TargetType { #[serde(rename = "slack")] Slack(SlackWebHook), #[serde(rename = "webhook")] Other(OtherWebHook), #[serde(rename = "alertManager")] AlertManager(AlertManager), } impl TargetType { pub async fn call(&self, tenant_id: &Option, payload: &Context) { match self { TargetType::Slack(target) => target.call(tenant_id, payload).await, TargetType::Other(target) => target.call(tenant_id, payload).await, TargetType::AlertManager(target) => target.call(tenant_id, payload).await, } } } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct SlackWebHook { endpoint: Url, } #[async_trait] impl CallableTarget for SlackWebHook { async fn call(&self, tenant_id: &Option, payload: &Context) { let alert = match payload.alert_info.alert_state { AlertState::Triggered => { serde_json::json!({ "text": payload.message }) } AlertState::NotTriggered => { serde_json::json!({ "text": payload.default_resolved_string() }) } AlertState::Disabled => { serde_json::json!({ "text": payload.default_disabled_string() }) } }; // Revalidate immediately before delivery because stored targets and DNS can change. let prepared = match outbound_http_policy::prepare_alert_target( tenant_id, &self.endpoint, AlertTargetKind::Slack, false, None, ) .await { Ok(prepared) => prepared, Err(err) => { error!("Slack alert target rejected by outbound policy:{err}"); return; } }; if let Err(e) = prepared .client .post(self.endpoint.clone()) .json(&alert) .send() .await { error!("Couldn't make call to webhook, error:{}", e) } } } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] pub struct OtherWebHook { endpoint: Url, #[serde(default)] headers: HashMap, #[serde(default)] skip_tls_check: bool, } #[async_trait] impl CallableTarget for OtherWebHook { async fn call(&self, tenant_id: &Option, payload: &Context) { let alert = match payload.alert_info.alert_state { AlertState::Triggered => payload.message.clone(), AlertState::NotTriggered => payload.default_resolved_string(), AlertState::Disabled => payload.default_disabled_string(), }; let prepared = match outbound_http_policy::prepare_alert_target( tenant_id, &self.endpoint, AlertTargetKind::Webhook, self.skip_tls_check, Some(&self.headers), ) .await { Ok(prepared) => prepared, Err(err) => { error!("Webhook alert target rejected by outbound policy:{err}"); return; } }; let request = prepared .client .post(self.endpoint.clone()) .headers(prepared.headers); if let Err(e) = request.body(alert).send().await { error!("Couldn't make call to webhook, error:{}", e) } } } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] pub struct AlertManager { endpoint: Url, #[serde(default)] skip_tls_check: bool, #[serde(flatten)] auth: Option, } #[async_trait] impl CallableTarget for AlertManager { async fn call(&self, tenant_id: &Option, payload: &Context) { let mut alerts = serde_json::json!([{ "labels": { "alertname": payload.alert_info.alert_name, // "stream": payload.stream, "deployment_instance": payload.deployment_info.deployment_instance, "deployment_id": payload.deployment_info.deployment_id, "deployment_mode": payload.deployment_info.deployment_mode }, "annotations": { "message": "MESSAGE", "reason": "REASON" } }]); let alert = &mut alerts[0]; // fill in status label accordingly match payload.alert_info.alert_state { AlertState::Triggered => alert["labels"]["status"] = "triggered".into(), AlertState::NotTriggered => { alert["labels"]["status"] = "not-triggered".into(); alert["annotations"]["reason"] = serde_json::Value::String(payload.default_resolved_string()); alert["endsAt"] = Utc::now() .to_rfc3339_opts(chrono::SecondsFormat::Millis, true) .into(); } AlertState::Disabled => alert["labels"]["status"] = "disabled".into(), }; let prepared = match outbound_http_policy::prepare_alert_target( tenant_id, &self.endpoint, AlertTargetKind::AlertManager, self.skip_tls_check, None, ) .await { Ok(prepared) => prepared, Err(err) => { error!("Alertmanager target rejected by outbound policy:{err}"); return; } }; let mut request = prepared.client.post(self.endpoint.clone()).json(&alerts); if let Some(Auth { username, password }) = &self.auth { // Credentials are only safe when the destination itself is explicitly allowlisted. if !prepared.authorization_allowed { error!( "Alertmanager credentials blocked by outbound policy for destination:{}", self.endpoint ); return; } let basic_auth_value = "Basic ".to_string() + &base64::prelude::BASE64_STANDARD.encode(format!("{username}:{password}")); request = request.header(AUTHORIZATION, basic_auth_value); } if let Err(e) = request.send().await { error!("Couldn't make call to alertmanager, error:{}", e) } } } #[derive(Debug, serde::Serialize, serde::Deserialize, Clone)] pub struct NotificationConfig { pub interval: u64, #[serde(skip)] pub times: Retry, #[serde(skip)] pub state: Arc>, } impl Default for NotificationConfig { fn default() -> Self { Self { interval: 1, times: Retry::default(), state: Arc::>::default(), } } } #[derive(Debug, Default, Clone, Copy)] pub struct TimeoutState { pub alert_state: AlertState, pub timed_out: bool, pub awaiting_resolve: bool, } #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct Auth { username: String, password: String, } #[cfg(test)] mod tests { use super::*; #[test] fn test_masked_target_hides_secrets() { let target = Target { name: "test-target".into(), id: Ulid::new(), tenant: None, target: TargetType::AlertManager(AlertManager { endpoint: "https://internal.corp/api".parse().unwrap(), auth: Some(Auth { username: "admin".into(), password: "SuperSecretPassword123".into(), }), skip_tls_check: false, }), }; let masked = target.mask(); let masked_str = serde_json::to_string(&masked).unwrap(); // These assertions MUST pass, or the build fails assert!( !masked_str.contains("SuperSecretPassword123"), "Password leaked in masked response!" ); assert!( masked_str.contains("********"), "Expected redaction marker not found!" ); } }