#! /bin/bash set -euo pipefail # localdns systemd unit. # This systemd unit runs coredns as a caching with serve-stale functionality for both pod DNS and node DNS queries. # It also upgrades to TCP for better reliability of upstream connections. # Also defined in cse_helper file. Not sourcing the entire cse_helper file here. # These exit codes will be handled in cse_config file. ERR_LOCALDNS_FAIL=216 # Unable to start localdns systemd unit. ERR_LOCALDNS_COREFILE_NOTFOUND=217 # Localdns corefile not found. ERR_LOCALDNS_SLICEFILE_NOTFOUND=218 # Localdns slicefile not found. ERR_LOCALDNS_BINARY_ERR=219 # Localdns binary not found or not executable. # Global constants used in this file. # ------------------------------------------------------------------------------------------------- # Localdns script path. LOCALDNS_SCRIPT_PATH="/opt/azure/containers/localdns" # Cgroup v2 directory for the localdns service (matches Slice=localdns.slice in localdns.service). LOCALDNS_CGROUP_DIR="/sys/fs/cgroup/localdns.slice/localdns.service" # Localdns corefile is created only when localdns profile has state enabled. # This should match with 'path' defined in parts/linux/cloud-init/nodecustomdata.yml. LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/localdns.corefile" # This is the localdns corefile that has updated UpstreamDNSServerIPs and will be used by the localdns systemd unit. UPDATED_LOCALDNS_CORE_FILE="${LOCALDNS_SCRIPT_PATH}/updated.localdns.corefile" # This is slice file used by localdns systemd unit. # This should match with 'path' defined in parts/linux/cloud-init/nodecustomdata.yml. LOCALDNS_SLICE_PATH="/etc/systemd/system" LOCALDNS_SLICE_FILE="${LOCALDNS_SLICE_PATH}/localdns.slice" # Azure DNS IP. AZURE_DNS_IP="168.63.129.16" # Localdns node listener IP. LOCALDNS_NODE_LISTENER_IP="169.254.10.10" # Localdns cluster listener IP. LOCALDNS_CLUSTER_LISTENER_IP="169.254.10.11" # Localdns shutdown delay. LOCALDNS_SHUTDOWN_DELAY=5 # Localdns pid file. LOCALDNS_PID_FILE="/run/localdns.pid" # Path of coredns binary used by localdns. COREDNS_BINARY_PATH="${LOCALDNS_SCRIPT_PATH}/binary/coredns" # Path to systemd resolv. RESOLV_CONF="/run/systemd/resolve/resolv.conf" # Curl check if localdns is running. # This is used by start_localdns_watchdog and wait_for_localdns_ready. # Declared as an array so it expands with correct word-splitting/quoting; --noproxy ensures # the request to the node-local listener bypasses any system-wide HTTP proxy (inherited via # systemd DefaultEnvironment on http-proxy clusters), which cannot route to a link-local address. CURL_COMMAND=(curl -s --noproxy "${LOCALDNS_NODE_LISTENER_IP}" --connect-timeout 5 --max-time 10 "http://${LOCALDNS_NODE_LISTENER_IP}:8181/ready") # Constant for networkctl reload command. # This is used by disable_dhcp_use_clusterlistener and cleanup_localdns_configs functions. NETWORKCTL_RELOAD_CMD="networkctl reload" START_LOCALDNS_TIMEOUT=10 # DNS health check timeout. DNS_HEALTH_CHECK_TIMEOUT=2 DNS_HEALTH_CHECK_TRIES=2 # Function definitions used in this file. # functions defined until "${__SOURCED__:+return}" are sourced and tested in - # spec/parts/linux/cloud-init/artifacts/localdns_spec.sh. # ------------------------------------------------------------------------------------------------- # Verify that the localdns corefile exists and is not empty. verify_localdns_corefile() { if [ -z "${LOCALDNS_CORE_FILE:-}" ]; then echo "LOCALDNS_CORE_FILE is not set or is empty." return 1 fi # Check if corefile exists, is not empty, else attempt to regenerate it. if [ ! -f "${LOCALDNS_CORE_FILE}" ] || [ ! -s "${LOCALDNS_CORE_FILE}" ]; then echo "Localdns corefile either does not exist or is empty at ${LOCALDNS_CORE_FILE}." echo "Attempting to regenerate localdns corefile..." if regenerate_localdns_corefile; then echo "Localdns corefile regenerated successfully." return 0 else echo "Failed to regenerate localdns corefile." return 1 fi fi return 0 } # Verify that the localdns slice file exists and is not empty. verify_localdns_slicefile() { if [ -z "${LOCALDNS_SLICE_FILE:-}" ]; then echo "LOCALDNS_SLICE_FILE is not set or is empty." return 1 fi if [ ! -f "${LOCALDNS_SLICE_FILE}" ] || [ ! -s "${LOCALDNS_SLICE_FILE}" ]; then echo "Localdns slice file does not exist at ${LOCALDNS_SLICE_FILE}." return 1 fi return 0 } # Verify that the localdns binary exists and is executable. verify_localdns_binary() { if [ -z "${COREDNS_BINARY_PATH:-}" ]; then echo "COREDNS_BINARY_PATH is not set or is empty." return 1 fi if [ ! -f "${COREDNS_BINARY_PATH}" ] || [ ! -x "${COREDNS_BINARY_PATH}" ]; then echo "Coredns binary either doesn't exist or isn't executable at ${COREDNS_BINARY_PATH}." return 1 fi if ! "${COREDNS_BINARY_PATH}" --version >/dev/null 2>&1; then echo "Failed to execute '--version'." return 1 fi return 0 } # Regenerate the localdns corefile from base64 encoded content. # This is used when the corefile goes missing. regenerate_localdns_corefile() { local corefile_to_use corefile_to_use=$(select_localdns_corefile) || true if [ -z "${corefile_to_use}" ]; then echo "No corefile selected. Cannot regenerate corefile." return 1 fi echo "Regenerating localdns corefile at ${LOCALDNS_CORE_FILE}" mkdir -p "$(dirname "${LOCALDNS_CORE_FILE}")" # Decode base64 corefile content and write to corefile. if ! echo "${corefile_to_use}" | base64 -d > "${LOCALDNS_CORE_FILE}"; then echo "Failed to decode and write corefile." return 1 fi chmod 0644 "${LOCALDNS_CORE_FILE}" || { echo "Failed to set permissions on ${LOCALDNS_CORE_FILE}" return 1 } echo "Successfully regenerated localdns corefile." return 0 } # Replace AzureDNSIP in corefile with VNET DNS ServerIPs if necessary. replace_azurednsip_in_corefile() { if [ -z "${RESOLV_CONF:-}" ]; then echo "RESOLV_CONF is not set or is empty." return 1 fi if [ -z "${AZURE_DNS_IP:-}" ]; then echo "AZURE_DNS_IP is not set or is empty." return 1 fi if [ ! -f "$RESOLV_CONF" ]; then echo "$RESOLV_CONF not found." return 1 fi # Get the upstream VNET DNS servers from /run/systemd/resolve/resolv.conf. UPSTREAM_VNET_DNS_SERVERS=$(awk '/^nameserver/ {print $2}' "$RESOLV_CONF" | paste -sd' ') if [ -z "${UPSTREAM_VNET_DNS_SERVERS}" ] || [ "$UPSTREAM_VNET_DNS_SERVERS" = '""' ]; then echo "No Upstream VNET DNS servers found in $RESOLV_CONF." return 1 fi echo "Found upstream VNET DNS servers: ${UPSTREAM_VNET_DNS_SERVERS}" # Based on customer input, corefile was generated in pkg/agent/baker.go. # Replace 168.63.129.16 with VNET DNS ServerIPs only if VNET DNS ServerIPs is not equal to 168.63.129.16 # and also not equal to the localdns node listener IP to avoid creating a circular dependency. # Corefile will have 168.63.129.16 when user input has VnetDNS value for forwarddestination. # Note - For root domain under VnetDNSOverrides, all DNS traffic should be forwarded to VnetDNS. cp "${LOCALDNS_CORE_FILE}" "${UPDATED_LOCALDNS_CORE_FILE}" || { echo "Failed to copy ${LOCALDNS_CORE_FILE} to ${UPDATED_LOCALDNS_CORE_FILE}" return 1 } if [ "${UPSTREAM_VNET_DNS_SERVERS}" != "${AZURE_DNS_IP}" ] && [ "${UPSTREAM_VNET_DNS_SERVERS}" != "${LOCALDNS_NODE_LISTENER_IP}" ]; then echo "Replacing Azure DNS IP ${AZURE_DNS_IP} with upstream VNET DNS servers ${UPSTREAM_VNET_DNS_SERVERS} in corefile ${UPDATED_LOCALDNS_CORE_FILE}" sed -i -e "s|${AZURE_DNS_IP}|${UPSTREAM_VNET_DNS_SERVERS}|g" "${UPDATED_LOCALDNS_CORE_FILE}" || { echo "Replacing AzureDNSIP in corefile failed." return 1 } echo "Successfully updated ${UPDATED_LOCALDNS_CORE_FILE}" else echo "Skipping DNS IP replacement. Upstream VNET DNS servers (${UPSTREAM_VNET_DNS_SERVERS}) match either Azure DNS IP (${AZURE_DNS_IP}) or localdns node listener IP (${LOCALDNS_NODE_LISTENER_IP})" fi if [ ! -f "${UPDATED_LOCALDNS_CORE_FILE}" ] || [ ! -s "${UPDATED_LOCALDNS_CORE_FILE}" ]; then echo "Updated Localdns corefile either does not exist or is empty at ${UPDATED_LOCALDNS_CORE_FILE}." return 1 fi chmod 0644 "${UPDATED_LOCALDNS_CORE_FILE}" || { echo "Failed to set permissions on ${UPDATED_LOCALDNS_CORE_FILE}" return 1 } # Export forward IPs to .prom file for metrics exporter (best-effort) # This avoids parsing the corefile on every metrics scrape # Metrics export is non-fatal - if it fails, log warning and continue so DNS service isn't affected local FORWARD_IPS_PROM_FILE="${LOCALDNS_SCRIPT_PATH}/forward_ips.prom" # Parse forward IPs from ALL server blocks in the corefile. # Each metric line includes a "block" label identifying which zone the forward came from # (e.g., block=".:53", block="cluster.local:53"). # # Single-pass awk outputs "bind_ip|zone|forward_ip" triples: # - Tracks current zone from block headers like ".:53 {" or "cluster.local:53 {" # - Tracks bind IP from "bind 169.254.10.10" or "bind 169.254.10.11" # - Extracts forward IPs from "forward . ..." lines # - Resets state on block close "}" local forward_entries bind_ip block fwd_ip local vnetdns_found=false kubedns_found=false forward_entries=$(awk ' /^[^ #].*:53 / { block = $1 bind_ip = "" } /bind / { for (i=2; i<=NF; i++) { if ($i ~ /^169\.254\.10\.(10|11)$/) bind_ip = $i } } /forward \. / && bind_ip != "" { for (i=3; i<=NF; i++) { if ($i ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) print bind_ip "|" block "|" $i } } /^}/ { block = ""; bind_ip = "" } ' "${UPDATED_LOCALDNS_CORE_FILE}") # Write Prometheus metrics to temp file, then atomically rename # This prevents the exporter from reading a partially-written file during scrapes # Generate one metric line per IP+block combination local tmp if ! tmp="$(mktemp "${FORWARD_IPS_PROM_FILE}.XXXXXX")"; then echo "WARNING: Failed to create temp file for ${FORWARD_IPS_PROM_FILE}. Metrics export skipped." elif ! { echo "# HELP localdns_vnetdns_forward_info VnetDNS forward plugin IP address from corefile" echo "# TYPE localdns_vnetdns_forward_info gauge" while IFS='|' read -r bind_ip block fwd_ip; do [ "$bind_ip" = "${LOCALDNS_NODE_LISTENER_IP}" ] || continue echo "localdns_vnetdns_forward_info{ip=\"${fwd_ip}\",block=\"${block}\",status=\"ok\"} 1" vnetdns_found=true done <<< "$forward_entries" if [ "$vnetdns_found" = false ]; then echo 'localdns_vnetdns_forward_info{ip="unknown",block="none",status="missing"} 0' fi echo "# HELP localdns_kubedns_forward_info KubeDNS forward plugin IP address from corefile" echo "# TYPE localdns_kubedns_forward_info gauge" while IFS='|' read -r bind_ip block fwd_ip; do [ "$bind_ip" = "${LOCALDNS_CLUSTER_LISTENER_IP}" ] || continue echo "localdns_kubedns_forward_info{ip=\"${fwd_ip}\",block=\"${block}\",status=\"ok\"} 1" kubedns_found=true done <<< "$forward_entries" if [ "$kubedns_found" = false ]; then echo 'localdns_kubedns_forward_info{ip="unknown",block="none",status="missing"} 0' fi } > "$tmp"; then echo "WARNING: Failed to write metrics to temp file $tmp. Metrics export skipped." rm -f "$tmp" elif ! chmod 0644 "$tmp"; then echo "WARNING: Failed to set permissions on temp file $tmp. Metrics export skipped." rm -f "$tmp" elif ! mv -f "$tmp" "${FORWARD_IPS_PROM_FILE}"; then echo "WARNING: Failed to move temp file to ${FORWARD_IPS_PROM_FILE}. Metrics export skipped." rm -f "$tmp" else echo "Successfully exported forward IPs to ${FORWARD_IPS_PROM_FILE}" fi # Persist the upstream DNS server(s) so aks-localdns-hosts-setup.sh can resolve against them # directly, bypassing localdns. Without this, the timer's dig queries would go through # localdns (169.254.10.10) → hosts plugin → stale answers from /etc/localdns/hosts, # creating a self-referential loop where IPs can never refresh. local upstream_dns_file="/etc/localdns/upstream-dns" mkdir -p "$(dirname "${upstream_dns_file}")" 2>/dev/null || true echo "${UPSTREAM_VNET_DNS_SERVERS}" > "${upstream_dns_file}" || { echo "WARNING: Failed to write upstream DNS to ${upstream_dns_file}" # Non-fatal: if this file is missing or empty, aks-localdns-hosts-setup.sh falls back to the system resolver. } chmod 0644 "${upstream_dns_file}" 2>/dev/null || true echo "Persisted upstream DNS servers to ${upstream_dns_file}: ${UPSTREAM_VNET_DNS_SERVERS}" return 0 } # Build iptables rules to skip conntrack for DNS traffic to localdns. build_localdns_iptable_rules() { # These rules skip conntrack for DNS traffic to the local DNS service IPs to save conntrack table space. # OUTPUT rules affect node services and hostNetwork: true pods. # PREROUTING rules affect traffic from regular pods. # Loop over chains, IPs, and protocols to create the rules for CHAIN in OUTPUT PREROUTING; do for IP in ${LOCALDNS_NODE_LISTENER_IP} ${LOCALDNS_CLUSTER_LISTENER_IP}; do for PROTO in tcp udp; do # Add rule to IPTABLES_RULES array IPTABLES_RULES+=("${CHAIN} -p ${PROTO} -d ${IP} --dport 53 -j NOTRACK") done done done } # Verify that the default route interface is set and not empty. verify_default_route_interface() { if [ -z "${DEFAULT_ROUTE_INTERFACE:-}" ]; then echo "Unable to determine the default route interface for ${AZURE_DNS_IP}." # Extracting default route interface using AzureDNSIP should work in most of the cases. # But if user or some process blackholes the route to AzureDNSIP, then - # Attempt to determine the default route interface using the upstream VNET DNS servers. # This will typically be eth0. VNET_DNS_SERVERS=$(awk '/^nameserver/ {print $2}' "$RESOLV_CONF" | paste -sd' ') # Extract first DNS server for route determination. FIRST_DNS_SERVER=$(echo "${VNET_DNS_SERVERS}" | awk '{print $1}') if [ -n "${FIRST_DNS_SERVER}" ] && [ "${FIRST_DNS_SERVER}" != "${LOCALDNS_NODE_LISTENER_IP}" ]; then echo "Using upstream VNET DNS server: ${FIRST_DNS_SERVER} to determine default route interface." DEFAULT_ROUTE_INTERFACE="$(ip -j route get "${FIRST_DNS_SERVER}" 2>/dev/null | jq -r 'if type == "array" and length > 0 then .[0].dev else empty end')" fi if [ -z "${DEFAULT_ROUTE_INTERFACE:-}" ]; then echo "Unable to determine the default route interface using fallback method with ${FIRST_DNS_SERVER}." return 1 fi fi return 0 } # Verify that the network file exists and is not empty. verify_network_file() { if [ ! -f "${NETWORK_FILE:-}" ]; then echo "Unable to determine network file for interface." return 1 fi return 0 } # Verify that the network drop-in directory exists and is not empty. verify_network_dropin_dir() { if [ -z "${NETWORK_DROPIN_DIR:-}" ]; then echo "NETWORK_DROPIN_DIR is not set or is empty." return 1 fi if [ ! -d "${NETWORK_DROPIN_DIR}" ]; then echo "Network drop-in directory does not exist." return 1 fi return 0 } # Initialize network variables that may not be set during restarts or cleanup traps. # This function can be called both during startup and in cleanup functions to ensure variables are available. initialize_network_variables() { # Determine the default route interface if not already set. # This will typically be eth0. if [ -z "${DEFAULT_ROUTE_INTERFACE:-}" ]; then DEFAULT_ROUTE_INTERFACE="$(ip -j route get "${AZURE_DNS_IP}" 2>/dev/null | jq -r 'if type == "array" and length > 0 then .[0].dev else empty end')" if ! verify_default_route_interface; then echo "Failed to determine default route interface during variable initialization." return 1 fi fi # Get the network file associated with the default route interface if not already set. # This will typically be /run/systemd/network/10-netplan-eth0.network. if [ -z "${NETWORK_FILE:-}" ]; then NETWORK_FILE="$(networkctl --json=short status "${DEFAULT_ROUTE_INTERFACE}" 2>/dev/null | jq -r '.NetworkFile')" if ! verify_network_file; then echo "Failed to determine network file during variable initialization." return 1 fi fi # Get the network drop-in directory. # This will typically be /run/systemd/network/10-netplan-eth0.network.d if [ -z "${NETWORK_DROPIN_DIR:-}" ]; then NETWORK_DROPIN_DIR="/run/systemd/network/${NETWORK_FILE##*/}.d" fi # Set the network drop-in file path if not already set. # 70-localdns.conf file is written later in disable_dhcp_use_clusterlistener function. # disable_dhcp_use_clusterlistener function is called after the cleanup function. if [ -z "${NETWORK_DROPIN_FILE:-}" ]; then NETWORK_DROPIN_FILE="${NETWORK_DROPIN_DIR}/70-localdns.conf" fi return 0 } # Start localdns service. start_localdns() { echo "Starting localdns: ${COREDNS_COMMAND}." rm -f "${LOCALDNS_PID_FILE}" # '&' is needed to put the command in the background. # This is because systemd will wait for the process to exit before continuing. # This is not what we want, as we want to run in the background. # The PID file will be created by coredns when it starts. # The PID file is used to track the process ID of coredns. # This is needed to send a SIGINT to coredns when we want to stop it. ${COREDNS_COMMAND} & # Wait until the PID file is created. local elapsed=0 while [ ! -f "${LOCALDNS_PID_FILE}" ]; do sleep 1 elapsed=$((elapsed + 1)) if [ "$elapsed" -ge "$START_LOCALDNS_TIMEOUT" ]; then echo "Timed out waiting for CoreDNS to create PID file at ${LOCALDNS_PID_FILE}." return 1 fi done COREDNS_PID="$(cat ${LOCALDNS_PID_FILE})" echo "Localdns PID is ${COREDNS_PID}." return 0 } # Wait for localdns to be ready to serve traffic. wait_for_localdns_ready() { local maxattempts=$1 local timeout_duration=$2 declare -i attempts=0 local starttime=$(date +%s) echo "Waiting for localdns to start and be able to serve traffic." until [ "$("${CURL_COMMAND[@]}")" = "OK" ]; do if [ $attempts -ge $maxattempts ]; then echo "Localdns failed to come online after $maxattempts attempts." return 1 fi # Check for timeout based on elapsed time. currenttime=$(date +%s) elapsedtime=$((currenttime - starttime)) if [ $elapsedtime -ge $timeout_duration ]; then echo "Localdns failed to come online after $timeout_duration seconds (timeout)." return 1 fi sleep 1 ((attempts++)) done echo "Localdns is online and ready to serve traffic." return 0 } # Sentinel file: presence means "this node may already have the hosts-plugin annotation, # so reconcile it on startup." Created after a successful annotation set, removed after # a successful annotation removal. Avoids unnecessary kubeconfig waits on nodes that # never used the feature. LOCALDNS_HOSTS_PLUGIN_ANNOTATION_MARKER="${LOCALDNS_HOSTS_PLUGIN_ANNOTATION_MARKER:-/opt/azure/containers/localdns-hosts-plugin-annotation.present}" # Set node annotation to indicate hosts plugin is in use if the hosts file has contents. annotate_node_with_hosts_plugin_status() { # Check if the running localdns corefile actually contains the hosts plugin block. # This is the ground truth - we check the actual corefile being used by the service, # not just what was selected during CSE, in case the file was modified or regenerated. local corefile_path="${UPDATED_LOCALDNS_CORE_FILE:-/opt/azure/containers/localdns/updated.localdns.corefile}" local should_annotate=false if [ ! -f "${corefile_path}" ]; then echo "Localdns corefile not found at ${corefile_path}, skipping annotation." return 0 fi # Determine whether hosts plugin is active based on corefile and hosts file state if grep -q "hosts /etc/localdns/hosts" "${corefile_path}"; then # Additionally verify that the hosts file exists and has content local hosts_file="${LOCALDNS_HOSTS_FILE:-/etc/localdns/hosts}" if [ -f "${hosts_file}" ] && grep -qE '^[0-9a-fA-F.:]+[[:space:]]+[[:alnum:]]' "${hosts_file}"; then echo "Localdns is using hosts plugin and hosts file has $(grep -cE '^[0-9a-fA-F.:]+[[:space:]]+[[:alnum:]]' "${hosts_file}" 2>/dev/null || echo 0) entries." should_annotate=true else echo "Corefile has hosts plugin block but hosts file is missing or empty, skipping annotation." return 0 fi else echo "Localdns corefile does not contain hosts plugin block." fi # Only proceed if we have the necessary kubectl binary and configuration if [ ! -x /opt/bin/kubectl ]; then echo "kubectl binary not found at /opt/bin/kubectl, skipping annotation." return 0 fi local kubeconfig="${KUBECONFIG:-/var/lib/kubelet/kubeconfig}" # Wait for kubelet to finish TLS bootstrapping and create the kubeconfig file # This is necessary because localdns starts in basePrep(), before kubelet starts in nodePrep() local wait_count=0 local max_wait="${KUBECONFIG_WAIT_ATTEMPTS:-60}" # Default: wait up to 3 minutes (60 * 3 seconds), but configurable for testing while [ ! -f "${kubeconfig}" ]; do if [ $wait_count -ge $max_wait ]; then echo "Timeout waiting for kubeconfig at ${kubeconfig} after ${max_wait} attempts, skipping annotation." return 0 fi echo "Waiting for TLS bootstrapping to complete (attempt $((wait_count + 1))/${max_wait})..." sleep 3 wait_count=$((wait_count + 1)) done echo "Kubeconfig found at ${kubeconfig}" # Get node name local node_name node_name=$(hostname) if [ -z "${node_name}" ]; then echo "Cannot get node name, skipping annotation." return 0 fi # Azure cloud provider assigns node name as the lower case of the hostname node_name=$(echo "$node_name" | tr '[:upper:]' '[:lower:]') # Wait for node to be registered in the cluster # The kubeconfig exists but the node might not be registered yet echo "Waiting for node ${node_name} to be registered in the cluster..." local node_wait_count=0 local max_node_wait="${NODE_REGISTRATION_WAIT_ATTEMPTS:-30}" # Default: wait up to 90 seconds (30 * 3 seconds) while [ $node_wait_count -lt $max_node_wait ]; do if /opt/bin/kubectl --kubeconfig "${kubeconfig}" get node "${node_name}" >/dev/null 2>&1; then echo "Node ${node_name} is registered in the cluster." break fi echo "Waiting for node registration (attempt $((node_wait_count + 1))/${max_node_wait})..." sleep 3 node_wait_count=$((node_wait_count + 1)) done # Check if we timed out waiting for node registration if [ $node_wait_count -ge $max_node_wait ]; then echo "Timeout waiting for node ${node_name} to be registered after ${max_node_wait} attempts, skipping annotation." return 0 fi # Set or remove annotation based on hosts plugin state if [ "${should_annotate}" = "true" ]; then echo "Setting annotation to indicate hosts plugin is in use for node ${node_name}." if /opt/bin/kubectl --kubeconfig "${kubeconfig}" annotate --overwrite node "${node_name}" kubernetes.azure.com/localdns-hosts-plugin=enabled; then echo "Successfully set hosts plugin annotation." touch "${LOCALDNS_HOSTS_PLUGIN_ANNOTATION_MARKER}" else echo "Warning: Failed to set hosts plugin annotation (this is non-fatal)." fi else # Remove stale annotation after rollback to avoid advertising hosts plugin as enabled # when it's no longer in use. The - suffix tells kubectl to remove the annotation. echo "Removing hosts plugin annotation for node ${node_name} (hosts plugin not active)." if /opt/bin/kubectl --kubeconfig "${kubeconfig}" annotate --overwrite node "${node_name}" kubernetes.azure.com/localdns-hosts-plugin-; then echo "Successfully removed hosts plugin annotation." rm -f "${LOCALDNS_HOSTS_PLUGIN_ANNOTATION_MARKER}" else # Keep marker so the next restart retries removal echo "Warning: Failed to remove hosts plugin annotation (this is non-fatal, annotation may not have existed)." fi fi return 0 } # Add iptables rules to skip conntrack for DNS traffic to localdns. add_iptable_rules_to_skip_conntrack_from_pods(){ # Check if the localdns interface already exists and delete it. if ip link show localdns >/dev/null 2>&1; then echo "Interface localdns already exists, deleting it." ip link delete localdns fi ip link add name localdns type dummy ip link set up dev localdns ip addr add ${LOCALDNS_NODE_LISTENER_IP}/32 dev localdns ip addr add ${LOCALDNS_CLUSTER_LISTENER_IP}/32 dev localdns # Add IPtables rules that skip conntrack for DNS connections coming from pods. echo "Adding iptables rules to skip conntrack for queries to localdns." for RULE in "${IPTABLES_RULES[@]}"; do eval "${IPTABLES}" -A "${RULE}" done } # Wait for localdns IP to be removed from resolv.conf after networkctl reload. # Arguments: # $1: max_wait_seconds - Maximum time to wait for the change (default: 5). wait_for_localdns_removed_from_resolv_conf() { local max_wait_seconds="${1:-5}" local sleep_interval=0.25 local max_iterations=$((max_wait_seconds * 4)) # 4 iterations per second with 0.25s sleep local iteration=0 echo "Waiting for localdns (${LOCALDNS_NODE_LISTENER_IP}) to be removed from resolv.conf..." while [ "$iteration" -lt "$max_iterations" ]; do local current_dns current_dns=$(awk '/^nameserver/ {print $2}' "$RESOLV_CONF" 2>/dev/null | paste -sd' ') # Use word boundary matching (-w) with fixed string (-F) to avoid partial IP matches. if ! grep -qwF "$LOCALDNS_NODE_LISTENER_IP" <<< "$current_dns"; then echo "DNS configuration refreshed successfully. Current DNS: ${current_dns}" return 0 fi sleep $sleep_interval iteration=$((iteration + 1)) done echo "Timed out waiting for localdns to be removed from resolv.conf after ${max_wait_seconds} seconds." echo "Current DNS: $(awk '/^nameserver/ {print $2}' "$RESOLV_CONF" 2>/dev/null | paste -sd' ')" return 1 } # Disable DNS provided by DHCP and point the system at localdns. disable_dhcp_use_clusterlistener() { mkdir -p "${NETWORK_DROPIN_DIR}" # verify that the network drop-in directory was created successfully. verify_network_dropin_dir || return 1 cat > "${NETWORK_DROPIN_FILE}" </dev/null; then echo "Successfully removed existing localdns iptables rule from $chain chain (rule $rule_num)." else echo "Failed to remove existing localdns iptables rule from $chain chain (rule $rule_num)." failure_occurred=true fi done done if [ "$failure_occurred" = true ]; then return 1 fi else echo "No existing localdns iptables rules found." fi # Revert DNS configuration and network reload. echo "Removing network drop-in file ${NETWORK_DROPIN_FILE}." rm -f "$NETWORK_DROPIN_FILE" if [ "$?" -ne 0 ]; then echo "Failed to remove network drop-in file ${NETWORK_DROPIN_FILE}." return 1 fi echo "Successfully removed network drop-in file." echo "Attempt to reload network configuration." eval "$NETWORKCTL_RELOAD_CMD" if [ "$?" -ne 0 ]; then echo "Failed to reload network after removing the DNS configuration." return 1 fi echo "Reloading network configuration succeeded." return 0 } # Cleanup function to remove localdns related configurations. cleanup_localdns_configs() { # Disable error handling so that we don't get into a recursive loop. set +e # Kill orphaned background annotation process if still running. if [ -n "${ANNOTATION_PID:-}" ] && kill -0 "${ANNOTATION_PID}" 2>/dev/null; then echo "Killing background annotation process (PID: ${ANNOTATION_PID})" kill "${ANNOTATION_PID}" 2>/dev/null || true fi # Remove iptables rules and revert DNS configuration cleanup_iptables_and_dns || return 1 # Trigger localdns shutdown, if running. if [ ! -z "${COREDNS_PID:-}" ]; then # Check if the process exists by using `ps`. if ps -p "${COREDNS_PID}" >/dev/null 2>&1; then if [ "${LOCALDNS_SHUTDOWN_DELAY}" -gt 0 ]; then # Wait after removing iptables rules and DNS configuration so that we can let connections transition. echo "Sleeping ${LOCALDNS_SHUTDOWN_DELAY} seconds to allow connections to terminate." sleep "${LOCALDNS_SHUTDOWN_DELAY}" fi echo "Sending SIGINT to localdns and waiting for it to terminate." # Send SIGINT to localdns to trigger a graceful shutdown. kill -SIGINT "${COREDNS_PID}" kill_status=$? if [ "$kill_status" -eq 0 ]; then echo "Successfully sent SIGINT to localdns." else echo "Failed to send SIGINT to localdns. Exit status: $kill_status." return 1 fi # Wait for the process to terminate. if wait "${COREDNS_PID}"; then echo "Localdns terminated successfully." else echo "Localdns failed to terminate properly." return 1 fi fi fi # Delete the dummy interface if present. if ip link show dev localdns >/dev/null 2>&1; then echo "Removing localdns dummy interface." ip link del name localdns if [ "$?" -eq 0 ]; then echo "Successfully removed localdns dummy interface." else echo "Failed to remove localdns dummy interface." return 1 fi fi # Indicate successful cleanup. echo "Successfully cleanup localdns related configurations." return 0 } # Export resource metrics (CPU, memory, status) to a .prom file for the metrics exporter. # Reads CPU and memory directly from the cgroup v2 filesystem to avoid the # systemctl → D-Bus → systemd overhead. The exporter (DynamicUser=yes) cannot # query systemd over D-Bus, so we write a world-readable .prom file from the # root-privileged localdns.sh — the same pattern used for forward_ips.prom. # This function is called once at startup and periodically from the watchdog loop. # Failures are non-fatal to avoid affecting DNS service availability. export_resource_metrics() { local resources_prom_file="${LOCALDNS_SCRIPT_PATH}/resources.prom" local raw_cpu raw_mem cpu_sec mem_bytes service_status tmp # Determine service status by checking if the CoreDNS child process is alive. # Note: systemctl is-active would always return "active" here because this function # runs inside localdns.sh which IS the localdns.service ExecStart process. if [ -n "${COREDNS_PID:-}" ] && kill -0 "$COREDNS_PID" 2>/dev/null; then service_status="active" else service_status="inactive" fi # Read resource values directly from cgroup v2 filesystem (all supported distros use cgroupv2). # This avoids the systemctl → D-Bus → systemd → cgroup round-trip. # localdns.service runs in localdns.slice (see localdns.service Slice= directive). local cgroup_dir="${LOCALDNS_CGROUP_DIR}" raw_cpu=$(awk '/^usage_usec/ {print $2}' "$cgroup_dir/cpu.stat" 2>/dev/null || echo "0") raw_mem=$(cat "$cgroup_dir/memory.current" 2>/dev/null || echo "0") # Convert CPU microseconds → seconds (%.9f preserves sub-microsecond precision for rate() calculations) # Memory is already in bytes from cgroup — expose as-is per Prometheus base-unit convention cpu_sec=$(awk -v val="$raw_cpu" 'BEGIN {printf "%.9f", val / 1000000}') mem_bytes="$raw_mem" # Write Prometheus metrics to temp file, then atomically rename if ! tmp="$(mktemp "${resources_prom_file}.XXXXXX")"; then echo "WARNING: Failed to create temp file for ${resources_prom_file}. Resource metrics export skipped." return 0 elif ! { echo "# HELP localdns_service_status CoreDNS process status (1=active, 0=inactive)" echo "# TYPE localdns_service_status gauge" if [ "$service_status" = "active" ]; then echo "localdns_service_status{status=\"active\"} 1" else echo "localdns_service_status{status=\"inactive\"} 0" fi echo "# HELP localdns_memory_usage_bytes Current memory usage in bytes" echo "# TYPE localdns_memory_usage_bytes gauge" echo "localdns_memory_usage_bytes $mem_bytes" echo "# HELP localdns_cpu_usage_seconds_total Total CPU time consumed in Seconds" echo "# TYPE localdns_cpu_usage_seconds_total counter" echo "localdns_cpu_usage_seconds_total $cpu_sec" echo "# HELP localdns_metrics_last_update_timestamp_seconds Unix timestamp of last metrics generation" echo "# TYPE localdns_metrics_last_update_timestamp_seconds gauge" echo "localdns_metrics_last_update_timestamp_seconds $(date +%s)" } > "$tmp"; then echo "WARNING: Failed to write resource metrics to temp file $tmp. Export skipped." rm -f "$tmp" return 0 elif ! chmod 0644 "$tmp"; then echo "WARNING: Failed to set permissions on temp file $tmp. Export skipped." rm -f "$tmp" return 0 elif ! mv -f "$tmp" "${resources_prom_file}"; then echo "WARNING: Failed to move temp file to ${resources_prom_file}. Export skipped." rm -f "$tmp" return 0 fi return 0 } # Start the localdns watchdog. # This function is used to check the health of localdns and restart it if necessary. # It uses systemd-notify to send a watchdog ping to the systemd service manager. # The watchdog interval is set in the systemd unit file and is passed to the script via the WATCHDOG_USEC environment variable. # The health check is a DNS request to the localdns service IPs. # The health check is run at 20% of the WATCHDOG_USEC interval. # If the health check fails, the script will exit and systemd will restart the service. start_localdns_watchdog() { if [ -n "${NOTIFY_SOCKET:-}" ] && [ -n "${WATCHDOG_USEC:-}" ]; then # Health check at 20% of WATCHDOG_USEC; this means that we should check. # five times in every watchdog interval, and thus need to fail five checks to get restarted. HEALTH_CHECK_INTERVAL=$((${WATCHDOG_USEC:-5000000} * 20 / 100 / 1000000)) echo "Starting watchdog loop at ${HEALTH_CHECK_INTERVAL} second intervals." # Sliding window failure detection: 10 failures in 10 minutes (600 seconds). # This catches intermittent but frequent failures that might not be consecutive. max_sliding_window_failures=10 sliding_window_duration_in_seconds=600 sliding_window_failure_count=0 sliding_window_start_time=0 # If health check failed 5 consecutive times or failed 10 times in a 10 minute sliding window, watchdog restarts the systemd unit. while true; do health_check_passed=true if [ "$("${CURL_COMMAND[@]}")" != "OK" ]; then echo "Health check failed: HTTP ready endpoint not responding." health_check_passed=false fi if ! dig +short +timeout=${DNS_HEALTH_CHECK_TIMEOUT} +tries=${DNS_HEALTH_CHECK_TRIES} health-check.localdns.local @${LOCALDNS_NODE_LISTENER_IP} >/dev/null 2>&1; then echo "Health check failed: DNS query to ${LOCALDNS_NODE_LISTENER_IP} failed." health_check_passed=false fi if ! dig +short +timeout=${DNS_HEALTH_CHECK_TIMEOUT} +tries=${DNS_HEALTH_CHECK_TRIES} health-check.localdns.local @${LOCALDNS_CLUSTER_LISTENER_IP} >/dev/null 2>&1; then echo "Health check failed: DNS query to ${LOCALDNS_CLUSTER_LISTENER_IP} failed." health_check_passed=false fi if [ "$health_check_passed" = true ]; then systemd-notify WATCHDOG=1 else echo "localdns health check failed - will be restarted if failures persist." current_time=$(date +%s) # If this is the first failure or window has expired, start a new window if [ "$sliding_window_start_time" -eq 0 ] || [ $((current_time - sliding_window_start_time)) -gt "$sliding_window_duration_in_seconds" ]; then sliding_window_start_time=$current_time sliding_window_failure_count=1 else sliding_window_failure_count=$((sliding_window_failure_count + 1)) fi # Check if sliding window threshold is exceeded if [ "$sliding_window_failure_count" -ge "$max_sliding_window_failures" ]; then echo "max sliding window failures (${max_sliding_window_failures} in ${sliding_window_duration_in_seconds}s) reached. Triggering restart." systemd-notify WATCHDOG=trigger exit $ERR_LOCALDNS_FAIL fi fi # Update resource metrics .prom file for the exporter (best-effort, non-fatal) export_resource_metrics sleep "${HEALTH_CHECK_INTERVAL}" done else # No watchdog configured — write metrics once then wait for CoreDNS to exit export_resource_metrics wait "${COREDNS_PID}" fi } # Selects the appropriate corefile variant based on feature flags. # Reads globals from the localdns environment file: # LOCALDNS_COREFILE_BASE — base corefile (no hosts plugin) # LOCALDNS_COREFILE_WITH_HOSTS — corefile with hosts plugin # SHOULD_ENABLE_HOSTS_PLUGIN — whether hosts plugin is enabled # # Selection logic: # 1. If both BASE and WITH_HOSTS are available, select based on SHOULD_ENABLE_HOSTS_PLUGIN. # The WITH_HOSTS corefile includes `hosts /etc/localdns/hosts { reload 5s; fallthrough }`, # so CoreDNS will start immediately and hot-reload the hosts file when it's populated # by aks-localdns-hosts-setup (no need to poll/wait here). # 2. If only BASE is available, use it (no dynamic selection). # 3. If nothing is available, return failure (caller handles error). # # Echoes the selected base64-encoded corefile to stdout. # All diagnostic messages go to stderr. select_localdns_corefile() { # Case 1: Both corefile variants available — feature-flag-based selection if [ -n "${LOCALDNS_COREFILE_WITH_HOSTS:-}" ] && \ [ -n "${LOCALDNS_COREFILE_BASE:-}" ]; then echo "Both corefile variants available, selecting based on feature flag..." >&2 echo "LocalDNS corefile selection: SHOULD_ENABLE_HOSTS_PLUGIN=${SHOULD_ENABLE_HOSTS_PLUGIN:-}" >&2 if [ "${SHOULD_ENABLE_HOSTS_PLUGIN:-}" = "true" ]; then # Defensive guard: if the hosts file doesn't exist on disk, # enableAKSLocalDNSHostsSetup may have bailed early (e.g. LOCALDNS_CRITICAL_FQDNS # was empty, or touch failed) without creating the file, while generateLocalDNSFiles # still wrote SHOULD_ENABLE_HOSTS_PLUGIN=true to the env file. Fall back to BASE # corefile to avoid CoreDNS referencing a missing hosts file. local hosts_file="${LOCALDNS_HOSTS_FILE:-/etc/localdns/hosts}" if [ ! -f "${hosts_file}" ]; then echo "SHOULD_ENABLE_HOSTS_PLUGIN=true but ${hosts_file} is missing — falling back to BASE corefile" >&2 echo "${LOCALDNS_COREFILE_BASE}" return 0 fi echo "Hosts plugin is enabled, using corefile with hosts plugin (reload will pick up hosts file when populated)" >&2 echo "${LOCALDNS_COREFILE_WITH_HOSTS}" return 0 else echo "Hosts plugin is not enabled, using corefile without hosts plugin" >&2 echo "${LOCALDNS_COREFILE_BASE}" return 0 fi fi # Case 2: Only BASE available — no dynamic selection if [ -n "${LOCALDNS_COREFILE_BASE:-}" ]; then echo "Using LOCALDNS_COREFILE_BASE (no dynamic selection)" >&2 echo "${LOCALDNS_COREFILE_BASE}" return 0 fi # Case 2.5: Legacy fallback — support older CSE versions that only set # LOCALDNS_BASE64_ENCODED_COREFILE (new VHD + old CSE compatibility). if [ -n "${LOCALDNS_BASE64_ENCODED_COREFILE:-}" ]; then echo "Using legacy LOCALDNS_BASE64_ENCODED_COREFILE" >&2 echo "${LOCALDNS_BASE64_ENCODED_COREFILE}" return 0 fi # Case 3: Nothing available — signal failure so callers don't proceed with empty corefile echo "No corefile variants available in environment." >&2 return 1 } ${__SOURCED__:+return} # --------------------------------------- Main Execution starts here -------------------------------------------------- # Regenerate corefile on every startup to enable dynamic variant selection. # --------------------------------------------------------------------------------------------------------------------- # This allows switching between WITH_HOSTS and BASE corefile variants based on feature flags. # When the hosts plugin is enabled, the WITH_HOSTS corefile uses `reload 5s` so CoreDNS # will hot-reload the hosts file when aks-localdns-hosts-setup populates it — no polling needed. regenerate_localdns_corefile || exit $ERR_LOCALDNS_COREFILE_NOTFOUND # Verify localdns required files exists. # --------------------------------------------------------------------------------------------------------------------- # Verify that generated corefile exists and is not empty. verify_localdns_corefile || exit $ERR_LOCALDNS_COREFILE_NOTFOUND # Verify that slice file exists and is not empty. verify_localdns_slicefile || exit $ERR_LOCALDNS_SLICEFILE_NOTFOUND # Verify that coredns binary is cached in VHD and is executable. # Coredns binary is extracted from cached coredns image and pre-installed in the VHD - # /opt/azure/containers/localdns/binary/coredns. verify_localdns_binary || exit $ERR_LOCALDNS_BINARY_ERR # Set required network environment variables. # --------------------------------------------------------------------------------------------------------------------- initialize_network_variables || exit $ERR_LOCALDNS_FAIL # Clean up any existing iptables rules and DNS configuration before starting. # --------------------------------------------------------------------------------------------------------------------- cleanup_iptables_and_dns || exit $ERR_LOCALDNS_FAIL # During startup, wait for the DNS configuration to be fully refreshed. # This ensures systemd-resolved has removed localdns from resolv.conf before we read upstream DNS servers. # The wait is necessary because networkctl reload is async - there's a delay before systemd-resolved # updates /run/systemd/resolve/resolv.conf. The next step (replace_azurednsip_in_corefile) reads # resolv.conf to get upstream DNS servers. Without this wait, we might still see 169.254.10.10 # (localdns IP) as a nameserver, which would create a circular dependency in the corefile. # Note: the shutdown path does not need this wait because it doesn't read from resolv.conf afterward - # it just cleans up and exits, so systemd-resolved can complete the update asynchronously. if ! wait_for_localdns_removed_from_resolv_conf 5; then echo "Error: DNS configuration was not refreshed within timeout." exit $ERR_LOCALDNS_FAIL fi # Replace AzureDNSIP in corefile with VNET DNS ServerIPs. # --------------------------------------------------------------------------------------------------------------------- replace_azurednsip_in_corefile || exit $ERR_LOCALDNS_FAIL # Build IPtable rules. # --------------------------------------------------------------------------------------------------------------------- IPTABLES='iptables -w -t raw -m comment --comment "localdns: skip conntrack"' IPTABLES_RULES=() build_localdns_iptable_rules # Setup traps to trigger cleanup_localdns_configs if anything goes wrong. # --------------------------------------------------------------------------------------------------------------------- # cleanup_localdns_configs function will be run on script exit/crash to revert config. # Ensure cleanup runs before exiting on an error. trap 'echo "Error occurred. Cleaning up..."; cleanup_localdns_configs; exit $ERR_LOCALDNS_FAIL' ABRT ERR INT PIPE # Always cleanup when exiting. trap 'echo "Executing cleanup function."; cleanup_localdns_configs || echo "Cleanup failed with error code: $ERR_LOCALDNS_FAIL."' EXIT # Configure interface listening on Node listener and cluster listener IPs. # -------------------------------------------------------------------------------------------------------------------- # Create a dummy interface listening on the link-local IP and cluster DNS service IP. echo "Setting up localdns dummy interface with IPs ${LOCALDNS_NODE_LISTENER_IP} and ${LOCALDNS_CLUSTER_LISTENER_IP}." add_iptable_rules_to_skip_conntrack_from_pods # Start localdns service. # -------------------------------------------------------------------------------------------------------------------- COREDNS_COMMAND="${COREDNS_BINARY_PATH} -conf ${UPDATED_LOCALDNS_CORE_FILE} -pidfile ${LOCALDNS_PID_FILE}" if [ -n "${SYSTEMD_EXEC_PID:-}" ]; then # We're running in systemd, so pass the coredns output via systemd-cat. COREDNS_COMMAND="systemd-cat --identifier=localdns-coredns --stderr-priority=3 -- ${COREDNS_COMMAND}" fi # Start localdns. start_localdns || exit $ERR_LOCALDNS_FAIL # Wait to direct traffic to localdns until it's ready. wait_for_localdns_ready 60 60 || exit $ERR_LOCALDNS_FAIL # Disable DNS from DHCP and point the system at localdns. # -------------------------------------------------------------------------------------------------------------------- echo "Updating network DNS configuration to point to localdns via ${NETWORK_DROPIN_FILE}." disable_dhcp_use_clusterlistener || exit $ERR_LOCALDNS_FAIL echo "Startup complete - serving node and pod DNS traffic." # Export initial resource metrics so the exporter has data before the first watchdog tick. export_resource_metrics # Set node annotation to indicate hosts plugin is in use (if applicable). # -------------------------------------------------------------------------------------------------------------------- # Only run when hosts plugin is currently enabled or was previously enabled (marker exists). # The marker avoids unnecessary kubeconfig waits on nodes that never used the feature. if [ "${SHOULD_ENABLE_HOSTS_PLUGIN:-false}" = "true" ] || [ -f "${LOCALDNS_HOSTS_PLUGIN_ANNOTATION_MARKER}" ]; then annotate_node_with_hosts_plugin_status & ANNOTATION_PID=$! echo "Started hosts plugin annotation in background (PID: ${ANNOTATION_PID})" else echo "Hosts plugin not enabled and no prior annotation marker found, skipping annotation." fi # Systemd notify: send ready if service is Type=notify. # -------------------------------------------------------------------------------------------------------------------- if [ -n "${NOTIFY_SOCKET:-}" ]; then systemd-notify --ready fi # Systemd watchdog: send pings so we get restarted if we go unhealthy. # -------------------------------------------------------------------------------------------------------------------- # If the watchdog is defined, we check status and pass success to systemd. start_localdns_watchdog # The cleanup function is called on exit, so it will be run after the # wait ends (which will be when a signal is sent or localdns crashes) or the script receives a terminal signal. # --------------------------------------- Main execution ends here --------------------------------------------------- # end of line