#!/bin/bash

if [ "$(docker ps | grep openvswitch)" ]; then
    OVS_OFCTL="docker exec openvswitch ovs-ofctl"
    OVS_APPCTL="docker exec openvswitch ovs-appctl"
else
    OVS_OFCTL="ovs-ofctl"
    OVS_APPCTL="ovs-appctl"
fi

# echoes args to stderr and exits
die () {
    echo "$*" 1>&2
    exit 1
}

# echoes the command provided as $@ and then runs it
echo_and_eval () {
    echo "> $*"
    echo ""
    eval "$@"
}

# runs the command provided as $@, and either returns silently with
# status 0 or else logs an error message with the command's output
try_eval () {
    tmpfile=`mktemp`
    if ! eval "$@" >& $tmpfile; then
	status=1
	echo "ERROR: Could not run '$*':"
	sed -e 's/^/  /' $tmpfile
	echo ""
    else
	status=0
    fi
    rm -f $tmpfile
    return $status
}

# The environment may contain sensitive information like passwords or private keys
filter_env () {
    awk '/ env:$/ { indent = index($0, "e"); skipping = 1; next } !skipping { print; } skipping { ch = substr($0, indent, 1); if (ch != " " && ch != "-") { skipping = 0; print } }'
}

log_service () {
    logpath=$1
    service=$2
    start_args=$3

    echo_and_eval  journalctl -u $service                         &> $logpath/journal-$service
    echo_and_eval  systemctl show $service                        &> $logpath/systemctl-show-$service

    config_file=$(get_config_path_from_service "$start_args" "$service")
    if [ -f "$config_file" ]; then
        echo_and_eval  cat $config_file                           &> $logpath/CONFIG-$service
    fi
}

log_system () {
    logpath=$1

    echo_and_eval  journalctl --boot                                  &> $logpath/journal-full
    echo_and_eval  nmcli --nocheck -f all dev show                    &> $logpath/nmcli-dev
    echo_and_eval  nmcli --nocheck -f all con show                    &> $logpath/nmcli-con
    echo_and_eval  head -1000 /etc/sysconfig/network-scripts/ifcfg-*  &> $logpath/ifcfg
    echo_and_eval  ip addr show                                       &> $logpath/addresses
    echo_and_eval  ip route show                                      &> $logpath/routes
    echo_and_eval  ip -s neighbor show                                &> $logpath/arp
    echo_and_eval  iptables-save                                      &> $logpath/iptables
    echo_and_eval  cat /etc/hosts                                     &> $logpath/hosts
    echo_and_eval  cat /etc/resolv.conf                               &> $logpath/resolv.conf
    echo_and_eval  lsmod                                              &> $logpath/modules
    echo_and_eval  sysctl -a                                          &> $logpath/sysctl

    echo_and_eval  oc version                                         &> $logpath/version
    echo                                                             &>> $logpath/version
    echo_and_eval  docker version                                    &>> $logpath/version
    echo                                                             &>> $logpath/version
    echo_and_eval  cat /etc/system-release-cpe                       &>> $logpath/version
}

do_master () {
    kubelet_port=10250
    if ! nodes=$(oc get nodes --template '{{range .items}}{{.spec.externalID}} {{end}}'); then
	if [ -z "$KUBECONFIG" -o ! -f "$KUBECONFIG" ]; then
	    die "KUBECONFIG is unset or incorrect"
	else
	    die "Could not get list of nodes"
	fi
    fi

    logmaster=$logdir/master
    mkdir -p $logmaster

    # Grab master service logs and config files
    if [ -n "$aos_master_service" ]; then
        log_service $logmaster $aos_master_service "master"
    fi
    if [ -n "$aos_master_controllers_service" ]; then
        log_service $logmaster $aos_master_controllers_service "master controllers"
    fi
    if [ -n "$aos_master_api_service" ]; then
        log_service $logmaster $aos_master_api_service "master api"
    fi

    # Log the generic system stuff
    log_system $logmaster

    # And the master specific stuff
    echo_and_eval  oc get nodes                      -o yaml               &> $logmaster/nodes
    echo_and_eval  oc get pods      --all-namespaces -o yaml  | filter_env &> $logmaster/pods
    echo_and_eval  oc get services  --all-namespaces -o yaml               &> $logmaster/services
    echo_and_eval  oc get endpoints --all-namespaces -o yaml               &> $logmaster/endpoints
    echo_and_eval  oc get routes    --all-namespaces -o yaml               &> $logmaster/aos_routes
    echo_and_eval  oc get clusternetwork             -o yaml               &> $logmaster/clusternetwork
    echo_and_eval  oc get hostsubnets                -o yaml               &> $logmaster/hostsubnets
    echo_and_eval  oc get netnamespaces              -o yaml               &> $logmaster/netnamespaces

    for node in $nodes; do
	reg_ip=$(oc get node $node --template '{{range .status.addresses}}{{if eq .type "InternalIP"}}{{.address}}{{end}}{{end}}')
	if [ -z "$reg_ip" ]; then
	    echo "Node $node: no IP address in OpenShift"
	    continue
	fi

	resolv_ip=$(getent ahostsv4 $node | awk '/STREAM/ { print $1; exit; }')

	if [ "$reg_ip" != "$resolv_ip" ]; then
	    echo "Node $node: the IP in OpenShift ($reg_ip) does not match DNS/hosts ($resolv_ip)"
	fi

	try_eval curl $node:$kubelet_port
    done

    # Outputs a list of nodes in the form "nodename IP"
    oc get nodes --template '{{range .items}}{{$name := .metadata.name}}{{range .status.addresses}}{{if eq .type "InternalIP"}}{{$name}} {{.address}}{{"\n"}}{{end}}{{end}}{{end}}' > $logdir/meta/nodeinfo

    # Outputs a list of pods in the form "minion-1 172.17.0.1 mypod namespace 10.1.0.2 e4f1d61b"
    oc get pods --all-namespaces --template '{{range .items}}{{if .status.containerStatuses}}{{if (index .status.containerStatuses 0).ready}}{{if not .spec.hostNetwork}}{{.spec.nodeName}} {{.status.hostIP}} {{.metadata.name}} {{.metadata.namespace}} {{.status.podIP}} {{printf "%.21s" (index .status.containerStatuses 0).containerID}}{{"\n"}}{{end}}{{end}}{{end}}{{end}}' | sed -e 's|docker://||' > $logdir/meta/podinfo

    # Outputs a list of services in the form "myservice namespace 172.30.0.99 tcp 5454"
    oc get services --all-namespaces --template '{{range .items}}{{if ne .spec.clusterIP "None"}}{{.metadata.name}} {{.metadata.namespace}} {{.spec.clusterIP}} {{(index .spec.ports 0).protocol}} {{(index .spec.ports 0).port}}{{"\n"}}{{end}}{{end}}' | sed -e 's/ TCP / tcp /g' -e 's/ UDP / udp /g' > $logdir/meta/serviceinfo
}

get_port_for_addr () {
    addr=$1
    # The nw_src line works with all current installs. The nw_dst line is needed for
    # older installs using the original single-tenant rules.
    sed -n -e "s/.*in_port=\([0-9]*\).*nw_src=${addr}.*/\1/p" \
	   -e "s/.*nw_dst=${addr}.*output://p" \
           $lognode/flows | head -1
}

get_vnid_for_addr () {
    addr=$1
    # On multitenant, the sed will match, and output something like "xd1", which we prefix
    # with "0" to get "0xd1". On non-multitenant, the sed won't match, and outputs nothing,
    # which we prefix with "0" to get "0". So either way, $base_pod_vnid is correct.
    echo 0$(sed -ne "s/.*reg0=0\(x[^,]*\),.*nw_dst=${addr}.*/\1/p" $lognode/flows | head -1)
}

do_pod_to_pod_connectivity_check () {
    where=$1
    namespace=$2
    base_pod_name=$3
    base_pod_addr=$4
    base_pod_pid=$5
    base_pod_port=$6
    base_pod_vnid=$7
    base_pod_ether=$8
    other_pod_name=$9
    other_pod_addr=${10}
    other_pod_nodeaddr=${11}

    echo $where pod, $namespace namespace: | tr '[a-z]' '[A-Z]'
    echo ""

    other_pod_port=$(get_port_for_addr $other_pod_addr)
    if [ -n "$other_pod_port" ]; then
	other_pod_vnid=$(get_vnid_for_addr $other_pod_addr)
	in_spec="in_port=${other_pod_port}"
    else
	case $namespace in
	    default)
		other_pod_vnid=0
		;;
	    same)
		other_pod_vnid=$base_pod_vnid
		;;
	    different)
		# VNIDs 1-10 are currently unused, so this is always different from $base_pod_vnid
		other_pod_vnid=6
		;;
	esac
	in_spec="in_port=1,tun_src=${other_pod_nodeaddr},tun_id=${other_pod_vnid}"
    fi

    echo "$base_pod_name -> $other_pod_name"
    echo_and_eval $OVS_APPCTL ofproto/trace br0 "in_port=${base_pod_port},reg0=${base_pod_vnid},ip,nw_src=${base_pod_addr},nw_dst=${other_pod_addr}"
    echo ""
    echo "$other_pod_name -> $base_pod_name"
    echo_and_eval $OVS_APPCTL ofproto/trace br0 "${in_spec},ip,nw_src=${other_pod_addr},nw_dst=${base_pod_addr},dl_dst=${base_pod_ether}"
    echo ""

    if nsenter -n -t $base_pod_pid -- ping -c 1 -W 2 $other_pod_addr  &> /dev/null; then
	echo "ping $other_pod_addr  ->  success"
    else
	echo "ping $other_pod_addr  ->  failed"
    fi

    echo ""
    echo ""
}

do_pod_external_connectivity_check () {
    base_pod_name=$1
    base_pod_addr=$2
    base_pod_pid=$3
    base_pod_port=$4
    base_pod_vnid=$5
    base_pod_ether=$6

    echo "EXTERNAL TRAFFIC:"
    echo ""
    echo "$base_pod_name -> example.com"
    # This address is from a range which is reserved for documentation examples
    # (RFC 5737) and not allowed to be used in private networks, so it should be
    # guaranteed to only match the default route.
    echo_and_eval $OVS_APPCTL ofproto/trace br0 "in_port=${base_pod_port},reg0=${base_pod_vnid},ip,nw_src=${base_pod_addr},nw_dst=198.51.100.1"
    echo ""
    echo "example.com -> $base_pod_name"
    echo_and_eval $OVS_APPCTL ofproto/trace br0 "in_port=2,ip,nw_src=198.51.100.1,nw_dst=${base_pod_addr},dl_dst=${base_pod_ether}"
    echo ""

    if nsenter -n -t $base_pod_pid -- ping -c 1 -W 2 www.redhat.com  &> /dev/null; then
	echo "ping www.redhat.com  ->  success"
    else
	echo "ping www.redhat.com  ->  failed"
    fi
}

do_pod_service_connectivity_check () {
    namespace=$1
    base_pod_name=$2
    base_pod_addr=$3
    base_pod_pid=$4
    base_pod_port=$5
    base_pod_vnid=$6
    base_pod_ether=$7
    service_name=$8
    service_addr=$9
    service_proto=${10}
    service_port=${11}

    echo service, $namespace namespace: | tr '[a-z]' '[A-Z]'
    echo ""

    echo "$base_pod_name -> $service_name"
    echo_and_eval $OVS_APPCTL ofproto/trace br0 "in_port=${base_pod_port},reg0=${base_pod_vnid},${service_proto},nw_src=${base_pod_addr},nw_dst=${service_addr},${service_proto}_dst=${service_port}"
    echo ""
    echo "$service_name -> $base_pod_name"
    echo_and_eval $OVS_APPCTL ofproto/trace br0 "in_port=2,${service_proto},nw_src=${service_addr},nw_dst=${base_pod_addr},dl_dst=${base_pod_ether}"
    echo ""

    # In bash, redirecting to /dev/tcp/HOST/PORT or /dev/udp/HOST/PORT opens a connection
    # to that HOST:PORT. Use this to test connectivity to the service; we can't use ping
    # like in the pod connectivity check because only connections to the correct port
    # get redirected by the iptables rules.
    if nsenter -n -t $base_pod_pid -- timeout 1 bash -c "echo -n '' > /dev/${service_proto}/${service_addr}/${service_port} 2>/dev/null"; then
	echo "connect ${service_addr}:${service_port}  ->  success"
    else
	echo "connect ${service_addr}:${service_port}  ->  failed"
    fi

    echo ""
    echo ""
}

get_config_path_from_service() {
    # 'node', 'master', 'master api', 'master controllers'
    service_type=$1
    # 'atomic-openshift-node.service', 'atomic-openshift-master-controllers.service', etc
    service_name=$2
    config=$(ps wwaux | grep -v grep | sed -ne "s/.*openshift start ${service_type} --.*config=\([^ ]*\.yaml\).*/\1/p")
    if [ -z "$config" ]; then
	config=$(systemctl show -p ExecStart $service_name | sed -ne 's/.*--config=\([^ ]*\).*/\1/p')
	if [ "$config" == "\${CONFIG_FILE}" ]; then
	    varfile=$(systemctl show $service_name | grep EnvironmentFile | sed -ne 's/EnvironmentFile=\([^ ]*\).*/\1/p')
	    if [ -f "$varfile" ]; then
	        config=$(cat $varfile | sed -ne 's/CONFIG_FILE=//p')
	    fi
	fi
    fi
    echo "$config"
}

do_node () {
    config_file=$(get_config_path_from_service "node" ${aos_node_service})
    if [ -z "$config_file" ]; then
	die "Could not find node-config.yaml from 'ps' or 'systemctl show'"
    fi
    node=$(sed -ne 's/^nodeName: //p' $config_file)
    if [ -z "$node" ]; then
	die "Could not find node name in $config_file"
    fi

    lognode=$logdir/nodes/$node
    mkdir -p $lognode

    # Grab node service logs and config file
    log_service $lognode $aos_node_service "node"

    # Log the generic system stuff
    log_system $lognode

    # Log some node-only information
    echo_and_eval  brctl show                              &> $lognode/bridges
    echo_and_eval  docker ps -a                            &> $lognode/docker-ps
    echo_and_eval  $OVS_OFCTL -O OpenFlow13 dump-flows br0  &> $lognode/flows
    echo_and_eval  $OVS_OFCTL -O OpenFlow13 show br0        &> $lognode/ovs-show
    echo_and_eval  tc qdisc show                           &> $lognode/tc-qdisc
    echo_and_eval  tc class show                           &> $lognode/tc-class
    echo_and_eval  tc filter show                          &> $lognode/tc-filter
    echo_and_eval  systemctl cat docker.service            &> $lognode/docker-unit-file

    # Work out the docker env file we install (if we do) and log its contents
    envfile=`systemctl cat docker.service | grep EnvironmentFile.\*openshift-sdn | awk -F=- '\
{print $2}'`
    if [ -n "$envfile" ]; then
        echo_and_eval  cat "$envfile"                      &> $lognode/docker-network-file
    fi


    # Iterate over all pods on this node, and log some data about them.
    # Remember the name, address, namespace, and pid of the first pod we find on
    # this node which is not in the default namespace
    base_pod_addr=
    while read pod_node pod_nodeaddr pod_name pod_ns pod_addr pod_id; do
	if [ "$pod_node" != "$node" ]; then
	    continue
	fi

	logpod=$lognode/pods/$pod_name
	mkdir -p $logpod

	pid=$(docker inspect -f '{{.State.Pid}}' $pod_id)
	if [ -z "$pid" ]; then
	    echo "$node:$pod_name: could not find pid of $pod"
	    continue
	fi

	echo_and_eval nsenter -n -t $pid -- ip addr  show  &> $logpod/addresses
	echo_and_eval nsenter -n -t $pid -- ip route show  &> $logpod/routes

	# If we haven't found a local pod yet, or if we have, but it's
	# in the default namespace, then make this the new base pod.
	if [ -z "$base_pod_addr" -o "$base_pod_ns" = "default" ]; then
	    base_pod_addr=$pod_addr
	    base_pod_ns=$pod_ns
	    base_pod_name=$pod_name
	    base_pod_pid=$pid
	fi
    done < $logdir/meta/podinfo

    if [ -z "$base_pod_addr" ]; then
	echo "No pods on $node, so no connectivity tests"
	return
    fi

    base_pod_port=$(get_port_for_addr $base_pod_addr)
    if [ -z "$base_pod_port" ]; then
	echo "Could not find port for ${base_pod_addr}!"
	return
    fi
    base_pod_vnid=$(get_vnid_for_addr $base_pod_addr)
    if [ -z "$base_pod_vnid" ]; then
	echo "Could not find VNID for ${base_pod_addr}!"
	return
    fi
    base_pod_ether=$(nsenter -n -t $base_pod_pid -- ip a | sed -ne "s/.*link.ether \([^ ]*\) .*/\1/p")
    if [ -z "$base_pod_ether" ]; then
	echo "Could not find MAC address for ${base_pod_addr}!"
	return
    fi

    unset did_local_default   did_local_same   did_local_different
    unset did_remote_default  did_remote_same  did_remote_different
    unset did_service_default did_service_same did_service_different
    if [ "$base_pod_ns" = "default" ]; then
	# These would be redundant with the "default" tests
	did_local_same=1
	did_remote_same=1
	did_service_same=1
    fi

    # Now find other pods of various types to test connectivity against
    touch $lognode/pod-connectivity
    while read pod_node pod_nodeaddr pod_name pod_ns pod_addr pod_id; do
	if [ "$pod_addr" = "$base_pod_addr" ]; then
	    continue
	fi

	if [ "$pod_node" = "$node" ]; then
	    where=local
	else
	    where=remote
	fi
	if [ "$pod_ns" = "default" ]; then
	    namespace=default
	elif [ "$pod_ns" = "$base_pod_ns" ]; then
	    namespace=same
	else
	    namespace=different
	fi

	if [ "$(eval echo \$did_${where}_${namespace})" = 1 ]; then
	    continue
	fi

	do_pod_to_pod_connectivity_check $where $namespace \
					 $base_pod_name $base_pod_addr \
					 $base_pod_pid $base_pod_port \
					 $base_pod_vnid $base_pod_ether \
					 $pod_name $pod_addr $pod_nodeaddr \
					 &>> $lognode/pod-connectivity
	eval did_${where}_${namespace}=1
    done < $logdir/meta/podinfo

    do_pod_external_connectivity_check $base_pod_name $base_pod_addr \
				       $base_pod_pid $base_pod_port \
				       $base_pod_vnid $base_pod_ether \
				       &>> $lognode/pod-connectivity

    # And now for services
    touch $lognode/service-connectivity
    while read service_name service_ns service_addr service_proto service_port; do
	if [ "$service_ns" = "default" ]; then
	    namespace=default
	elif [ "$service_ns" = "$base_pod_ns" ]; then
	    namespace=same
	else
	    namespace=different
	fi

	if [ "$(eval echo \$did_service_${namespace})" = 1 ]; then
	    continue
	fi

	do_pod_service_connectivity_check $namespace \
					  $base_pod_name $base_pod_addr \
					  $base_pod_pid $base_pod_port \
					  $base_pod_vnid $base_pod_ether \
					  $service_name $service_addr $service_proto $service_port \
					  &>> $lognode/service-connectivity
	eval did_service_${namespace}=1
    done < $logdir/meta/serviceinfo
}

run_self_via_ssh () {
    args=$1
    host=$2

    SSH_OPTS='-o StrictHostKeyChecking=no -o PasswordAuthentication=no'

    if ! try_eval ssh $SSH_OPTS root@$host /bin/true; then
	return 1
    fi

    if ! try_eval ssh $SSH_OPTS root@$host mkdir -m 0700 -p $logdir; then
	return 1
    fi

    if ! try_eval scp -pr $SSH_OPTS $logdir/meta root@$host:$logdir; then
	return 1
    fi

    ssh $SSH_OPTS root@$host $extra_env /bin/bash $logdir/meta/debug.sh $args
}

do_master_and_nodes ()
{
    master="$1"

    echo "Analyzing master"

    if [ -z "$master" ]; then
	do_master
    else
	if run_self_via_ssh --master $master < /dev/null; then
        try_eval scp $SSH_OPTS -pr root@$master:$logdir/master $logdir
        try_eval scp $SSH_OPTS -pr root@$master:$logdir/meta $logdir
    else
	    return 1
	fi
    fi

    while read name addr; do
	echo ""
	echo "Analyzing $name ($addr)"

	if ip addr show | grep -q "inet $addr/"; then
	    # Running on master which is also a node
	    cp $self $logdir/debug.sh
	    /bin/bash $logdir/debug.sh --node
	else
	    run_self_via_ssh --node $addr < /dev/null && \
		try_eval scp $SSH_OPTS -pr root@$addr:$logdir/nodes $logdir
	fi
    done < $logdir/meta/nodeinfo
}

######## Main program starts here

for systemd_dir in /etc/systemd/system /usr/lib/systemd/system; do
    for name in openshift origin atomic-openshift; do
	if [ -f $systemd_dir/$name-master.service ]; then
	    aos_master_service=$name-master.service
	fi
	if [ -f $systemd_dir/$name-master-controllers.service ]; then
	    aos_master_controllers_service=$name-master-controllers.service
	fi
	if [ -f $systemd_dir/$name-master-api.service ]; then
	    aos_master_api_service=$name-master-api.service
	fi
	if [ -f $systemd_dir/$name-node.service ]; then
	    aos_node_service=$name-node.service
	fi
    done
done

case "$1" in
    --node)
	logdir=$(dirname $0 | sed -e 's|/meta$||')
	do_node
	exit 0
	;;

    --master)
	logdir=$(dirname $0 | sed -e 's|/meta$||')
	do_master
	exit 0
	;;

    "")
	if [ -z "$aos_master_service" ] && [ -z "$aos_master_controllers_service" ]; then
	    echo "Usage:"
	    echo "  [from master]"
	    echo "    $0"
	    echo "  Gathers data on the master and then connects to each node via ssh"
	    echo ""
	    echo "  [from any other machine]"
	    echo "    $0 MASTER-NAME"
	    echo "  Connects to MASTER-NAME via ssh and then connects to each node via ssh"
	    echo ""
	    echo "  The machine you run from must be able to ssh to each other machine"
	    echo "  via ssh with no password."
	    exit 1
	fi
	;;
esac

case "$0" in
    /*)
	self=$0
	;;
    *)
	self=$(pwd)/$0
	;;
esac

logdir=$(mktemp --tmpdir -d openshift-sdn-debug-XXXXXXXXX)
mkdir $logdir/meta
cp $self $logdir/meta/debug.sh
mkdir $logdir/master
mkdir $logdir/nodes
do_master_and_nodes "$1" |& tee $logdir/log

dumpname=openshift-sdn-debug-$(date --iso-8601).tgz
(cd $logdir; tar -cf - --transform='s/^\./openshift-sdn-debug/' .) | gzip -c > $dumpname
echo ""
echo "Output is in $dumpname"