#!/bin/bash

# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

COMMON="/usr/sbin/so-common"

if ! [ -f $COMMON ]; then
	echo "$COMMON not found."
	echo "This script requires Security Onion Elastic Stack (16.04.5.2 ISO) or later."
	exit
fi

source $COMMON

SKIP=0
TEMPLATE=""
DATE=`date +%Y.%m.%d`
SUFFIX="reindexed-$DATE"

function full_reindex() {
        # Check for all indices that match specified pattern
        for i in $(curl -s localhost:9200/$INDEX | jq 'keys[]' | sed 's/"//g' | sed 's/,//g'); do
                # Double-check the keys we received back from JQ to see if index exists
                INDEX_EXISTS=`curl -s localhost:9200/$i`
                if `echo "$INDEX_EXISTS" | grep -q "index_not_found_exception"`; then
                echo "Found no indices matching that pattern ('$i').  Exiting..."
                exit 0
                fi

                echo
		DUMMY=`echo $i-$SUFFIX`
		

                # Create dummy index
                echo "Creating index: $DUMMY for $i ..."
                curl -s -XPUT "localhost:9200/$DUMMY?pretty" #> /dev/null 2&>1
                # -H 'Content-Type: application/json' -d"$TEMPLATE" > /dev/null 2&>1
                echo

                curl -s -XPUT "localhost:9200/$DUMMY/_settings" -H 'Content-Type: application/json' -d'{"index" : {"refresh_interval" : -1 }}'


                # Re-index original index to dummy index
                echo "Re-indexing index: $i => $DUMMY ..."
                curl -s -XPOST 'localhost:9200/_reindex?pretty' -H 'Content-Type: application/json' -d" { \"source\": { \"index\": \"$i\" }, \"dest\": { \"index\": \"$DUMMY\" } } " #> /dev/null 2&>1
                echo

                # We need to refresh the dummy index so it is available for copying immediately
                curl -s -XPOST "localhost:9200/$DUMMY/_refresh"

                # Delete original index
                echo "Deleting index: $i ..."
                curl -s -XDELETE localhost:9200/$i #> /dev/null 2&>1
                echo

                # Create index with original name
                echo "Creating new index: $i ..."
                curl -s -XPUT "localhost:9200/$i?pretty" #> /dev/null 2&>1
                #-H 'Content-Type: application/json' -d"$TEMPLATE" > /dev/null 2&>1
                echo

                #echo "Waiting $WAIT for index to refresh..."
                #sleep $WAIT

                # Re-index dummy index to original name
                echo "Re-indexing index: $DUMMY => $i ..."
                curl -s -XPOST 'localhost:9200/_reindex?pretty' -H 'Content-Type: application/json' -d" { \"source\": { \"index\": \"$DUMMY\" }, \"dest\": { \"index\": \"$i\" } } " #> /dev/null 2&>1
		echo

                # Delete dummy index
                echo "Deleting index: $DUMMY ..."
                curl -s -XDELETE localhost:9200/$DUMMY #> /dev/null 2&>1
                echo

                # Refresh our new index so we can see that everything is copied
                curl -s -XPOST "localhost:9200/$i/_refresh"
        done
}

function alias_reindex(){
	for i in $(curl -s localhost:9200/$INDEX | jq 'keys[]' | sed 's/"//g' | sed 's/,//g'); do
                # Double-check the keys we received back from JQ to see if index exists
                INDEX_EXISTS=`curl -s localhost:9200/$i`
                if `echo "$INDEX_EXISTS" | grep -q "index_not_found_exception"`; then
                echo "Found no indices matching that pattern ('$i').  Exiting..."
                exit 0
                fi

                echo
		DUMMY=`echo $i-$SUFFIX`

                # Create dummy index
                echo "Creating index: $DUMMY for $i ..."
                curl -s -XPUT "localhost:9200/$DUMMY?pretty" #> /dev/null 2&>1
                # -H 'Content-Type: application/json' -d"$TEMPLATE" > /dev/null 2&>1
                echo

                curl -s -XPUT "localhost:9200/$DUMMY/_settings" -H 'Content-Type: application/json' -d'{"index" : {"refresh_interval" : -1 }}'


                # Re-index original index to dummy index
                echo "Re-indexing index: $i => $DUMMY ..."
                curl -s -XPOST 'localhost:9200/_reindex?pretty' -H 'Content-Type: application/json' -d" { \"source\": { \"index\": \"$i\" }, \"dest\": { \"index\": \"$DUMMY\" } } " #> /dev/null 2&>1
                echo

                # We need to refresh the dummy index so it is available for copying immediately
                curl -s -XPOST "localhost:9200/$DUMMY/_refresh"
   
                # Delete original index
                echo "Deleting index: $i ..."
                curl -s -XDELETE localhost:9200/$i #> /dev/null 2&>1
                echo

		# Alias newly created index to original index name
		curl -XPOST "localhost:9200/_aliases" -H 'Content-Type: application/json' -d" {    \"actions\" : [        { \"add\" : { \"index\" : \"$DUMMY\", \"alias\" : \"$i\" } }    ]}"


	done	
}

#WAIT="3s"

#########################################
# Options
#########################################
usage()
{
cat <<EOF
Security Onion Elastic Re-Index
  Options:
  -f         Full reindex (maintaining original index names)
	     (The default option is to use a suffix of "reindexed", and alias the new
	     index to the original index name, in an effort to be a quicker re-indexing 
	     process.)
  -h         This message
  -i         Index(es) to re-index (wildcards need to be quoted)
  -s         Suffix to use when creating new indices 
	     Example: a value of "_reindexed" would result in an index 	     
	     name like "logstash-ossec-2019-01.14_reindexed"

EXAMPLES:
	     Re-index with aliases --> sudo ./so-elastic-reindex -i "logstash-*"
	     Re-index with original index name --> sudo ./so-elastic-reindex -f -i "logstash-*"  
EOF
}

while getopts 'i:s:hyf' OPTION; do
  case "$OPTION" in
    f)
      FULL_REINDEX=1
      ;;
    i)
      INDEX="$OPTARG"
      SKIP=1
      ;;
    h)
      echo
      usage
      echo
      exit 0
      ;;
    s)
      SUFFIX="$OPTARG"
      ;;
  esac
done

if [ $SKIP -ne 1 ]; then

	header "Security Onion Elastic Re-Index"
	echo
	echo "This script will stop Logstash while it re-indexes indices specified by the provided pattern."
	echo
	echo "If you would like to proceed, please type "YES" and hit ENTER."
	echo

	# Read user input
	read INPUT
	if [ "$INPUT" != "YES" ] ; then exit 0; fi

        echo
	echo "Please provide an index pattern for the indices you would like to re-index: [ENTER]"
        echo
	echo "Ex: logstash-bro-*"
	echo
	read INDEX
fi

# Stop Logstash
echo
echo "Stopping Logstash to prevent new documents from being written to indices..."
docker stop so-logstash

if docker ps | grep so-logstash; then
        echo
        echo "Logstash still running -- please stop before continuing..."
        exit 1
else

    # Check to see if we are still processing records
    echo
    echo "We will wait for 30 seconds, then check ES to see if records are still being processed..."
    sleep 30s
    PROCESSING="true"
    while [ "$PROCESSING" = "true" ]; do
        if curl -s localhost:9200/_cat/tasks | grep 'indices:data/write/bulk'; then
            echo "Records still being processed...will wait another 10 secs..."
            sleep 10s
        else
            echo "No records left to process!  Continuing..."
	    PROCESSING="false"
        fi
    done
    
    # See if we are performing full or alias-based re-indexing
    if [[ "$FULL_REINDEX" -eq 1 ]]; then
    	full_reindex
    else
	alias_reindex
    fi
fi

# Start Logstash
echo "Starting Logstash..."
echo
so-logstash-start

echo "Operation complete!"
echo
echo "If you were previously receiving mapping conflicts in Kibana, please try refreshing your index pattern(s) to confirm you are no longer having issues."
echo