Rebalancing script

[yoho]

To rebalance data, splunk only offers you to rebalance one single index or all the indexes but there is no option to provide a list of them. I've created the following shell script to do this and I hope it will help the community.

To use it:

• Create an empty directory and copy the script there
• Change the parameters at the beginning of the script to correspond to your platform
• Create a file named ~/.creds containing MY_CREDS="admin:password" (you can replace admin and password with any user account with admin privileges)
• Create a file in the same directory as the script named indexes.conf with a list of indexes you want to rebalance (one per line)
• Launch the script and provide a value for the timeout in minutes ("-m" switch;  e.g. "./rebalance.sh -m 2880")
• Tip: if you access your manager node using ssh, consider using "screen" to keep your session open for a long period of time.

#!/bin/sh
HOST=my_manager_node.example.com
PORT=8089
source $HOME/.creds
CURL_OPTS="-su ${MY_CREDS} --write-out HTTPSTATUS:%{http_code}"
INDEXES_FILE="indexes.conf"
CONFIG_CMD="/usr/bin/curl ${CURL_OPTS} https://${HOST}:${PORT}/services/cluster/config/config -d rebalance_threshold=0.9"
/cluster/manager/control/control/rebalance_buckets -d action=start"
START_CMD="/usr/bin/curl ${CURL_OPTS} https://${HOST}:${PORT}/services/cluster/manager/control/control/rebalance_buckets -d action=start"
configuration endpoint or by normal endpoint with action=start
MAXRUNTIME_OPT="-d max_time_in_min="
INDEX_OPT="-d index="
STOP_CMD="/usr/bin/curl ${CURL_OPTS} https://${HOST}:${PORT}/services/cluster/manager/control/control/rebalance_buckets -d action=stop"
STATUS_CMD="/usr/bin/curl ${CURL_OPTS} https://${HOST}:${PORT}/services/cluster/manager/control/control/rebalance_buckets -d action=status"
LOG="rebalance.log"
MAXRUNTIME="1"

while getopts ":m:" opt; do
  case $opt in
    m) MAXRUNTIME=${OPTARG} ;;
     echo "Missing value for argument -$OPTARG"
       exit 1
       ;;
    ?) echo "Unknown option -$OPTARG"
       exit 1
       ;;
  esac
done

if [[ "$OPTIND" -ne "3" ]]; then
  echo "Please specify timeout in minute with -m"
  exit
fi

echo -n "Starting $0: " >>$LOG
echo $(date) >>$LOG

[[ -f $INDEXES_FILE ]] || exit
echo "Configuring rebalancing"
echo "Configuring rebalancing" >>$LOG
EPOCH=$(date +"%s")
MAX_EPOCH=$(( EPOCH + ( 60 * MAXRUNTIME ) ))
echo "Will end at EPOCH: $MAX_EPOCH" >>$LOG
HTTP_RESPONSE=$($CONFIG_CMD)
HTTP_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
if [[ ! $HTTP_STATUS == "200" ]]; then
  echo "HTTP status: $HTTP_STATUS"
  echo "HTTP body: $HTTP_BODY"
  exit
fi

RUNNING=0
for INDEX in `cat $INDEXES_FILE`; do
  EPOCH=$(date +"%s")
  MINS_REMAINING=$(( ( MAX_EPOCH - EPOCH ) / 60 ))
  if [[ "$MINS_REMAINING" -le "0" ]]; then
    echo "Timout reached"
    echo "Timout reached" >>$LOG
    exit
  fi
  echo "Rebalancing $INDEX"
  echo "Rebalancing $INDEX" >>$LOG
  echo "Remaining time: $MINS_REMAINING" >>$LOG
  echo $(date) >>$LOG
  #HTTP_RESPONSE=$(${START_CMD} ${INDEX_OPT}${INDEX})
  HTTP_RESPONSE=$(${START_CMD} ${INDEX_OPT}${INDEX} ${MAXRUNTIME_OPT}${MINS_REMAINING})
  #HTTP_RESPONSE=200
  HTTP_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
  HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
  echo "HTTP status: $HTTP_STATUS" >>$LOG
  echo "HTTP body: $HTTP_BODY" >>$LOG
  if [[ ! $HTTP_STATUS == "200" ]]; then
    echo "HTTP status: $HTTP_STATUS"
    echo "HTTP body: $HTTP_BODY"
    exit
  fi
  RUNNING=1
  WAS_RUNNING=0
  sleep 1
  while [[ $RUNNING == 1 ]]; do
    HTTP_RESPONSE=$($STATUS_CMD)
    HTTP_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
    HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
    if [[ ! $HTTP_STATUS == "200" ]]; then
      echo "HTTP status: $HTTP_STATUS"
      echo "HTTP body: $HTTP_BODY"
      exit
    fi
    echo "$HTTP_BODY" | grep "Data rebalance is not running" >/dev/null
    if [ $? -eq 0 ]; then
      RUNNING=0
    else
      WAS_RUNNING=1
      RUNNING=1
      echo -n "."
      sleep 1
    fi
  done
  if [[ $WAS_RUNNING == 1 ]]; then
    # We need a CR after the last echo -n "."
    echo
  fi
done