A little script to roll a cluster, useful if managing your own.

#!/bin/bash
# roll-cluster 

# get list of nodes
NODES=(`kubectl get nodes -o jsonpath='{.items[*].status.addresses[1].address}'`)
for NODE in "${NODES[@]}"
do
  echo ""
  echo "[$NODE]"
  TMP=`kubectl get node $NODE | grep Ready | grep -v 'control-plane'`
  if [ "$TMP" == "" ]; then
    continue
  fi

  echo "- draining $NODE"
  kubectl drain $NODE --ignore-daemonsets --delete-emptydir-data
  echo "- sending reboot command, enter password if prompted by sudo"
  ssh root@$NODE reboot
  echo "- waiting for node go down (no ping)"
  while [ "$(ping $NODE -c 4 | grep packet | grep -c ' 0\% packet loss')" == 1 ]; do
    sleep 1;
    done
  echo "- wait for node to show as ' Ready'"
  while (true); do
    a=`kubectl get node $NODE | grep " Ready"`
    if [ "$a" != "" ];
      then break;
      else sleep 1;
    fi
    done;
  echo "- uncordon $NODE"
  kubectl uncordon $NODE
done

Posted in Development, Infrastructure, Kubernetes.

Leave a Reply