#!/bin/ksh
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r714 src/43haes/usr/sbin/cluster/utilities/clautover.sh 1.13 
#  
# Licensed Materials - Property of IBM 
#  
# Restricted Materials of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2004,2009 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)01	1.13 src/43haes/usr/sbin/cluster/utilities/clautover.sh, hacmp.utils, 61haes_r714 9/25/09 14:19:20
 
###############################################################################
#
#   COMPONENT_NAME: UTILS
#
#   FUNCTIONS:  
#		 usage
#                isMigrtion
#                isClverNode
#		 logInfo
#		 wallInfo
#                main
#
###############################################################################
#
#  Name:  clautover
#
#       The purpose of clautover feature is to run cluster verification
#	and store the results to be availalbe for the user.
#	Originally called automatically once in 24 hours by Automatic Cluster
#	Configuration Monitoring feature (if enabled by user).
#
#
#  Arguments
#	None: 	Basic functionality - run clverify and report results.
#		1) Checks for cluster being in migration state. 
#		2) Checks for the local node name and if this node is to
#		   execute cluster verification (by defualt or set by a user).
#		   By default, the first cluster node in alphabetical order runs 
#		   cluster verification.
#		3) If the local node is to run cluster verification, execut
#		   clver command and store the result (error count).
#		4) Create the message to be logged in the log file. Call
#		   clautover remotely wil 2 parameters (message and time on
#		   the local node) to login the information.
#		5) If clver detected any errors, call additional alerts 
#		   (broadcast and event).
#	1 argument: Message to broadcast.
#		    Walls the passed message and exits.
#	2 arguments: Message to login and date/Time on the remote node
#    		     Logs the message with passed date/Time from the remote node.
#
#  Called From
#        clcomd, HACMPtimersvc
#	 clautover (self-calls remotely).
#
#  Calls To
#        clnodename
#	 getLocalNodename
#	 dsh
#	 clautover (self-calls)
#
#  Returns
#       0       Success
#       1       Failure
#
#  Globals Affected
#       None
#
#  Environment:
#
###############################################################################

###############################################################################
#  Name:  isMigration
#
#  This routine checks if the cluster is in migration state, and it looks for both 
#  HAS->ES migration and mixed ES versions on cluster nodes.
#
#  Arguments:    none
#  Usage:        isMigration
#  Returns:      0 if the all nodes have the same version on ES, non-zero otherwise
#  Environment:
#
###############################################################################
isMigration()
{
    typeset PS4_FUNC="isMigration"
    #Check for mixed versions of ES products
    /usr/es/sbin/cluster/utilities/clmixver > /dev/null
    if [ $? -ne 0 ]
    then
	return 1
    fi
    #Check for HAS->ES migration
    /usr/es/sbin/cluster/utilities/cl_migcheck "ANY"
    return $?
}

###############################################################################
#  Name:  isClverNode
#
#  This routine checks for the node to run cluster verification and compares it with 
#  the name of the local node.
#
#  Arguments:    Current node name
#  Usage:	 isClverNode
#  Returns:      0 the local node is the node to execute Cluster Verification
#  Environment:
#
###############################################################################
isClverNode()
{
    typeset PS4_FUNC="isClverNode"
    #For debugging purposes, verify the usage of this function
    if [ -z "$1" ]
    then
	echo $PROGNAME: Incorrect usage of function isClverNode. No parameters are specified.
	return 1
    fi
	
    #Look for the user-specified node name
    CLVER_NODENAME=$(odmget HACMPcluster | grep clvernodename | cut -d'"' -f2)

    #If there is no user-specified node, get the first cluster node in 
    #alphabetical order
    if [ -z "$CLVER_NODENAME" ]
    then
	/usr/es/sbin/cluster/utilities/clnodename | sort | read CLVER_NODENAME
    fi
    
    #Compare the name of the current node with the name of the node to execute clver
    if [ "$1" = "$CLVER_NODENAME" ]
    then
	return 0
    else
	return 1
    fi
}

###############################################################################
#  Name:  logInfo
#
#  This routine is to store in log the information aabout the cluster verification 
#  results received from the remote node.
#
#  Arguments:   String to log, date/time from remote node, spaces replaces with '_'
#  Usage:       logInfo Message
#  Returns:     Nothing
#
###############################################################################
logInfo()
{
    typeset PS4_FUNC="logInfo"
    #Restore the message to log
    logMessage="$(print $1 | /usr/es/sbin/cluster/cspoc/cldecodearg)"
    DATETIME="$(print $2 | /usr/es/sbin/cluster/cspoc/cldecodearg)"

    #Getting the log file information
    logFile=clutils.log

    path=$(odmget -q name=$logFile HACMPlogs | grep value | cut -d'"' -f2)
    LOGFILENAME=$(echo $path/$logFile)

    echo $DATETIME: $logMessage >> $LOGFILENAME
}


###############################################################################
#  Name:  wallInfo
#
#  This routine is to display the information aabout the cluster verification
#  results received from the remote node.
#
#  Arguments:   String to display.
#  Usage:       wallInfo Message
#  Returns:     Nothing
#
###############################################################################
wallInfo()
{
    typeset PS4_FUNC="wallInfo"
    #Restore message to wall to its original 
    wallMessage="$(print $1 | /usr/es/sbin/cluster/cspoc/cldecodearg)"
    wall $wallMessage
}



###############################################################################
#
# Main Starts Here
#
###############################################################################

# Initialize variables

PROGNAME=$(basename ${0})
OUTPUTFILE="/var/hacmp/log/autoclstrcfgmonitor.out"
export ODMDIR=/etc/es/objrepos
#export PATH="$($(dirname ${0})/cl_get_path all)"
[[ "$VERBOSE_LOGGING" = "high" ]] && set -x
[[ "$VERBOSE_LOGGING" = "high" ]] && version='1.13'

# The ability to enable clver debugging was added with the syntax "clautover -D".
# To avoid affecting existing code, if $1 is -D, we'll save it and stip it off.
DEBUG=""
if [ "$1" = "-D" ]
then
    DEBUG=$1
    shift
fi

# If the command has been executed with two parameters, call the function to
# wall the information and exit
if [ $# -eq 2 ]
then
    logInfo $1 $2
    exit $?
fi

# If the command has been executed with one parameter, call the function to
# log on the information and exit
if [[ $# -eq 1 ]] && [[ -n "$1" ]];
then
    wallInfo $1
    exit $?
fi

#Do not execute clverify when the cluster is in migrated state
isMigration
RC=$?

if [[ $RC = 1 ]]
then
    #This message is here only for maintance purposes
    echo "Cluster is in migration state"
    exit 1
fi

if [[ $RC = 3 ]]
then
    #This message is here only for maintance purposes
    echo "A communications error was encountered while trying to get the VRMF from a remote node\nPlease make sure clcomd is running\n"
    exit 1
fi

if [[ $RC = 4 ]]
then
    #This message is here only for maintance purposes
    echo "An internal error occurred while parsing the VRMF from a remote node\n"
fi

#Get the name of the current node
LOCAL_NODE=$( /usr/es/sbin/cluster/utilities/get_local_nodename )

#Check if this node is assigned to execute cluster verification
isClverNode $LOCAL_NODE
if [ $? -ne 0 ]
then
    exit 1
fi

#Clean the output file before storing the output there
if [ -f "$OUTPUTFILE" ]
then
    rm $OUTPUTFILE
fi

#Run cluster verification and store the result (error count) in a variable
/usr/es/sbin/cluster/diag/clconfig -v '-tr' -e 100 -R $OUTPUTFILE -V 'normal' -O $DEBUG

#Store results in a variable. This is the number of errors detected
RESULT=$?

#Create a message to be written to the log file
logMessage=$(dspmsg scripts.cat 9681 "Cluster verification is complete on node $LOCAL_NODE with $RESULT errors detected.\n" $LOCAL_NODE $RESULT)

#Get local date/time info
DATETIME=$(date)

#Code passed strings so they can be passed via dsh as 1 argument each
logMessage="$(print $logMessage | /usr/es/sbin/cluster/cspoc/clencodearg)"
DATETIME=$(print $DATETIME | /usr/es/sbin/cluster/cspoc/clencodearg)

#Create a command to be remotely executed - call clatover with 
#2 parameter (message and date/time on the local node)
COMMAND=$(echo /usr/es/sbin/cluster/utilities/clautover $logMessage $DATETIME)

#Get the list of the nodes and list them separated via ','
NODELIST=$(/usr/es/sbin/cluster/utilities/clnodename)
NODELIST=$(echo $NODELIST | tr ' ' ',')

#Execute the same script on all the cluster node in parallel providing 
#clver exit result and current date/time as parameters.
/usr/es/sbin/cluster/cspoc/dsh -w $NODELIST $COMMAND

#If clverify fail, additional alerts will be required.
if [ $RESULT -ne 0 ]
then
    #Call the notification event with message #1
    /usr/es/sbin/cluster/events/utils/cl_RMupdate cluster_notify 1 >/dev/null 2> /dev/null

    #Generate the message to send out accross the cluster
    wallMessage=$(dspmsg scripts.cat 9680 "\
Cluster Verification detected cluster configuration errors on \n\
node $LOCAL_NODE.  Detailed clverify output is available in file\n\
$OUTPUTFILE on node $LOCAL_NODE.\n" $LOCAL_NODE $OUTPUTFILE)

    #Code the message to pass it as a single argument
    wallMessage="$(print $wallMessage | /usr/es/sbin/cluster/cspoc/clencodearg)"

    #Generate the command to remotely wall the notification message
    COMMAND=$(echo /usr/es/sbin/cluster/utilities/clautover $wallMessage)
    #Call the remote command on all the available cluster nodes
    /usr/es/sbin/cluster/cspoc/dsh -w $NODELIST $COMMAND
fi

exit 0