#!/bin/ksh
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
#  
#  
# Licensed Materials - Property of IBM 
#  
# (C) COPYRIGHT International Business Machines Corp. 2018,2019 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
#
# sccsid = "@(#)90 1.42 src/rsct/utils/ctsyschk.sh, common_utils, rsct_rady, rady2035a 8/14/18 06:24:12"
#
#
#ctsyschk  -TV -h -n -k -s -l
#                -C [p|m|a]			// Cluster scope - p : PeerDomain  m : Management Domain  a : All 
#                -i [4|6|a]			// interfaces  - 4 : IPv4  6 : IPv6  a : All / Any
#                -p [r|s|a]			// port - r : rmc port  s : HAGS & HATS port a -- All
#		 -U username			//username for ssh
#		 -P password			//password for ssh
#                -t [tiebreaker_disk_info]	//PVID or WWID of the disk
#                -m				// MTU size compatibility check amongst nodes
#                -n				// nodeid of all nodes to be verified for non uniqueness
#                -k				//  key exchange with remote node needs to be verified
#                -s				// rmc session to remote node - authenticated / unauthenticated
#                [node_name1 node_name2 ...] 
#
#
#
#
#
# set -x
PATH=/opt/rsct/install/bin:/opt/rsct/bin:/usr/bin:/bin:/usr/sbin:/sbin
export PATH
BASEOS=`uname`

SPACE=" "
MINUS="-"

typeset Argcount=0
typeset CLUSTER_SCOPE=''
typeset LOCAL_NODE=""
typeset UNKNOWN_SCOPEOPT=0
typeset VALID_SCOPE=0
typeset UNKNOWN_COMMOPT=0
typeset UNKNOWN_PORTOPT=0
typeset UNKNOWN_SESSOPT=0

typeset VERBOSE=0
typeset DEFAULT_COMMOPT="a"
typeset DEFAULT_PORTOPT='a'
typeset COMMOPT=${DEFAULT_COMMOPT}
typeset PORTOPT=${DEFAULT_PORTOPT}
typeset optI=0
typeset optP=0
typeset optM=0
typeset optN=0
typeset optS=0
typeset optK=0
typeset optUSR=0
typeset optPSWD=0

typeset LOCAL_IPV4="127.0."
typeset LOCAL_IPV4_2="127.0.0.1/8"
typeset LINK_LOCAL_IPV4="169.254."
typeset LINK_LOCAL_IPV6="fe80:"
typeset EMPTY_ZERO_IPADDRV4="0.0.0.0"
typeset EMPTY_ZERO_IPADDRV6="::"
typeset VALID_IFS_V4_LST=""

typeset CONN_PATH_SUCCESS=0
typeset NODE_ID_SUCCESS=0
typeset PORT_CHECK_SUCCESS=0
typeset PROCESS_OPTS_SUCCESS=0
typeset MTU_COMPAT_SUCCESS=0

typeset PD_REQ_SUCCESS=0
typeset PD_REQ_FAIL=1

typeset NO_VALID_IP_FOUND=1
typeset NO_MATCHING_SUBNET_FOUND=2
typeset PORT_ERROR=3

typeset NODE_ID_NOT_UNIQUE=5
typeset MTU_CHECK_ERROR=6
typeset REMOTE_NODE_CONN_REFUSED=7
typeset OTHER_ERROR=8


typeset NO_VALID_IPV4_FOUND=11
typeset NO_VALID_IPV4_FOUND_RMT_NODE=12
typeset NO_VALID_IPV6_FOUND=13

#Port Availability check return codes in case of errors

typeset RMC_PORT_BOUND_BY_OTHER_RC=51
typeset TS_PORT_BOUND_BY_OTHER_RC=52
typeset GS_PORT_BOUND_BY_OTHER_RC=53
typeset RMC_NOT_LISTENING_RC=55
typeset TS_NOT_LISTENING_RC=56
typeset GS_NOT_LISTENING_RC=57
typeset RMC_PORT_HELD_BY_OTHER_RC=58
typeset TS_PORT_HELD_BY_OTHER_RC=59
typeset GS_PORT_HELD_BY_OTHER_RC=60
typeset FIREWALL_CHECK_FAILED_RC=62
typeset MTU_COMPAT_CHECK_FAILED=71

typeset NO_NODE_ID_FILE=111  #this is a warning

typeset MTU_NO_OP_4_IPV6=113 #warning that user is trying to check MTU size of IPv6 network
typeset UNSUPPORTED_OS_RC=114 #warning that user is running script on unsupported OS
typeset WRONG_OPT_FOR_PORT_CHECK_RC=115 # warning that user entered unsupported port option.

typeset NO_VALID_IPV4_FOUND_STR="No valid IPv4 interfaces found"
typeset NO_VALID_IPV4_FOUND_RMT_NODE_STR="No valid IPv4 interfaces found on remote node"
typeset NO_VALID_IPV6_FOUND_STR="No valid IPv6 interfaces found"
typeset NO_VALID_IP_FOUND_STR="No valid ip connectivity found on the node"
typeset NO_MATCHING_SUBNET_FOUND_STR="There are no matching subnets found between the nodes"
typeset REMOTE_NODE_CONN_REFUSED_STR="connection refused by remote node"
typeset NODE_ID_NOT_UNIQUE_STR="Duplicate node id found on the node"
typeset NO_NODE_ID_FILE_STR="Node ID file is not found on the node"


typeset RMC_PORT_BOUND_BY_OTHER_STR="RMC Port is bound by some other process. Please check /etc/services"
typeset TS_PORT_BOUND_BY_OTHER_STR="HATS Port is bound by some other process. Please check /etc/services"
typeset GS_PORT_BOUND_BY_OTHER_STR="HAGS Port is bound by some other process. Please check /etc/services"

typeset RMC_NOT_LISTENING_STR="RMC is not listening on port $CT_SYSCHK_RMC_PORT"
typeset TS_NOT_LISTENING_STR="Topology Services is not active on port $CT_SYSCHK_TS_PORT"
typeset GS_NOT_LISTENING_STR="Group Services is not active on port $CT_SYSCHK_TS_PORT"

typeset RMC_PORT_HELD_BY_OTHER_STR="RMC Port is being used by some other process"
typeset TS_PORT_HELD_BY_OTHER_STR="HATS Port is being used by some other process"
typeset GS_PORT_HELD_BY_OTHER_STR="HAGS Port is being used by some other process"

typeset UNSUPPORTED_OS_STR="This is an unsupported OS. Supported OS are AIX and Linux."
typeset WRONG_OPT_FOR_PORT_CHECK_STR="Unsupported option for port check!! Supported options are r (RMC), s (GS,TS), a (RMC, GS and TS)."
typeset FIREWALL_CHECK_FAILED_STR="Firewall check failed. A firewall rule is blocking communication on RMC, HATS or HAGS port. Please check firewall rules."

typeset MTU_NO_OP_4_IPV6_STR="MTU test is valid only for IPv4 interfaces"
typeset MTU_COMPAT_CHECK_FAILED_STR="MTU compatibility check failed"

LOCAL_NODE=`hostname`
LOCAL_NODE_SHORT_NAME=`hostname -s`

NODE_ID_FILE="/var/ct/cfg/ct_node_id"
LSHSC="/opt/hsc/bin/lshsc"
CT_SYSCHK_RMC_PORT=657

# If HAGS and HATS ports are not set by env. vars., assign default port-numbers

if [ -z "${CT_SYSCHK_TS_PORT}" ]; then
	CT_SYSCHK_TS_PORT=12347
fi

if [ -z "${CT_SYSCHK_GS_PORT}" ]; then
	CT_SYSCHK_GS_PORT=12348
fi

if [ -z "${CT_SYSCHK_SHELL}" ]; then
	CT_SYSCHK_SHELL="ssh"
fi

# Fuctions start from here.

# Function to show usage (will be shown with -h option and in general)
function usage
{
    [[ "$CTSYSCHK_VERBOSE" == "high" ]] && set -x

    echo "Usage: ctsyschk [-U username] [-P password] [-C cluster_type ] [-i interface_type]  [-p port ] [-m] [-n] [-s session_scope ] Node_Name1 Node_Name2"
    exit_code=${OTHER_ERROR}

    exit $exit_code
}

#Function to print debug messages (will be shown with -V option)
function print_dbgmsg
{
   [[ "$CTSYSCHK_VERBOSE" == "high" ]] && set -x

   if [[ $VERBOSE == 1 ]] 
   then
       #echo ${basecmd} "($$)": "$1" 
	echo "$1" 
   fi
   if [[ -n $CTSYSCHK_LOG_FILE ]]
   then
       echo ${basecmd} "($$)": "$1" >> ${CTSYSCHK_LOG_FILE}
   fi
}


function print_errmsg
{
       if [[ -n $CTSYSCHK_ERR_FILE ]]
       then
           echo ${basecmd} "($$)" : "$1" >> ${CTSYSCHK_ERR_FILE}
       else
		# echo ${basecmd} "($$)": "$1" 
		echo "$1" 
       fi
}

############################################################
# Function: REMOTE_EXEC
# This function is intended to run command on remote node
# either via expect script or directly via remote_shell.
#
# Environment variables:
#	1. CT_SYSCHK_SHELL (default remote_shell=> ssh)
#
# Inputs: 
#	1. Nodename
#	2. Command
# 
# Output:
#	1. command's output
#
# Return-code:
#	0	: Success
#	Non-zero: Failure
############################################################

function REMOTE_EXEC {

 remote_node=$1
 shift
 cmd=$*

 if [ -z "${USERNAME}" ]; then
	username=root
 else
	username=${USERNAME}
 fi

 if [ -z "${PASSWD}" ]; then
	password=root123
 else
	password=${PASSWD}
 fi

 if [ -z "${CT_SYSCHK_SHELL}" ]; then
	remote_shell=ssh
 else 
	remote_shell=${CT_SYSCHK_SHELL}
 fi
 
 remote_exec_rc=0
 
 command -v expect 1>/dev/null
 if [[ $? -eq 0 ]];then
	REMOTE_EXEC_VIA_EXPECT ${remote_node} ${cmd}
	remote_exec_rc=$?
 else
	${remote_shell} ${remote_node} -l ${username} "export PATH=/sbin:/usr/sbin:$PATH; ${cmd}"
	remote_exec_rc=$?
 fi

 return $remote_exec_rc
}


function REMOTE_EXEC_VIA_EXPECT {

if [ -z "${USERNAME}" ]; then
	username=root
else
	username=${USERNAME}
fi

if [ -z "${PASSWD}" ]; then
	password=root123
else
	password=${PASSWD}
fi

if [ -z "${CT_SYSCHK_SHELL}" ]; then
	remote_shell=ssh
else 
	remote_shell=${CT_SYSCHK_SHELL}
fi

 remote_node=$1
 shift
 cmd=$*

expect << SSH_EXPECT
log_user 0

# Current limitations: 
# 1. CT_SYSCHK_SHELL should be set. otherwise it will take ssh by default.

set timeout 10
set output ""
set buf_captured 0
match_max 300000
# ==> Spawnning remote shell and getting output

spawn ${remote_shell} ${remote_node} -l ${username} "export PATH=/sbin:/usr/sbin:$PATH; ${cmd}"

expect {
	"(yes/no)?*" {
		send "yes\r"
		expect "*?assword:" { 
			send "${password}\r"
			}
		} 

	"*?assword: "
		{ send "${password}\r" }
	
	"*ould not resolve hostname *" {
		puts stderr "Invalid hostname. remote shell invocation failed!!\r"
		exit 1
	}

	"*onnection refused*" {
		puts stderr "Connection refused by .!!\r"
		exit 1
	}

	"${remote_node}*NOT FOUND" {
		puts stderr "Invalid hostname. rsh failed!!\r"
		exit 1
	}

	"rshd*The remote user login is not correct" {
		puts stderr "Invalid username. rsh failed!!\r"
		exit 1

	}

	"Permission denied*" {
		puts stderr "Permission denied for Host: ${remote_node}, User: ${username} .\r"
		exit 1
	}
	timeout {
		puts stderr "Timeout..."
		exit 1
	}
	eof { set output \$expect_out(buffer)
		set buf_captured 1}
}

if {\$buf_captured == 0}	{ 
	expect eof
}

# ==> By now, login process is complete. User either has got output or error.

set output \$expect_out(buffer)
puts stdout \$output 
exit 0
SSH_EXPECT

} # End of REMOTE_EXEC

################################################################################
# Function: check_firewall_on_aix_internal
# This function takes cues from a firewall rule to identify whether the rule can
# block cluster communication on AIX nodes.
# Inputs: 
#	1. Source IP Address
#	2. Source Port Comparison Operator (le/lt/ge/gt/eq/neq)
#	3. Source Port number
#	4. Destination IP Address
#	5. Destination Port Comparison Operator
#	6. Destination Port Number
#	7. Port option: r|s|a
# Output: None
# Return values: 
#      0	: IP Filter rules does not affect cluster communication
#      Non-zero	: IP Filter rules may affect cluster communication
################################################################################

function check_firewall_on_aix_internal
{

	src_ipaddr=$1 # if this is 0.0.0.0, don't use it
	src_operator=$2
	src_port_num=$3

	dest_ipaddr=$4 # if this is 0.0.0.0, don't use it
	dest_operator=$5
	dest_port_num=$6
	port_opt=$7

	firewall_check_internal_rc=0
	
	if [ $src_ipaddr = "0.0.0.0" ] && [ $dest_ipaddr = "0.0.0.0" ];then
		return $firewall_check_internal_rc
	fi

	if [[ $src_ipaddr = "0.0.0.0" ]]; then
		port_num=$dest_port_num
		operator=$dest_operator
	elif [[ $dest_ipaddr = "0.0.0.0" ]]; then
		port_num=$src_port_num
		operator=$src_operator
	fi

	if [ $src_ipaddr = "0.0.0.0" ] || [ $dest_ipaddr = "0.0.0.0" ];then
	
	  case "$port_opt" in 
	  r)
		if [ $operator = "eq" ] || [ $operator = "le" ] || [ $operator = "ge" ]; then
			if [ ${port_num} -eq ${CT_SYSCHK_RMC_PORT} ];then
				firewall_check_internal_rc=1
			fi
		elif [ $operator = "lt" ] && [ ${CT_SYSCHK_RMC_PORT} -lt ${port_num} ]; then
			firewall_check_internal_rc=2
		elif [ $operator = "gt" ] && [ ${CT_SYSCHK_RMC_PORT} -gt ${port_num} ]; then
			firewall_check_internal_rc=3
		elif [ $operator = "neq" -a ${CT_SYSCHK_RMC_PORT} -ne ${port_num} ] || [ $operator = "any" ]; then
			#neq => only 1 port allowed. any => No port allowed.
			firewall_check_internal_rc=4
		else 
			firewall_check_internal_rc=0
		fi
		;;

	  s)
		if [ $operator = "eq" ] || [ $operator = "le" ] || [ $operator = "ge" ]; then
			if [ ${port_num} -eq ${CT_SYSCHK_GS_PORT} ] || [ ${port_num} -eq ${CT_SYSCHK_TS_PORT} ];then
				firewall_check_internal_rc=1
			fi
		elif [ $operator = "lt" ] && [ ${CT_SYSCHK_GS_PORT} -lt ${port_num} || ${CT_SYSCHK_TS_PORT} -lt ${port_num} ]; then
			firewall_check_internal_rc=2
		elif [ $operator = "gt" ] && [ ${CT_SYSCHK_GS_PORT} -gt ${port_num} || ${CT_SYSCHK_TS_PORT} -gt ${port_num} ]; then
			firewall_check_internal_rc=3
		elif [ $operator = "neq" ] || [ $operator = "any" ]; then
			#neq => only 1 port allowed. any => No port allowed. 3 ports are needed for RPD (RMC, HATS, HAGS).
			firewall_check_internal_rc=4
		else 
			firewall_check_internal_rc=0
		fi
		;;

	  a)
		if [ $operator = "eq" ] || [ $operator = "le" ] || [ $operator = "ge" ];then
			if [ ${port_num} -eq ${CT_SYSCHK_RMC_PORT} ] || [ ${port_num} -eq ${CT_SYSCHK_GS_PORT} ] || [ ${port_num} -eq ${CT_SYSCHK_TS_PORT} ];then
				firewall_check_internal_rc=1
			fi
		elif [ $operator = "lt" ] &&
			[ ${CT_SYSCHK_RMC_PORT} -lt ${port_num} || ${CT_SYSCHK_GS_PORT} -lt ${port_num} || ${CT_SYSCHK_TS_PORT} -lt ${port_num} ]; then
			firewall_check_internal_rc=2
		elif [ $operator = "gt" ] &&
			${CT_SYSCHK_RMC_PORT} -gt ${port_num} || ${CT_SYSCHK_GS_PORT} -gt ${port_num} || ${CT_SYSCHK_TS_PORT} -gt ${port_num} ]; then
			firewall_check_internal_rc=3
		elif [ $operator = "neq" ] || [ $operator = "any" ]; then
			#neq => only 1 port allowed. any => No port allowed. 3 ports are needed for RPD (RMC, HATS, HAGS).
			firewall_check_internal_rc=4
		else 
			firewall_check_internal_rc=0
		fi
		;;
	  *)   firewall_check_internal_rc=$FIREWALL_CHECK_FAILED_RC
		;;
	  esac

	elif [ $src_ipaddr != "0.0.0.0" ] && [ $dest_ipaddr != "0.0.0.0" ];then

: <<'SRC_DEST_CHECK_IN_NODES'
		If there is a special firewall from a particular source IP to particular destination IP,
		we need to check first whether source and destination IP are not amongst any of the nodes' IPs.
SRC_DEST_CHECK_IN_NODES
		src_or_dest_ip_on_node=0

		for node_name in $nodes
		do
			REMOTE_EXEC ${node_name} "ifconfig -a inet|grep -w inet|tr '\n' ' '" >${TMPDIR}/IP_ADDR_OUT
			egrep -w '$src_ipaddr|$dest_ipaddr' ${TMPDIR}/IP_ADDR_OUT
			if [[ $? -eq 0 ]]; then
				src_or_dest_ip_on_node=1
				break
			fi
		done
		rm -rf ${TMPDIR}/IP_ADDR_OUT
		if [[ $src_or_dest_ip_on_node -ne 1 ]];then
			return 0
		fi

		# Do, actual test now for RMC, HAGS and HATS ports.
		#Right now, port_num has destination's port number.

		operator=$dest_operator
		port_num=$dest_port_num

		if [ $port_num -ne 0 ];then
			if [ $operator = "eq" || $operator = "le" || $operator = "ge" ] &&
				[ ${port_num} -eq ${CT_SYSCHK_RMC_PORT} || ${port_num} -eq ${CT_SYSCHK_GS_PORT} || ${port_num} -eq ${CT_SYSCHK_TS_PORT} ];then
				firewall_check_internal_rc=1
			elif [ $operator = "lt" ] &&
				[ ${CT_SYSCHK_RMC_PORT} -lt ${port_num} || ${CT_SYSCHK_GS_PORT} -lt ${port_num} || ${CT_SYSCHK_TS_PORT} -lt ${port_num} ]; then
				firewall_check_internal_rc=2
			elif [ $operator = "gt" ] &&
				${CT_SYSCHK_RMC_PORT} -gt ${port_num} || ${CT_SYSCHK_GS_PORT} -gt ${port_num} || ${CT_SYSCHK_TS_PORT} -gt ${port_num} ]; then
				firewall_check_internal_rc=3
			elif [ $operator = "neq" ] || [ $operator = "any" ]; then
				#neq => only 1 port allowed. any => No port allowed. 3 ports are needed for RPD (RMC, HATS, HAGS).
				firewall_check_internal_rc=4
			else 
				firewall_check_internal_rc=0
			fi
		fi

		#Now, check whether the source port is blocked

		operator=$src_operator
		port_num=$src_port_num

		if [[ $port_num -ne 0 ]] ;then
			if [[ $operator = "eq" || $operator = "le" || $operator = "ge" ]] && 
				[[ ${port_num} -eq ${CT_SYSCHK_RMC_PORT} || ${port_num} -eq ${CT_SYSCHK_GS_PORT} || ${port_num} -eq ${CT_SYSCHK_TS_PORT} ]];then
				firewall_check_internal_rc=1
			elif [[ $operator = "lt" ]] &&
				[[ ${CT_SYSCHK_RMC_PORT} -lt ${port_num} || ${CT_SYSCHK_GS_PORT} -lt ${port_num} || ${CT_SYSCHK_TS_PORT} -lt ${port_num} ]]; then
				firewall_check_internal_rc=2
			elif [[ $operator = "gt" ]] &&
				[[ ${CT_SYSCHK_RMC_PORT} -gt ${port_num} || ${CT_SYSCHK_GS_PORT} -gt ${port_num} || ${CT_SYSCHK_TS_PORT} -gt ${port_num} ]]; then
				firewall_check_internal_rc=3
			elif [[ $operator = "neq" ]] || [[ $operator = "any" ]]; then
				#neq => only 1 port allowed. any => No port allowed. 3 ports are needed for RPD (RMC, HATS, HAGS).
				firewall_check_internal_rc=4
			else 
				firewall_check_internal_rc=0
			fi
		fi
	else 
		firewall_check_internal_rc=0
	fi

  return $firewall_check_internal_rc
} # end of check_firewall_on_aix_internal


################################################################################
# Function: check_firewall_on_aix
# This function checks whether any source or destination ports are blocked on
# cluster nodes. This function is intended for AIX nodes.
# Inputs: 
#	1. Port option: r|s|a
# Output: None
# Return values: 
#      0	: IP Filter rules does not affect cluster communication
#      Non-zero	: IP Filter rules may affect cluster communication
################################################################################

function check_firewall_on_aix
{
 firewall_rc=0
 port_opt=$1
 # Search for IP filter rules denying communication on ports
 print_dbgmsg "Running lsfilt to check AIX firewall rules..."
 
 rm -rf ${TMPDIR}/tmp_lsfilt_file

 for each_node in $nodes
 do
	REMOTE_EXEC ${each_node} "lsfilt|grep -p deny" > ${TMPDIR}/tmp_lsfilt_file

  while IFS= read -r line
  do
        # display line or do somthing on $line
	case $line in 
		Rule*[0-9]*)		rule_num_str=`echo $line`
					
					;;
		Rule*ction*)		rule_action=`echo $line|cut -d: -f2`
					;;
		Source*Address*)	src_addr=`echo $line|cut -d: -f2`
					;;
		Destination*Address*)	dest_addr=`echo $line|cut -d: -f2`
					;;

		Source*Port*)	src_port_str=`echo $line|cut -d: -f2`
				src_operator=`echo ${src_port_str}|awk '{print $1}'`
				src_port_num=`echo ${src_port_str}|awk '{print $2}'`
				;;

		Dest*Port*)	dest_port_str=`echo $line|cut -d: -f2`
				dest_operator=`echo ${dest_port_str}|awk '{print $1}'`;
				dest_port_num=`echo ${dest_port_str}|awk '{print $2}'`;
				check_firewall_on_aix_internal $src_addr $src_operator $src_port_num $dest_addr $dest_operator $dest_port_num $port_opt
				firewall_rc=$?
				if [[ $firewall_rc -ne 0 ]]; then
					if [ $src_ipaddr != "0.0.0.0" ] || [ $dest_ipaddr != "0.0.0.0" ];then
						print_errmsg "${each_node}: Warning: Firewall of $rule_num_str is blocking RSCT comm.!!"
						return $firewall_rc
					fi
					firewall_rc=1
				fi

				;;

		Interface*)	interface=`echo $line|cut -d: -f2`
		
				;;
		*) ;;
	esac

 done < ${TMPDIR}/tmp_lsfilt_file
 rm -rf ${TMPDIR}/tmp_lsfilt_file
 done
  rm -rf tmp_lsfilt_file

return $firewall_rc
} # end of check_firewall_on_aix


function validate_ipv6_addr
{
ipaddr_v6=$1
set invalid_ip=0

	if [[ -z "${ipaddr_v6##$EMPTY_ZERO_IPADDRV6*}" ]] ;then
		 invalid_ip=1
	fi
	if [[ -z "${ipaddr_v6##$LINK_LOCAL_IPV6*}" ]] ;then
		 invalid_ip=1
	fi
	return $invalid_ip
}

function validate_ipv4_addr
{
	ipaddr_v4=$1
	invalid_ip=0

        if [ -z "${ipaddr_v4##$LOCAL_IPV4*}" ] ;then
                 invalid_ip=1
        fi

        if [[ ${ipaddr_v4} == ${EMPTY_ZERO_IPADDRV4} ]]
        then
                 invalid_ip=1
        fi

        if [ -z "${ipaddr_v4##$LINK_LOCAL_IPV4*}" ] ;then
                 invalid_ip=1
        fi

        return $invalid_ip
}

################################################################################
# Function: get_subnet_id_from_mask
# This function generates subnet id from the given host ip address and netmask 
# On AIX nodes netmask will be in hexadecimal [ for e.g 0xffffff00] form it will
# be converted to IP address form in this function.
# However on Linux, the new "ip" command will provide the CIDR prefix instead of netmask
# hence on Linux an equivalent netmask in IP address form will be generated and provided
# to this get_subnet_id_form_mask function. 
# The subnet / network id will be calculated as below.
#
# IPAddress          netmask                    subnet
# ip1.ip2.ip3.ip4    nm1.nm2.nm3.nm4     ((ip1 & nm1).(ip2 & nm2).(ip3 & nm3).(ip4 & nm4))
#
################################################################################


function get_subnet_id_from_mask
{
        ipaddr=$1

#       echo "ipaddr= ${ipaddr}"
#       echo "nmask= ${nmask}"

	if [[ ${BASEOS} == "AIX" ]]; then
		nmask=$2
		printf  "%d.%d.%d.%d\n" `echo ${nmask} | sed 's/../ 0x&/g'`>${TMPDIR}/nmfile
		nmask_ip_fmt=`cat ${TMPDIR}/nmfile`
		rm ${TMPDIR}/nmfile
	elif [[ ${BASEOS} == "Linux" ]]; then
		nmask_ip_fmt=$2
	fi

        OLDIFS=$IFS
        IFS='.'
        set -A ip_arry ${ipaddr}
        set -A nm_arry ${nmask_ip_fmt}

        j=0
        while [[ $j -lt 4 ]]
        do
                subnt_id[$j]=$(( ${ip_arry[$j]} & ${nm_arry[$j]} ))
                j=$(($j+1))
        done

        subnet_id="${subnt_id[0]}.${subnt_id[1]}.${subnt_id[2]}.${subnt_id[3]}"

        IFS=$OLDIFS
        echo "${subnet_id}"

}

function compare_subnets
{
        # Create a file /tmp/matched_subnet_lst_file for containing list of matched subnets.
        # This file will be used for MTU size compatibility check.

        rm -rf ${TMPDIR}/matched_subnet_lst_file
        rm -rf ${TMPDIR}/unmatched_subnet_lst_file

        compatible_count=0
        for each_lcl_nm in `cat ${TMPDIR}/${first_node}_subnets`
        do
                lcl_nm="$each_lcl_nm"
                match_count=0
                for each_node in ${other_nodes[@]}
                do
                        rc=0
                        matched_nm=`cat ${TMPDIR}/${each_node}_subnets | grep ${lcl_nm}`
                        rc=$?
                        if [[ $rc == 0 ]]; then
                                match_count=$(( $match_count + 1 ))
                        fi
                done

                if [[ $match_count == ${#other_nodes[@]} ]]; then
                        echo ${lcl_nm} >> ${TMPDIR}/matched_subnet_lst_file
                        compatible_count=$(( $compatible_count + 1 ))
                else
                        echo ${lcl_nm} >> ${TMPDIR}/unmatched_subnet_lst_file
                fi
        done

        if [[ $compatible_count == 0 ]];then
                return ${NO_MATCHING_SUBNET_FOUND}
        fi

        return 0
}

function get_valid_ipv6_list
{
	ipv6_file="${TMPDIR}/${first_node}_ipv6_list"
	if [[ ${LOCAL_NODE} == ${first_node} ]] || [[ ${LOCAL_NODE_SHORT_NAME} == ${first_node} ]]
	then
		if [[ ${BASEOS} == "AIX" ]]; then
			ifconfig -a |grep inet6 > ${ipv6_file}
		elif [[ ${BASEOS} == "Linux" ]]; then
			ip -6 a |grep inet6 > ${ipv6_file}
		fi
		ipaddr6_list=`cat ${ipv6_file} | awk '{print $2}'`
	else
		if [[ ${BASEOS} == "AIX" ]]; then
			REMOTE_EXEC ${first_node} "ifconfig -a |grep inet6" > ${ipv6_file}
			rc=$?
		elif [[ ${BASEOS} == "Linux" ]]; then
			REMOTE_EXEC ${first_node} "ip -6 a |grep inet6" > ${ipv6_file}
		fi
		
                if [[ $rc != 0 ]]
                then
                        return ${REMOTE_NODE_CONN_REFUSED}
                fi
		ipaddr6_list=`cat ${ipv6_file} | awk '{print $2}'`
	fi

	valid_ip_cnt=0

        for ipaddr_v6 in ${ipaddr6_list[@]}
        do
		$(validate_ipv6_addr ${ipaddr_v6})
		valid=$?
                if [[ ${valid} != 1 ]]
                then
                        valid_ipaddr_v6_list[$valid_ip_cnt]="$ipaddr_v6"
			valid_ip_cnt=$(($valid_ip_cnt + 1))
		fi
	done


        if [[ $valid_ip_cnt == 0 ]]
        then
            if [[ $node_cnt > 1 ]]; then
	      echo "IPv6 is not configured or not available on ${first_node}, will not check IPv6 on other nodes"
            fi
            if [[ $node_cnt == 1 ]]; then
	      echo "IPv6 is not configured or not available on ${first_node}"
            fi
            return ${NO_VALID_IPV6_FOUND}
        fi

	## Now check remote nodes

	each_node=""
        for each_node in ${other_nodes[@]}
        do
		valid_ip_cnt=0
                if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]
                then
                        ipv6_file="${TMPDIR}/${each_node}_ipv6_list"

			if [[ ${BASEOS} == "AIX" ]]; then
				REMOTE_EXEC ${each_node} "ifconfig -a | grep inet6" > ${ipv6_file}
				rc=$?
			elif [[ ${BASEOS} == "Linux" ]]; then
				REMOTE_EXEC ${each_node} "ip -6 a |grep inet6" > ${ipv6_file}
				rc=$?
			fi

			if [[ $rc != 0 ]]
			then
				return ${REMOTE_NODE_CONN_REFUSED}
			fi
		else
                        ipv6_file="${TMPDIR}/${LOCAL_NODE_SHORT_NAME}_ipv6_list"
			if [[ ${BASEOS} == "AIX" ]]; then
				ifconfig -a |grep inet6 > ${ipv6_file}
			elif [[ ${BASEOS} == "Linux" ]]; then
				ip -6 a |grep inet6 > ${ipv6_file}
			fi


		fi
		rmt_ipaddr6_list=`cat ${ipv6_file} | awk '{print \$2}'`

		valid_ip_cnt=0
		for ipaddr_v6 in ${rmt_ipaddr6_list[@]}
		do	
			echo "ipaddr_v6=" ${ipaddr_v6}
			$(validate_ipv6_addr ${ipaddr_v6})
			valid=$?
			if [[ ${valid} != 1 ]]
			then
				rmt_valid_ipaddr_v6_list[$valid_ip_cnt]="$ipaddr_v6"
				valid_ip_cnt=$(($valid_ip_cnt + 1))
			fi
		done
			
		if [[ $valid_ip_cnt -eq 0 ]]
		then
		     echo "No valid IPv6 interfaces on ${each_node}"
		    return ${NO_VALID_IPV6_FOUND}
		fi
	done
}

function get_nmlist_from_iplist_aix
{
	typeset t_input="$@"
	set -A input $t_input
	set -A tmp_nm_list 

	ip_cnt=${#input}
	tmpfile=${input[0]}

	set ifs_invalid=0
	set valid_ip_cnt=0

	i=0
        while [[ $i < $ip_cnt ]]
        do
           ipaddr_list[$i]=${input[$i+1]}
           i=$(($i+1))
        done


        for ipaddr_v4 in ${ipaddr_list[@]}
        do
                ifs_invalid=0
                $(validate_ipv4_addr "${ipaddr_v4}")
                ifs_invalid=$?

                if [[ ${ifs_invalid} != 1 ]]
                then
                        valid_ipaddr_list[$valid_ip_cnt]="$ipaddr_v4"
                        raw_nmask=`cat ${tmpfile} |grep -w ${ipaddr_v4} | awk '{print $4}' | cut -b 3-10`
                        subnet_id=$(get_subnet_id_from_mask "${ipaddr_v4}" "${raw_nmask}")
                        tmp_nm_list[$valid_ip_cnt]=${subnet_id}
                        valid_ip_cnt=$(( $valid_ip_cnt + 1 ))
                fi
        done

	echo ${tmp_nm_list[@]}

	return ${valid_ip_cnt}

}


function check_subnet_compat_v4_aix
{
set valid_ip_cnt=0
set rc=0

	## get valid ipv4 addrs & netmasks on first node treating it as base

	filename="${TMPDIR}/${first_node}_iplist"
	if [[ ${LOCAL_NODE} == ${first_node} ]] || [[ ${LOCAL_NODE_SHORT_NAME} == ${first_node} ]]
	then
		ifconfig -a inet |grep netmask > ${filename}
		ipaddr_list=`cat ${filename} |grep inet | awk '{print $2}'`
	else
		REMOTE_EXEC ${first_node} "ifconfig -a inet |grep netmask" > ${filename}
		rc=$?

		if [[ $rc != 0 ]]
		then
			return ${REMOTE_NODE_CONN_REFUSED}
		fi
		ipaddr_list=`cat ${filename} |grep inet | awk '{print \$2}'`
	fi
	
        nm_list=$(get_nmlist_from_iplist_aix ${filename} ${ipaddr_list[@]})
	valid_ip_cnt=$?

	set -A base_nm_list ${nm_list[@]}

        if [[ $valid_ip_cnt -eq 0 ]]
        then
	    print_errmsg "IPv4 is not configured or not available on ${first_node}."
            return ${NO_VALID_IPV4_FOUND}
 	else
#		echo "Valid IPv4 interfaces available on $first_node"
            for each_subnet in ${base_nm_list[@]}
            do
                echo ${each_subnet} >> ${TMPDIR}/${first_node}_subnets
            done
        fi

        ## Now check nodes other than first

	each_node=""
        for each_node in ${other_nodes[@]}
        do
		filename="${TMPDIR}/${each_node}_iplist"
		if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]
                then
                        REMOTE_EXEC ${each_node} "ifconfig -a inet |grep netmask" > ${filename}
			rc=$?

			if [[ $rc != 0 ]]
			then
				print_dbgmsg "Connection refused by remote node ${each_node}."
				return ${REMOTE_NODE_CONN_REFUSED}
			fi

                        nxt_ipaddr_list=`cat ${filename} |grep inet | awk '{print \$2}'`
		else

			ifconfig -a inet |grep netmask > ${filename}
			nxt_ipaddr_list=`cat ${filename} |grep inet | awk '{print $2}'`
                fi

		valid_ip_cnt=0
		nm_lst=$(get_nmlist_from_iplist_aix ${filename} ${nxt_ipaddr_list[@]})
		valid_ip_cnt=$?	
		set -A nxt_nm_list ${nm_lst[@]}

		if [[ $valid_ip_cnt -eq 0 ]]
		then
		    print_errmsg "IPv4 is not configured or not available on ${each_node}."
		    return ${NO_VALID_IPV4_FOUND}
                else
#                    echo "Valid IPv4 interfaces available on ${each_node}"

                    for each_subnet in ${nxt_nm_list[@]}
                    do
                        echo ${each_subnet} >> ${TMPDIR}/${each_node}_subnets
                    done

		fi
        done

	if [[ $node_cnt > 1 ]] ; then
                compare_subnets
                rc=$?
	        if [[ $rc == 0 ]]; then
			print_errmsg "Compatible subnets between nodes ${nodes[@]} are "
			uniq ${TMPDIR}/matched_subnet_lst_file > ${TMPDIR}/matched_subnets
			for matched_subnet in `cat ${TMPDIR}/matched_subnets`
			do
				echo $matched_subnet
			done
		fi

		if [[ -s ${TMPDIR}/unmatched_subnet_lst_file ]];then
	                uniq ${TMPDIR}/unmatched_subnet_lst_file > ${TMPDIR}/unmatched_subnets

			for unmatched_subnet in `cat ${TMPDIR}/unmatched_subnets`
			do
				echo "Subnet $unmatched_subnet is not found on ${other_nodes[@]} "
			done
		fi
        fi

	return $rc
}


function get_nmlist_from_iplist_linux
{
typeset t_input="$@"

set -A ipaddr_list "$@"
set -A tmp_nm_list
set ifs_invalid=0
set valid_ip_cnt=0

for ipaddr_v4 in ${ipaddr_list[@]}
do
	ifs_invalid=0
	ipaddr=`echo $ipaddr_v4 |awk 'BEGIN{FS="/"} {print $1}'`
	prefix=`echo $ipaddr_v4 |awk 'BEGIN{FS="/"} {print $2}'`

	$(validate_ipv4_addr "${ipaddr}")
	ifs_invalid=$?

	if [[ ${ifs_invalid} != 1 ]]
	then
		valid_ipaddr_list[$valid_ip_cnt]="${ipaddr}"
		ntmask=$(get_netmask_from_prefix  $prefix)
		subnet_id=$(get_subnet_id_from_mask "${ipaddr}" "${ntmask}")
		tmp_nm_list[$valid_ip_cnt]=${subnet_id}
		valid_ip_cnt=$(( $valid_ip_cnt + 1 ))
	fi
done
	echo ${tmp_nm_list[@]}
	return ${valid_ip_cnt}

}


function check_subnet_compat_v4_linux
{
set valid_ip_cnt=0
set rc=0

 ## get valid ipv4 addrs & netmasks on local_node first


        filename="${TMPDIR}/${first_node}_iplist"

        if [[ ${LOCAL_NODE} == ${first_node} ]] || [[ ${LOCAL_NODE_SHORT_NAME} == ${first_node} ]]
        then
                ip -4 a > ${filename}
                ipaddr_list=`cat ${filename} |grep inet | awk '{print $2}'`
        else
                REMOTE_EXEC ${first_node} "ip -4 a " > ${filename}
                rc=$?

                if [[ $rc != 0 ]]
                then
                        return ${REMOTE_NODE_CONN_REFUSED}
                fi
                ipaddr_list=`cat ${filename} |grep inet | awk '{print \$2}'`
        fi

        nm_list=$(get_nmlist_from_iplist_linux ${ipaddr_list[@]})
	valid_ip_cnt=$?

	set -A base_nm_list ${nm_list[@]}

        if [[ $valid_ip_cnt -eq 0 ]]
        then
            return ${NO_VALID_IPV4_FOUND}
	else
            print_dbgmsg "Valid IPv4 interfaces available on $first_node."
            for each_subnet in ${base_nm_list[@]}
            do
                echo ${each_subnet} >> ${TMPDIR}/${first_node}_subnets
            done
        fi

        ## Now check remaining nodes

	each_nodes=""
        for each_node in ${other_nodes[@]}
        do
#               echo $each_node
#               echo "LOCAL_NODE=" ${LOCAL_NODE} ", LOCAL_NODE_SHORT_NAME=" ${LOCAL_NODE_SHORT_NAME}

		filename="${TMPDIR}/${each_node}_iplist"
                if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]
                then
			REMOTE_EXEC ${each_node} "ip -4 a" > ${filename}
                       rc=$?
                        if [[ $rc != 0 ]]
                        then
                                return ${REMOTE_NODE_CONN_REFUSED}
                        fi
		else
			ip -4 a > ${filename}
			ipaddr_list=`cat ${filename} |grep inet | awk '{print $2}'`

                fi
		nxt_ipaddr_list=`cat ${filename} |grep inet | awk '{print \$2}'`
#	        print_dbgmsg "ip addr list for $each_node = ${nxt_ipaddr_list[@]}"

		valid_ip_cnt=0
		nm_lst=$(get_nmlist_from_iplist_linux ${nxt_ipaddr_list[@]})
		valid_ip_cnt=$?
		set -A nxt_nm_list ${nm_lst[@]}
#	        print_dbgmsg "Next node nmlist = ${nxt_nm_list[@]}"

		if [[ $valid_ip_cnt -eq 0 ]]
		then
		    return ${NO_VALID_IPV4_FOUND}
		 else
                    print_dbgmsg "Valid IPv4 interfaces available on ${each_node}."
                    for each_subnet in ${nxt_nm_list[@]}
                    do
                        echo ${each_subnet} >> ${TMPDIR}/${each_node}_subnets
                    done
		fi
	done

	if [[ $node_cnt > 1 ]] ; then
                compare_subnets
                rc=$?
	        if [[ $rc == 0 ]]; then
			print_errmsg "Compatible subnets between ${first_node} and ${other_nodes[@]} are"
			uniq ${TMPDIR}/matched_subnet_lst_file > ${TMPDIR}/matched_subnets
			for matched_subnet in `cat ${TMPDIR}/matched_subnets`
			do
				echo $matched_subnet
			done
		fi

		if [[ -s ${TMPDIR}/unmatched_subnet_lst_file ]]; then
			uniq ${TMPDIR}/unmatched_subnet_lst_file > ${TMPDIR}/unmatched_subnets
			for unmatched_subnet in `cat ${TMPDIR}/unmatched_subnets`
			do
				echo "Subnet $unmatched_subnet is not found on ${other_nodes[@]} "
			done
		fi
        fi

	return $rc
}


################################################################################
# Function: get_netmask_from_prefix
# This function generates netmask for the equivalent cidr prefix and is invoked 
# only on Linux nodes.
#  
#
# prefix             netmask
# 8                  255.0.0.0 
# 16                 255.255.0.0 
# 24                 255.255.255.0 
#
# This function will also take care of any other custom prefix say 21 , in that 
# case netmask will be 255.255.248.0
#
# input : prefix
# output : netmask
#
################################################################################


function get_netmask_from_prefix
{
	prefix=$1

#	echo $prefix
#
	quo=$(($prefix / 8 ))
	rem=$(($prefix % 8))

#	echo "quo =  $quo"
#	echo "rem = $rem"

	a=2
	b=$((8-$rem))
	val=`echo "$a^$b" |bc`
	nm_last_octet=$((256-$val))

	if [[ $quo > 0 ]] then
		nmask="255"
	fi

	i=1
	while [[ $i < $quo ]]
	do
		nm=".255"
		nmask=$nmask${nm}
		i=$(($i+1))
	done

	if [[ $quo < 4 &&  $rem != 0 ]]
	then
	  nmask="$nmask.${nm_last_octet}"
	fi

	cnt=0
	OLDIFS=$IFS
	IFS='.'
	for word in $nmask
	do
		cnt=$(($cnt+1))
	done
	IFS=$OLDIFS

##echo $cnt
	while [[ $cnt < 4 ]]
	do
 	  last_octet=".0"
	  nmask=$nmask$last_octet
	  cnt=$(($cnt+1))
	done
	echo "${nmask}"
}


function check_conn_path
{
typeset v4_rc=0
typeset v6_rc=0

conn_opt=$1

	if [[ $conn_opt == '4' ]] ; then

		print_errmsg "Performing Subnet availability (IPv4) checks for PeerDomain"
		if [[ ${BASEOS} == "AIX" ]]; then
			check_subnet_compat_v4_aix
			rc=$?
		elif [[ ${BASEOS} == "Linux" ]]; then
			check_subnet_compat_v4_linux
			rc=$?
		fi
	fi

	if [[ $conn_opt == '6' ]]; then
		print_errmsg "Looking for interfaces (IPv6) availability on nodes ${nodes[@]}"
		get_valid_ipv6_list
		rc=$?
	fi

	if [[ $conn_opt == 'a' ]]; then
		print_errmsg "Performing Subnet availability (IPv4) checks for PeerDomain"
		if [[ ${BASEOS} == "AIX" ]]; then
			check_subnet_compat_v4_aix
			v4_rc=$?
		elif [[ ${BASEOS} == "Linux" ]]; then
			check_subnet_compat_v4_linux
			v4_rc=$?
		fi

		print_errmsg "Looking for interfaces (IPv6) availability on nodes ${nodes[@]}"
		get_valid_ipv6_list
		v6_rc=$?

                if [[ $v4_rc != 0 && $v6_rc != 0 ]]
		then
#			print_errmsg "No valid connectivity found"
#			return ${NO_VALID_IP_FOUND}
			rc=$(($v4_rc || $v6_rc))
		fi
	fi

	return $rc
}

function do_cleanup
{

# remove all the files that got created during ifconfig read
 for each_node in $nodes
 do
    filename1="${TMPDIR}/${each_node}_iplist"
    filename2="${TMPDIR}/${each_node}_ipv6_list"
    filename3="${TMPDIR}/${each_node}_nmlist"
    filename4="${TMPDIR}/${each_node}_subnets"
    filename5="${TMPDIR}/matched_subnet_lst_file"
    filename6="${TMPDIR}/unmatched_subnet_lst_file"


    if [[ -f ${filename1} ]]; then
	rm ${filename1}
    fi
    if [[ -f ${filename2} ]]; then
	rm ${filename2}
    fi
    if [[ -f ${filename3} ]]; then
	rm ${filename3}
    fi
    if [[ -f ${filename4} ]]; then
        rm ${filename4}
    fi
    if [[ -f ${filename5} ]]; then
        rm ${filename5}
    fi
    if [[ -f ${filename6} ]]; then
        rm ${filename6}
    fi

 done
}


function check_port_available
{
  rc=0
  port_opt=$1
  tmp_node_cnt=0
  error_node_id=""

  for each_node in $nodes
  do
	tmp_node_cnt=$(($tmp_node_cnt+1))
#	print_dbgmsg "Checking connectivity with node $each_node using $CT_SYSCHK_SHELL..."
	REMOTE_EXEC $each_node "hostname" 1>/dev/null
	if [[ $? -ne 0 ]];then
	 return $REMOTE_NODE_CONN_REFUSED
	fi
  done
 
  if [[ $tmp_node_cnt -eq 0 ]];then
	nodes[0]=$LOCAL_NODE
  fi

  for each_node in $nodes
  do
	echo "Checking node $each_node for port availability..."
	IS_RPD_UP=0

        if [[ ${port_opt} != 'r' ]]; then

		RPD_ONLINE=$((REMOTE_EXEC $each_node "lsrpdomain -xo 2>/dev/null")|tr -d '\r')
		if [[ -n $RPD_ONLINE ]]; then 
			IS_RPD_UP=1
			print_dbgmsg "RPD is online on ${each_node} . Over-riding CT_SYSCHK_TS_PORT and CT_SYSCHK_GS_PORT env. vars. if any."
			CT_SYSCHK_TS_PORT=$(echo ${RPD_ONLINE}|awk '{print $5}')
			CT_SYSCHK_GS_PORT=$(echo ${RPD_ONLINE}|awk '{print $6}')
		fi
        fi

	# Check whether any ports are configured to be used by any other process
	# via /etc/services entries

	# etc_services_test_start
	rm -rf ${TMPDIR}/etc_services_out
	REMOTE_EXEC $each_node "cat /etc/services|egrep -w '${CT_SYSCHK_RMC_PORT}|${CT_SYSCHK_TS_PORT}|${CT_SYSCHK_GS_PORT}'" 1>${TMPDIR}/etc_services_out

	RMC_PORT_BOUND_BY_OTHER=$(grep -w "${CT_SYSCHK_RMC_PORT}" ${TMPDIR}/etc_services_out|egrep 'udp|tcp'|grep -vw rmc|grep -v ^#)
	TS_PORT_BOUND_BY_OTHER=$(grep -w "${CT_SYSCHK_TS_PORT}" ${TMPDIR}/etc_services_out|egrep 'udp|tcp'|grep -vw cthats|grep -v ^#)
	GS_PORT_BOUND_BY_OTHER=$(grep -w "${CT_SYSCHK_GS_PORT}" ${TMPDIR}/etc_services_out|egrep 'udp|tcp'|grep -vw cthags|grep -v ^#)

	rm -rf ${TMPDIR}/etc_services_out

	case "$port_opt" in
	r)
		if [[ -n "${RMC_PORT_BOUND_BY_OTHER}" ]];then
#			print_errmsg "$each_node : "
			error_node_id=$each_node
			return ${RMC_PORT_BOUND_BY_OTHER_RC}
		fi
		;;

	s)
		if [[ -n "${TS_PORT_BOUND_BY_OTHER}" ]];then
#			print_errmsg "$each_node : "
			error_node_id=$each_node
			return ${TS_PORT_BOUND_BY_OTHER_RC}
		fi

		if [[ -n "${GS_PORT_BOUND_BY_OTHER}" ]];then
#			print_errmsg "$each_node : "
			error_node_id=$each_node
			return ${GS_PORT_BOUND_BY_OTHER_RC}
		fi
		;;

	a)
		if [[ -n "${RMC_PORT_BOUND_BY_OTHER}" ]];then
#			print_errmsg "$each_node : "
			error_node_id=$each_node
			return ${RMC_PORT_BOUND_BY_OTHER_RC}
		fi
		if [[ -n "${TS_PORT_BOUND_BY_OTHER}" ]];then
#			print_errmsg "$each_node : "
			error_node_id=$each_node
			return ${TS_PORT_BOUND_BY_OTHER_RC}
		fi

		if [[ -n "${GS_PORT_BOUND_BY_OTHER}" ]];then
#			print_errmsg "$each_node : "
			error_node_id=$each_node
			return ${GS_PORT_BOUND_BY_OTHER_RC}
		fi
		;;
	esac
  
# etc_services_test_end

  #Initialize variables with blank strings

  CT_SYSCHK_RMC_PORT_TCP_OUT=""
  CT_SYSCHK_GS_PORT_OUT=""
  CT_SYSCHK_TS_PORT_OUT=""
  CT_SYSCHK_GS_PORT_HELD_BY_OTHER=""
  CT_SYSCHK_TS_PORT_HELD_BY_OTHER=""

unset OS_NAME
OS_NAME=$((REMOTE_EXEC $each_node "uname")|tr -d '\r'|tr -d '\n')
# On HMC, PATH variable cannot be set, so it will give output like this "OS_NAME='/bin/bash: PATH: readonly variableLinux'"
echo $OS_NAME|grep "readonly variableLinux" 1>/dev/null
if [[ $? -eq 0 ]];then
	OS_NAME="Linux"
fi

if [[ ${OS_NAME} == "Linux" ]]; then
	# Is it HMC? If yes, then RMC port check can be done with the help of lshmc --firewall command (via hscroot user)
	LSHSC_EXISTS=$(REMOTE_EXEC $each_node "ls $LSHSC  2>/dev/null")

	if [[ $CLUSTER_SCOPE == "m" ]] && [[ $LSHSC_EXISTS == $LSHSC ]]; then
		# the node is HMC so run lshmc command to get firewall details
		CT_SYSCHK_RMC_PORT_TCP_OUT=$((REMOTE_EXEC $each_node "export PATH=/hmcrbin/:/usr/hmcrbin:$PATH; lshmc --firewall|grep \"application=RMC.*ports=${CT_SYSCHK_RMC_PORT}:udp,tcp:${CT_SYSCHK_RMC_PORT}.*allowedhost=0.0.0.0/0.0.0.0\"|grep -v 255|grep -v ^$|tail -1")|tr -d '\r'|tr -d '\n')
	else
		CT_SYSCHK_RMC_PORT_TCP_OUT=$((REMOTE_EXEC $each_node "ss -anp|grep tcp.*${CT_SYSCHK_RMC_PORT}.*rmc")|tr -d '\r'|tr -d '\n')
	fi

	rm -rf ${TMPDIR}/SS_ANP_OUT
	
	REMOTE_EXEC $each_node "ss -anp|egrep '${CT_SYSCHK_GS_PORT}|${CT_SYSCHK_TS_PORT}'" > ${TMPDIR}/SS_ANP_OUT

	CT_SYSCHK_GS_PORT_OUT=$(grep udp.*${CT_SYSCHK_GS_PORT}.*hagsd ${TMPDIR}/SS_ANP_OUT)
	CT_SYSCHK_GS_PORT_HELD_BY_OTHER=$(grep udp.*${CT_SYSCHK_GS_PORT} ${TMPDIR}/SS_ANP_OUT|grep -v hagsd|grep -v ^$|tail -1)

	CT_SYSCHK_TS_PORT_OUT=$(grep udp.*${CT_SYSCHK_TS_PORT}.*hats_nim ${TMPDIR}/SS_ANP_OUT|grep -v ^$|tail -1)
	CT_SYSCHK_TS_PORT_HELD_BY_OTHER=$(grep udp.*${CT_SYSCHK_TS_PORT} ${TMPDIR}/SS_ANP_OUT|grep -v hats_nim|grep -v ^$|tail -1)

	rm -rf ${TMPDIR}/SS_ANP_OUT

elif [[ ${OS_NAME} == "AIX" ]]; then
	#RMC port 
	RMC_TCP_SOCKADDR=$((REMOTE_EXEC $each_node "netstat -Aan | grep -w ${CT_SYSCHK_RMC_PORT} | grep tcp|grep LISTEN|head -1 | cut -f1 -d ' '")|tr -d '\r'|tr -d '\n')
	if [[ -n $RMC_TCP_SOCKADDR ]];then
		CT_SYSCHK_RMC_PORT_TCP_ON=$(REMOTE_EXEC $each_node "rmsock $RMC_TCP_SOCKADDR tcpcb")
		CT_SYSCHK_RMC_PORT_TCP_OUT=$(echo $CT_SYSCHK_RMC_PORT_TCP_ON|grep "The socket.*is being held by proccess.*(rmcd)")
	fi

	#HAGS port
	GS_UDP_SOCKADDR=$((REMOTE_EXEC $each_node "netstat -Aan | grep -w ${CT_SYSCHK_GS_PORT} | grep udp |grep -v ^$|tail -1| cut -f1 -d ' '")|tr -d '\r'|tr -d '\n')
	if [[ -n $GS_UDP_SOCKADDR ]]; then
		CT_SYSCHK_GS_PORT_UDP_ON=$(REMOTE_EXEC $each_node "rmsock $GS_UDP_SOCKADDR inpcb")
		CT_SYSCHK_GS_PORT_OUT=$(echo $CT_SYSCHK_GS_PORT_UDP_ON|grep "The socket.*is being held by proccess.*(hagsd)")
		CT_SYSCHK_GS_PORT_HELD_BY_OTHER=$(echo $CT_SYSCHK_GS_PORT_UDP_ON|grep "The socket.*is being held by proccess.*"|grep -vw hagsd|grep -v ^$|tail -1)
	fi
	
	#HATS port
	TS_UDP_SOCKADDR=$((REMOTE_EXEC $each_node "netstat -Aan | grep -w ${CT_SYSCHK_TS_PORT} | grep udp |grep -v ^$|tail -1| cut -f1 -d ' '")|tr -d '\r'|tr -d '\n')
	if [[ -n $TS_UDP_SOCKADDR ]]; then
		CT_SYSCHK_TS_PORT_UDP_ON=$(REMOTE_EXEC $each_node "rmsock $TS_UDP_SOCKADDR inpcb")
		CT_SYSCHK_TS_PORT_OUT=$(echo $CT_SYSCHK_TS_PORT_UDP_ON|grep "The socket.*is being held by proccess.*(hats_nim)")
		CT_SYSCHK_TS_PORT_HELD_BY_OTHER=$(echo $CT_SYSCHK_TS_PORT_UDP_ON|grep "The socket.*is being held by proccess.*"|grep -vw hats_nim|grep -v ^$|tail -1)
	fi
else
	# print_errmsg "Unknown OS!!"
#		print_errmsg "${each_node} : "
	return ${UNSUPPORTED_OS_RC}
fi


    case "$port_opt" in 
	r)
		# If RMC is active, then check for port 657
		if [ -n "${CT_SYSCHK_RMC_PORT_TCP_OUT}" ]; then
			print_dbgmsg "SUCCESS: RMC is listening at port ${CT_SYSCHK_RMC_PORT} on ${each_node}."
			rc=0
		else
#			print_errmsg "$each_node : "
			error_node_id=$each_node
			return ${RMC_NOT_LISTENING_RC}
		fi	
            ;;

	s)
		print_dbgmsg "Checking HAGS and HATS ports for availability..."

		if [[ ${IS_RPD_UP} -eq 1 ]]; then
			if [[ -n "${CT_SYSCHK_GS_PORT_OUT}" ]];then
				print_dbgmsg "SUCCESS, Port ${CT_SYSCHK_GS_PORT} is being used by an online RPD on ${each_node} !!!"
			else
				# print_errmsg "Warning:Port ${CT_SYSCHK_GS_PORT} is not being used for Group Services!!"
			#	print_errmsg "$each_node : "
				return ${GS_NOT_LISTENING_RC}
			fi
		elif [[ -n "${CT_SYSCHK_GS_PORT_HELD_BY_OTHER}" ]];then
			# print_errmsg "Warning:Port ${CT_SYSCHK_GS_PORT} is not being used for Group Services!!"
			#print_errmsg "$each_node : "
			return ${GS_PORT_HELD_BY_OTHER_RC}
		else
			print_dbgmsg "SUCCESS: Port ${CT_SYSCHK_GS_PORT} is available on ${each_node}"
		fi
		if [[ ${IS_RPD_UP} -eq 1 ]]; then
			if [[ -n "${CT_SYSCHK_TS_PORT_OUT}" ]];then
				print_dbgmsg "SUCCESS, Port ${CT_SYSCHK_TS_PORT} is being used by an online RPD on ${each_node} !!!"
			else
				# print_errmsg "Warning:Port ${CT_SYSCHK_TS_PORT} is not being used for topology Services!!"
			#	print_errmsg "$each_node : "
				return ${TS_NOT_LISTENING_RC}
			fi
		elif [[ -n "${CT_SYSCHK_TS_PORT_HELD_BY_OTHER}" ]];then
			# print_errmsg "Warning:Port ${CT_SYSCHK_TS_PORT} is not being used for topology Services!!"
			#print_errmsg "$each_node : "
			return ${TS_PORT_HELD_BY_OTHER_RC}
		else
			print_dbgmsg "SUCCESS: Port ${CT_SYSCHK_TS_PORT} is available on ${each_node}"
		fi

            ;;

	a)

		# If RMC is active, then check for port 657
		if [ -n "${CT_SYSCHK_RMC_PORT_TCP_OUT}" ]; then
			print_dbgmsg "SUCCESS: RMC is listening at port ${CT_SYSCHK_RMC_PORT} on ${each_node}."
		else
			# print_errmsg "RMC is not listening at port ${CT_SYSCHK_RMC_PORT}"
#			print_errmsg "$each_node : "
			error_node_id=${each_node}
			return ${RMC_NOT_LISTENING_RC}
		fi	
		
		if [[ ${IS_RPD_UP} -eq 1 ]]; then
			if [[ -n "${CT_SYSCHK_GS_PORT_OUT}" ]];then
				print_dbgmsg "SUCCESS, Port ${CT_SYSCHK_GS_PORT} is being used by an online RPD on ${each_node} !!!"
			else
				# print_errmsg "Warning:Port ${CT_SYSCHK_GS_PORT} is not being used for Group Services!!"
			#	print_errmsg "$each_node : "
				return ${GS_PORT_HELD_BY_OTHER_RC}
			fi
		elif [[ -n "${CT_SYSCHK_GS_PORT_HELD_BY_OTHER}" ]];then
			# print_errmsg "Warning:Port ${CT_SYSCHK_GS_PORT} is not being used for Group Services!!"
			#print_errmsg "$each_node : "
			return ${GS_PORT_HELD_BY_OTHER_RC}
		else
			print_dbgmsg "SUCCESS: Port ${CT_SYSCHK_GS_PORT} is available on ${each_node} "
		fi
		if [[ ${IS_RPD_UP} -eq 1 ]]; then
			if [[ -n "${CT_SYSCHK_TS_PORT_OUT}" ]];then
				print_dbgmsg "SUCCESS, Port ${CT_SYSCHK_TS_PORT} is being used by an online RPD on ${each_node} !!!"
			else
				# print_errmsg "Warning:Port ${CT_SYSCHK_TS_PORT} is not being used for topology Services!!"
			#	print_errmsg "$each_node : "
				return ${TS_PORT_HELD_BY_OTHER_RC}
			fi
		elif [[ -n "${CT_SYSCHK_TS_PORT_HELD_BY_OTHER}" ]];then
			# print_errmsg "Warning:Port ${CT_SYSCHK_TS_PORT} is not being used for topology Services!!"
			#print_errmsg "$each_node : "
			return ${TS_PORT_HELD_BY_OTHER_RC}
		else
			print_dbgmsg "SUCCESS: Port ${CT_SYSCHK_TS_PORT} is available on ${each_node}."
		fi
            ;;
        *)    echo "Warning: WRONG OPTION FOR PORT CHECK !!"
		return ${WRONG_OPT_FOR_PORT_CHECK_RC}
            ;;
    esac


#######

## Check iptables command for blocked port on Linux on LPARs (NOT HMC)

# # iptables -L
# Chain INPUT (policy ACCEPT)
# target     prot opt source               destination
#            all  --  anywhere             anywhere
#
# Chain FORWARD (policy ACCEPT)
# target     prot opt source               destination
# REJECT     tcp  --  anywhere             anywhere             tcp dpt:2222 reject-with tcp-reset
# REJECT     tcp  --  anywhere             anywhere             tcp spt:2221 reject-with tcp-reset
#
# Chain OUTPUT (policy ACCEPT)
# target     prot opt source               destination

	firewall_check_rc=0
	if [[ ${OS_NAME} == "Linux" ]]; then
	  if [[ $CLUSTER_SCOPE == "m" ]] && [[ $LSHSC_EXISTS == $LSHSC ]]; then
			port_with_firewall=$((REMOTE_EXEC $each_node "export PATH=/hmcrbin/:/usr/hmcrbin:$PATH; lshmc --firewall|grep \"application=RMC.*ports=${CT_SYSCHK_RMC_PORT}:udp,tcp:${CT_SYSCHK_RMC_PORT}.*allowedhost=0.0.0.0/0.0.0.0\"|grep 255|grep -v ^$|tail -1")|tr -d '\r'|tr -d '\n')
			
			if [[ -n "${port_with_firewall}" ]];then
				return ${FIREWALL_CHECK_FAILED_RC}
			fi
	  else
		rm -rf ${TMPDIR}/IPTABLES_OUTFILE
		REMOTE_EXEC ${each_node} "iptables -L 2>/dev/null"|egrep -i "reject|drop"|egrep "${CT_SYSCHK_GS_PORT}|${CT_SYSCHK_TS_PORT}|${CT_SYSCHK_RMC_PORT}|rmc"|awk '{print $7}' >${TMPDIR}/IPTABLES_OUTFILE

		egrep -i "${CT_SYSCHK_GS_PORT}|${CT_SYSCHK_TS_PORT}|${CT_SYSCHK_RMC_PORT}|rmc" ${TMPDIR}/IPTABLES_OUTFILE
		if [[ $? -eq 0 && -s ${TMPDIR}/IPTABLES_OUTFILE ]];then
			for line in `cat ${TMPDIR}/IPTABLES_OUTFILE`
			do
				port_type=$(echo $line|cut -d':' -f1)
				port_num=$(echo $line|cut -d':' -f2)

				if [[ $port_type = "spt" ]];then
					port_type="Source Port"
				else
					port_type="Destination Port"
					echo $each_node
				fi
				return ${FIREWALL_CHECK_FAILED_RC}
			done
		fi
		rm -rf ${TMPDIR}/IPTABLES_OUTFILE
		firewall_check_rc=0
	    fi
	elif [[ ${BASEOS} == "AIX" ]]; then
	
		IsSecureAIX=$((REMOTE_EXEC $each_node "aixpert -t 2>/dev/null|grep 'Applied Profiles'|egrep 'HLS|MLS|LLS|DLS|SCBPS'")|tr -d '\r'|tr -d '\n')
		if [[ -n $IsSecureAIX ]]; then
			# Secure AIX test start
			rm -rf ${TMPDIR}/LSFILT_OUTFILE
			REMOTE_EXEC $each_node "lsfilt -O 2>/dev/null|grep permit" > ${TMPDIR}/LSFILT_OUTFILE
			if [[ -s ${TMPDIR}/LSFILT_OUTFILE ]]; then
				IS_RMC_PORT_PERMITTED=$(grep "eq|${CT_SYSCHK_RMC_PORT}|both|both" ${TMPDIR}/LSFILT_OUTFILE|grep "0.0.0.0|0.0.0.0")
				IS_GS_PORT_PERMITTED=$(grep "eq|${CT_SYSCHK_GS_PORT}|both|both" ${TMPDIR}/LSFILT_OUTFILE|grep "0.0.0.0|0.0.0.0")
				IS_TS_PORT_PERMITTED=$(grep "eq|${CT_SYSCHK_TS_PORT}|both|both" ${TMPDIR}/LSFILT_OUTFILE|grep "0.0.0.0|0.0.0.0")
			else
				IS_RMC_PORT_PERMITTED=""
				IS_GS_PORT_PERMITTED=""
				IS_TS_PORT_PERMITTED=""
			fi
			rm -rf ${TMPDIR}/LSFILT_OUTFILE
			firewall_check_rc=0

			case "$port_opt" in 
			r)
				if [[ -z $IS_RMC_PORT_PERMITTED ]]; then

					print_errmsg "$each_node: Warning: Please set filter rules to allow bi-directional RSCT communication through port: ${CT_SYSCHK_RMC_PORT}"
					firewall_check_rc=1
				fi
				;;
			s)
				if [[ -z $IS_GS_PORT_PERMITTED ]] || [[ -z $IS_TS_PORT_PERMITTED ]]; then
					print_errmsg "$each_node: Warning: Please set filter rules to allow bi-directional RSCT communication through ports: ${CT_SYSCHK_GS_PORT} and ${CT_SYSCHK_TS_PORT}!!"
					firewall_check_rc=1
				fi

				;;
			a)
				if [[ -z $IS_RMC_PORT_PERMITTED ]] || [[ -z $IS_GS_PORT_PERMITTED ]] || [[ -z $IS_TS_PORT_PERMITTED ]]; then
					print_errmsg "$each_node: Warning: Please set filter rules to allow bi-directional RSCT communication through ports: ${CT_SYSCHK_RMC_PORT}, ${CT_SYSCHK_GS_PORT} and ${CT_SYSCHK_TS_PORT}!!"
					firewall_check_rc=1 # At-least one of the ports is blocked
				fi
				;;
			esac
			# Secure AIX test end
		
		else 
			check_firewall_on_aix $port_opt
			firewall_check_rc=$?
		fi

		if [[ $firewall_check_rc -ne 0 ]]; then
			return ${FIREWALL_CHECK_FAILED_RC}
		else 
			firewall_check_rc=0
		fi
	else 
		echo "OS: ${OS_NAME} not supported or recognized!!"
		return ${UNSUPPORTED_OS_RC}
	fi

  done

 return $rc
}

################################################################################
# Function: check_mtu_size
# This function checks whether specified MTU size is supported by RSCT for communication between
# cluster nodes
# Input: 1. None
# Output: None
# Return values: 
#      0               : MTU size is supported by RSCT
#      Non-zero : MTU size is not supported by RSCT
################################################################################

function check_mtu_size 
{
  print_errmsg "Performing MTU compatibility check."
  tmp_node_cnt=0
  for each_node in $nodes
  do
	tmp_node_cnt=$(($tmp_node_cnt+1))
#	print_dbgmsg "Checking connectivity with node $each_node using $CT_SYSCHK_SHELL..."
	REMOTE_EXEC $each_node "hostname" 1>/dev/null
	if [[ $? -ne 0 ]];then
	 return $REMOTE_NODE_CONN_REFUSED
	fi
  done

  if [[ $tmp_node_cnt -le 1 ]];then
        print_errmsg "Performing MTU compatibility check - Done. Success"
	return $MTU_COMPAT_SUCCESS
  fi

  # Assume return-code as success initially
  rc=$MTU_COMPAT_SUCCESS

  # File "${TMPDIR}/matched_subnet_lst_file" is filled with matching subnet entries during 
  # subnet compatibility check. 
  # If file does not exist, run check_conn_path function and get entries in file
  # else just use the file for MTU compatibility check
  
  mtu_sz_compat_file="${TMPDIR}/matched_subnets"
  
  if [ ! -s ${mtu_sz_compat_file} ];then  
	  if [ $COMMOPT == '4' ] || [ $COMMOPT == 'a' ]; then
		# if there are matching subnet then only check for MTU size compatibility
		check_conn_path 4
		conn_path_rc_4_mtu=$?
		if [[ $conn_path_rc_4_mtu -eq 0 ]]; then
			if [[ ${BASEOS} == "AIX" ]]; then
				check_subnet_compat_v4_aix
				v4_rc=$?
			elif [[ ${BASEOS} == "Linux" ]]; then
				check_subnet_compat_v4_linux
				v4_rc=$?
			fi
		else 
			print_errmsg "MTU test: No matching subnets!!";
			rc=$MTU_COMPAT_CHECK_FAILED
		fi
	  else
		return $MTU_NO_OP_4_IPV6
	  fi
  fi

  if [ $rc -eq $MTU_NO_OP_4_IPV6 ] || [ $rc -eq $MTU_COMPAT_CHECK_FAILED ]; then
	return $rc
  fi 

  if [[ -s ${mtu_sz_compat_file} ]];then
    uniq ${mtu_sz_compat_file} > ${TMPDIR}/mtu_compat_file
    #Now ${TMPDIR}/mtu_compat_file contains unique compatible subnets. Remove old file
    rm -rf ${mtu_sz_compat_file}
  fi

  first=1

  MTUSZ=0 # Just initialize MTUSZ which is going to be compared with the other nodes' MTU

  mtu_match_cnt=0 # Initially match count is 0
  for each_node in $nodes
  do
	cmd_for_mtu="netstat -in|grep -vw sit|grep -vw lo0"
	REMOTE_EXEC $each_node "$cmd_for_mtu" > ${TMPDIR}/mtu_internal_file_$each_node
  done

  #netmask number
  nm_cnt=0
  for line in `cat ${TMPDIR}/mtu_compat_file 2>/dev/null` # Start loop; for each netmask in the file, 
  do
	
	nm_cnt=$(($nm_cnt+1))
	# Make netmask to compare with netstat outputs
	octet1=`echo $line|cut -d'.' -f1`
	octet2=`echo $line|cut -d'.' -f2`
	octet3=`echo $line|cut -d'.' -f3`

	# If 3rd octet is also 0, try grepping only first 2 octets
	if [[ $octet3 -eq 0 ]];then
		if [[ $octet2 -eq 0 ]];then
			nm_to_compare=$octet1\.
		else
			nm_to_compare=$octet1.$octet2\.
		fi
	else
		nm_to_compare=$octet1.$octet2.$octet3\.
	fi

	first=1
	unmatched_mtu_cnt=0
	
	for each_node in $nodes
	do
		unset OS_NAME
		OS_NAME=$((REMOTE_EXEC $each_node "uname")|tr -d '\r'|tr -d '\n')
		# On HMC, PATH variable cannot be set, so it will give output like this "OS_NAME='/bin/bash: PATH: readonly variableLinux'"
		echo $OS_NAME|grep "readonly variableLinux" 1>/dev/null
		if [[ $? -eq 0 ]];then
			OS_NAME="Linux"
		fi

		if [[ ${OS_NAME} == "AIX" ]]; then
			mtu_sz=$(grep -w $nm_to_compare ${TMPDIR}/mtu_internal_file_$each_node|grep -v ^$|tail -1|awk '{print $2}')
		elif [[ ${OS_NAME} == "Linux" ]]; then
			# For Linux, additionally one more cmd is to be run because netstat -in does not give MTU
			interface_name=$((REMOTE_EXEC $each_node "ip -4 a")|grep ${nm_to_compare}|grep -v ^$|tail -1|awk '{print $NF}'|tr -d '\r'|tr -d '\n')
			if [[ ! -z $interface_name ]]; then
				mtu_sz=$(grep -w $interface_name ${TMPDIR}/mtu_internal_file_$each_node|awk '{print $2}')
			fi
		else
			mtu_cleanup
			return $UNSUPPORTED_OS_RC
		fi

		# In first iteration, capture the MTU size of first node's interface 
		# which is being verified with other nodes' interfaces.
		if [[ $first -eq 1 ]];then
			MTUSZ=${mtu_sz}
			first=0
			continue
		fi

		if [[ $mtu_sz -eq $MTUSZ ]]; then
			mtu_match_cnt=$(($mtu_match_cnt+1))
		else
			unmatched_mtu_cnt=$(($unmatched_mtu_cnt+1))
			echo ${each_node}:MTU=$mtu_sz>> ${TMPDIR}/unmatched_mtu
			echo ${each_node}:MTU=$mtu_sz>> ${TMPDIR}/${each_node}_mtu_mismatch
		fi
	done
	
	if [[ $unmatched_mtu_cnt -gt 0 ]]; then
		cat ${TMPDIR}/unmatched_mtu|tr '\n' ' ' > ${TMPDIR}/unmatched_mtu.nodes_$nm_cnt
		rm -rf ${TMPDIR}/unmatched_mtu
		echo "Warning: MTU mismatch for matching subnet: ${line} between node ${first_node}:MTU=$MTUSZ and `cat ${TMPDIR}/unmatched_mtu.nodes_$nm_cnt`"
	fi

	# Is every node's interface having same MTU ?
	if [[ $mtu_match_cnt -eq $(($tmp_node_cnt-1)) ]]; then
		rc=$MTU_COMPAT_SUCCESS
	fi
  done
  # End loop; for each netmask in the file

 #Now check if a node has entry in all the unmatched mtu nodes_$nm_cnt files, then the node is incompatible.
 first=0
  for each_node in $nodes
  do
	if [[ $first -eq 0 ]];then
		first=$(($first+1))
		mtu_first_node=$each_node
		
		continue;
	fi
  
	if [[ -s ${TMPDIR}/${each_node}_mtu_mismatch ]]; then
		unmatch_mtu_cnt_4_node=$(wc -l ${TMPDIR}/${each_node}_mtu_mismatch |awk '{print $1}')
		if [[ $nm_cnt -eq $unmatch_mtu_cnt_4_node ]];then
			print_errmsg "Error: ${mtu_first_node} and $each_node don't have any matching subnets with matching MTU sizes!!"
			rc=$MTU_COMPAT_CHECK_FAILED
		fi
	fi
  rm -rf ${TMPDIR}/unmatched_mtu.nodes_$nm_cnt
  done

  mtu_cleanup

  if [[ $mtu_match_cnt -eq 0 ]];then
	rc=$MTU_COMPAT_CHECK_FAILED
  fi


  return $rc
}

function mtu_cleanup {
  rm -rf ${TMPDIR}/mtu_internal_file_*
  rm -rf ${TMPDIR}/mtu_compat_file
}

function check_node_id 
{
  print_errmsg "Performing Node Id checks."
  rm -rf ${TMPDIR}/node_id_*

  for each_node in $nodes 
  do
	rm -rf ${TMPDIR}/node_id_out
	rm -rf ${TMPDIR}/node_id_err

	REMOTE_EXEC ${each_node} "head -n 1 ${NODE_ID_FILE}" 1>${TMPDIR}/node_id_out 2>${TMPDIR}/node_id_err

	# Possibilities: 1. ssh/rsh failed. 2. ssh passes but cmd fails.
	if [ -s ${TMPDIR}/node_id_out ];then
		echo "${each_node}:`cat ${TMPDIR}/node_id_out|tr -d '\r'|tr -d '\n'`"  >> ${TMPDIR}/node_id_with_nodename_all
	else
		grep ${each_node} ${TMPDIR}/node_id_err
		if [[ $? -eq 0 ]]; then
			# meaning that error contains node name.
			return ${REMOTE_NODE_CONN_REFUSED}
		else
			error_node_id=${each_node}
			print_errmsg "${NODE_ID_FILE} does not exist on ${each_node}!!"
		fi
	fi
  done

  dup_node_id_found=0
  if [ -s ${TMPDIR}/node_id_with_nodename_all ];then
	cat ${TMPDIR}/node_id_with_nodename_all|cut -d':' -f2|sort > ${TMPDIR}/node_id_out_all
	for entry in `uniq -d ${TMPDIR}/node_id_out_all`
	do
		dup_node_id_found=1
		print_errmsg "Duplicate Node ID's found."
#		grep $entry ${TMPDIR}/node_id_with_nodename_all|cut -d':' -f1|tr -s '\n' ' '
                grep $entry ${TMPDIR}/node_id_with_nodename_all
	done
  fi
 
  rm -rf ${TMPDIR}/node_id_*
  if [[ ${dup_node_id_found} -eq 1 ]];then
#        print_errmsg "The node ids are duplicated between nodes"
	return ${NODE_ID_NOT_UNIQUE}
  fi

#  print_errmsg "Success. Node ID's are unique between ${nodes[@]}"
  print_errmsg "Node Id checks - Done. All nodes have unique Node ID's." 
  return ${NODE_ID_SUCCESS}

}

function check_session_scope
{
 UNAUTHENT_CMD="lsrsrc IBM.HostPublic PublicKey"
 AUTHENT_CMD="lsrsrc IBM.Host"

 if [[ $SESSOPT == 'u' ]]; then
	for each_node in $nodes
	do
	    if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]; then
		OUT=`REMOTE_EXEC $each_node $UNAUTHENT_CMD`
		rc=$?
                if echo $OUT |grep -q "PublicKey = "
                then
                  print_dbgmsg "Able to open unauthenticated session with $each_node."
                fi
            fi
        done
 fi
 if [[ $SESSOPT == 'a' ]]; then
	for each_node in $nodes
	do
	    if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]; then
		OUT=`REMOTE_EXEC $each_node $AUTHENT_CMD`
		rc=$?
                if echo $OUT |grep -q "Name "
                then
                  print_dbgmsg "Able to open authenticated session with $each_node."
                fi
            fi
        done
 fi
 return $rc
}


function check_peerdomain_req
{
rc=0
dflt_commopt='a'
print_errmsg "Checking PeerDomain pre-req's .."

# step-1 : check interfaces availability

check_conn_path "$dflt_commopt"
conn_chk_rc=$?
if [[ $conn_chk_rc != $CONN_PATH_SUCCESS ]]  
then
	return ${conn_chk_rc}
fi

if [[ -z $PORTOPT ]];then
	PORTOPT='a'
fi

# step-2 : check port availability

print_errmsg "Performing port availability checks"
check_port_available $PORTOPT
port_chk_rc=$? 
if [[ $port_chk_rc -ne  $PORT_CHECK_SUCCESS ]];then
 port_chk_failure_node=${each_node}
else
 print_errmsg "RMC port (${CT_SYSCHK_RMC_PORT}) is available on all nodes."
 print_errmsg "HAGS port (${CT_SYSCHK_GS_PORT}) is available on all nodes."
 print_errmsg "HATS port (${CT_SYSCHK_TS_PORT}) is available on all nodes."
 print_errmsg "Port availability checks.. Done."
fi

# step-3 : check node Id availability
check_node_id
node_chk_rc=$?

# Specific processing if basecmd contains -m (i.e. MTU size test)
# step-4 : check node Id availability
  check_mtu_size
  mtu_compat_rc=$?

# MTU size check end
  if [[ ${mtu_compat_rc} -eq $MTU_COMPAT_SUCCESS ]];then
	if [[ ${node_cnt} -ne 1 ]]; then
          print_errmsg "Performing MTU compatibility check - Done. Interfaces on all nodes have same MTU size."
	fi
  else
        print_errmsg "Performing MTU compatibility check - Done. MTU size is not compatible across all nodes."
  fi

if [[ $conn_chk_rc == $CONN_PATH_SUCCESS ]]  && [[ $port_chk_rc == $PORT_CHECK_SUCCESS ]] && [[ $node_chk_rc == $NODE_ID_SUCCESS ]] && [[ $mtu_compat_rc == $MTU_COMPAT_SUCCESS ]]
then
	return $PD_REQ_SUCCESS
else
	chk_rpd_req_rc=$(($conn_chk_rc | $port_chk_rc | $node_chk_rc | $mtu_compat_rc ))
	if [[ $conn_chk_rc -ne $CONN_PATH_SUCCESS ]];then
		rc=$conn_chk_rc
	elif [[ $port_chk_rc -ne $PORT_CHECK_SUCCESS ]]; then
		rc=$port_chk_rc
	elif [[ $node_chk_rc -ne $NODE_ID_SUCCESS ]]; then
		rc=$node_chk_rc
	elif [[ $mtu_compat_rc -ne $MTU_COMPAT_SUCCESS ]]; then
		rc=$mtu_compat_rc
	else
		rc=$chk_rpd_req_rc
	fi
	#process_exit_code $rc
	return $rc
fi

}

function check_management_domain_req
{
print_errmsg "Performing port availability check for management domain cluster.."
 
PORTOPT='r'
check_port_available $PORTOPT
rc=$?
if [[ $rc -ne  $PORT_CHECK_SUCCESS ]];then
# rc=${each_node}
 print_errmsg "RMC port check failed on ${each_node}"
else
 print_errmsg "RMC port (${CT_SYSCHK_RMC_PORT}) is available on all nodes."
 print_errmsg "Port availability checks.. Done."
fi

return $rc
}


#####################################################################################################
# Name: process_opts                                                                                #
# This function is called from main script and this function invokes the necessary checks related   #
# to interfaces connectivity between the nodes , all checks related to peer domain or management    #
# domain, port availability for hags/hats/rmc or firewall enabled etc.,.                            #
# This also validates the input options passed.                                                     #
# Return code: 0 if success, 1 otherwise                                                            #
#####################################################################################################

function process_opts
{

if [[ ${Argcount} == 0 ]]
then
   echo "$0 : Invalid number of operands."
   usage 1
fi

if [[ $UNKNOWN_SCOPEOPT == 1 ]] || [[ $UNKNOWN_PORTOPT == 1 ]] || [[ $UNKNOWN_SESSOPT == 1 ]] || [[ $UNKNOWN_COMMOPT == 1 ]]
then
   print_errmsg "Invalid inputs specified"
   exit 1
fi
   
if [[ $CLUSTER_SCOPE == 'p' ]]
then
	check_peerdomain_req 
	rc=$?
	if [[ $rc == 0 ]]; then
		print_errmsg "Checking PeerDomain pre-req's - Done. Given nodes can be used for PeerDomain."
	else
		print_errmsg "Checking PeerDomain pre-req's - Done. PeerDomain prereq's failed. Check configuration details. "
        fi
fi

if [[ $CLUSTER_SCOPE == 'm' ]]
then
	check_management_domain_req 
	rc=$?
fi

if [[ $CLUSTER_SCOPE == 'a' ]] 
then
	check_peerdomain_req 
	rc1=$?
	check_management_domain_req
	rc2=$?

	if [[ $rc1 == 0 && $rc2 == 0 ]]; then
		return $PD_MD_REQ_SUCCESS
	else
		return $(($rc1 | $rc2))
	fi
fi

#if [[ $CLUSTER_SCOPE != 'p' && $CLUSTER_SCOPE != 'm' && $CLUSTER_SCOPE != 'a' ]]
#then
   if [[ ${optI} == 1 ]] && [[ ${VALID_SCOPE} == 0 ]] ; then
	check_conn_path "$COMMOPT"
	conn_rc=$?
	rc=$conn_rc

	if [[ $conn_rc -eq 0 ]]; then
		if [[ $COMMOPT == '4' ]] || [[ $COMMOPT == '6' ]] ; then
			print_errmsg "IPv${COMMOPT} interfaces are available on the nodes ${nodes[@]}."
		else
			print_errmsg "IPv4 / IPv6 interfaces are available on the nodes ${nodes[@]}."
		fi
	fi

   fi

   if [[ ${optP} == 1 ]]  && [[ ${VALID_SCOPE} == 0 ]]; then
	if [[ $UNKNOWN_PORTOPT -eq 1 ]];then
		print_errmsg "Invalid Port Option"
		process_exit_code $WRONG_OPT_FOR_PORT_CHECK_RC
	else
		check_port_available $PORTOPT
		port_rc=$?

		if [[ $port_rc != $PORT_CHECK_SUCCESS ]]; then
			print_errmsg "Port availability check failed."
			process_exit_code $port_rc
			return $rc
		fi
	fi
   fi

   if [[ ${optN} == 1 ]]  && [[ ${VALID_SCOPE} == 0 ]]; then
	check_node_id
	node_id_rc=$?
	rc=${node_id_rc}
   fi

   if [[ ${optM} == 1 ]]  && [[ ${VALID_SCOPE} == 0 ]]; then
	# If no option is provided, just do it for all
	typeset COMMOPT="a"
	check_mtu_size
	mtu_compat_rc=$?

	if [[ ${mtu_compat_rc} -eq $MTU_COMPAT_SUCCESS ]];then
		if [[ ${node_cnt} -ne 1 ]];then
			print_errmsg "Performing MTU compatibility check - Done. Interfaces on all nodes have same MTU size."
		fi
	else
		print_errmsg "Performing MTU compatibility check - Done. MTU size is not compatible across all nodes."
	fi
	if [[ $mtu_compat_rc -eq $REMOTE_NODE_CONN_REFUSED ]]; then
		return $REMOTE_NODE_CONN_REFUSED
	fi
   fi

#fi

   if [[ ${optS} == 1 ]]; then
	check_session_scope 
	rc=$?
   fi

 do_cleanup
 return $rc
}

function process_exit_code
{
exit_code=$1

case $exit_code in
	$NO_VALID_IPV4_FOUND )
		print_errmsg  "${NO_VALID_IPV4_FOUND_STR}"
		rc=${NO_VALID_IP_FOUND}
		;;
	$NO_VALID_IPV4_FOUND_RMT_NODE )
		print_errmsg  "${NO_VALID_IPV4_FOUND_RMT_NODE_STR}"
		rc=${NO_VALID_IP_FOUND}
		;;
	$NO_VALID_IPV6_FOUND )
		print_errmsg  "${NO_VALID_IPV6_FOUND_STR}"
		rc=${NO_VALID_IP_FOUND}
		;;
	$NO_VALID_IP_FOUND )
		print_errmsg  "${NO_VALID_IP_FOUND_STR}"
		rc=${NO_VALID_IP_FOUND}
		;;
	$NO_MATCHING_SUBNET_FOUND )
		print_errmsg  "${NO_MATCHING_SUBNET_FOUND_STR}"
		rc=${NO_VALID_IP_FOUND}
		;;
	$REMOTE_NODE_CONN_REFUSED )
		print_errmsg  "${REMOTE_NODE_CONN_REFUSED_STR}"
		rc=${REMOTE_NODE_CONN_REFUSED}
		;;
	$NODE_ID_NOT_UNIQUE )
		print_errmsg  "${NODE_ID_NOT_UNIQUE_STR} ${error_node_id}"  
		rc=${NODE_ID_NOT_UNIQUE}
		;;
	$NO_NODE_ID_FILE )
		print_errmsg  "${NO_NODE_ID_FILE_STR} on ${error_node_id} "
		;;

	$RMC_NOT_LISTENING_RC )
		print_errmsg "${RMC_NOT_LISTENING_STR} on ${error_node_id} "
		rc=${PORT_ERROR}
		;;
	$TS_NOT_LISTENING_RC )
		print_errmsg "${TS_NOT_LISTENING_STR}"
		rc=${PORT_ERROR}
		;;
	$GS_NOT_LISTENING_RC )
		print_errmsg "${GS_NOT_LISTENING_STR}"
		rc=${PORT_ERROR}
		;;

	$RMC_PORT_BOUND_BY_OTHER_RC )
		print_errmsg "${RMC_PORT_BOUND_BY_OTHER_STR} on ${error_node_id}"
		rc=${PORT_ERROR}
		;;
	$TS_PORT_BOUND_BY_OTHER_RC )
		print_errmsg "${TS_PORT_BOUND_BY_OTHER_STR} on ${error_node_id}"
		rc=${PORT_ERROR}
		;;	
	$GS_PORT_BOUND_BY_OTHER_RC )
		print_errmsg "${GS_PORT_BOUND_BY_OTHER_STR} on ${error_node_id}"
		rc=${PORT_ERROR}
		;;

	$RMC_PORT_HELD_BY_OTHER_RC )
		print_errmsg "${RMC_PORT_HELD_BY_OTHER_STR} on ${error_node_id}"
		rc=${PORT_ERROR}
		;;
	$TS_PORT_HELD_BY_OTHER_RC )
		print_errmsg "${TS_PORT_HELD_BY_OTHER_STR} on ${error_node_id}"
		rc=${PORT_ERROR}
		;;	
	$GS_PORT_HELD_BY_OTHER_RC )
		print_errmsg "${GS_PORT_HELD_BY_OTHER_STR} on ${error_node_id}"
		rc=${PORT_ERROR}
		;;

	$WRONG_OPT_FOR_PORT_CHECK_RC )
		print_errmsg "${WRONG_OPT_FOR_PORT_CHECK_STR}"
		rc=${OTHER_ERROR}
		;;

	$FIREWALL_CHECK_FAILED_RC )
		print_errmsg "${port_chk_failure_node}: ${FIREWALL_CHECK_FAILED_STR}"
		rc=${PORT_ERROR}
		;;

	$INVALID_MTU_SZ_RC )
		print_errmsg "${INVALID_MTU_SZ_STR}"
		rc=${MTU_CHECK_ERROR}
		;;

	$MTU_NO_OP_4_IPV6 )
		print_errmsg "${MTU_NO_OP_4_IPV6_STR}"
		rc=${MTU_CHECK_ERROR}
		;;

	$MTU_COMPAT_CHECK_FAILED )
		print_errmsg "${MTU_COMPAT_CHECK_FAILED_STR}"
		rc=${MTU_CHECK_ERROR}
		;;
	$UNSUPPORTED_OS_RC )
		print_errmsg "${UNSUPPORTED_OS_STR}"
		rc=${OTHER_ERROR}
		;;

esac
return $rc
}

################################################################################
#                                                                              #
#     MAIN FUNCTION                                                            #
#                                                                              #
################################################################################

comment_str="########################################################################"

all_args=$@
# trap read debug
basecmd="$0"
TMPDIR=/tmp/CTSYSCHK_$$
OLDLOGDIR="/tmp/CTSYSCHK"
RMOLDDIRS=$(ls -d ${OLDLOGDIR}_* 2> /dev/null)

for i in ${RMOLDDIRS}
do
print_dbgmsg "Removing old DIRS in /tmp/CTSYSCHK*"
print_dbgmsg $i
pid=$(echo $i | sed 's:^'"${OLDLOGDIR}"\.'::g')
#echo $pid
if [[ -z $(ps -o pid= -p $pid 2> /dev/null) ]]; then
     rm -rf $i
fi
done

mkdir -p ${TMPDIR}
OPTLIST=":i:p:s:C:U:P:mnuhV"

while getopts $OPTLIST opt
do
    case $opt in

        C )
                CLUSTER_SCOPE=$OPTARG
                case $CLUSTER_SCOPE in
                    p | m | a  )
                         VALID_SCOPE=1
                         ;;
                    *)
                         UNKNOWN_SCOPEOPT=1
                         usage
                         ;;
                esac
	            optC=1
		    basecmd=$basecmd$SPACE$MINUS$opt$SPACE$CLUSTER_SCOPE 
                ;;

        i )
                COMMOPT=$OPTARG
                case $COMMOPT in
                     4 | 6 | a )
                        ;;

                     *)
                         UNKNOWN_COMMOPT=1
                         ;;
                esac
                    optI=1
		    basecmd=$basecmd$SPACE$MINUS$opt$SPACE$COMMOPT 
                ;;

        p )
                PORTOPT=$OPTARG
                case $PORTOPT in
                     r | s | a )
                        ;;

                     *)
                         UNKNOWN_PORTOPT=1
                         ;;
                esac
                    optP=1
		    basecmd=$basecmd$SPACE$MINUS$opt$SPACE$PORTOPT 
                ;;

	U )
		USERNAME=$OPTARG
		optUSR=1
		basecmd=$basecmd$SPACE$MINUS$opt$SPACE$USERNAME
		;;

	P )
		PASSWD=$OPTARG
		optPSWD=1
		basecmd=$basecmd$SPACE$MINUS$opt$SPACE$PASSWD
		;;
#        t )
#		echo "TB-disk test is a LIMITATION as of now."
#                TB_DISK_INFO=$OPTARG
#		    basecmd=$basecmd$SPACE$MINUS$opt$SPACE$TB_DISK_INFO 
#                ;;

        m )
		optM=1
		basecmd=$basecmd$SPACE$MINUS$opt
                ;;
        n )
                    optN=1
		    basecmd=$basecmd$SPACE$MINUS$opt

                ;;
        u )
                    optU=1
		    basecmd=$basecmd$SPACE$MINUS$opt
                ;;
#        k )
#                    optK=1
#		    basecmd=$basecmd$SPACE$MINUS$opt
#                ;;
        s )
                SESSOPT=$OPTARG
                case $SESSOPT in
                     u | a )
                        ;;
                     *)
                         UNKNOWN_SESSOPT=1
                         ;;
                esac
                    optS=1
		    basecmd=$basecmd$SPACE$MINUS$opt$SPACE$SESSOPT
                ;;
        V )
               VERBOSE=1
		    basecmd=$basecmd$SPACE$MINUS$opt
               ;;
        h )
                usage
                ;;

        ?|* )
		echo "Unknown option: $opt "
                usage
                ;;

    esac
                Argcount=$((Argcount + 1))
done

print_dbgmsg "${comment_str}"

typeset node_names=""
typeset each_node=""

#if [[ $Argcount == 0 ]]
#then
#   echo "$0 : Invalid number of operands."
#   usage 1
#fi


# Get the node names now
all_args=$@
local_node_present=0
ipargc="$#"

cmd_line="$0 $@"

set node_cnt=0
set -A nodes  ""
set -A total_nodes ""
typeset error_node_id=""


if [[ ${ipargc} == 0 ]] ; then
 cmd_line=${basecmd}
fi

if [[ "${cmd_line}" ==  "${basecmd}" ]] 
then
	if [[ $Argcount == 0 ]]
	then
	   echo "$0 : Invalid number of operands."
	   usage 1
	fi
	print_dbgmsg "No nodenames specified, running on local node"
	LOCAL_NODE=`hostname`
	first_node=${LOCAL_NODE}
	node_cnt=1
	process_opts
	rc=$?
	if [[ $rc == 0 ]]
	then 
		print_errmsg "ctsyschk is successful on $LOCAL_NODE."
	else
		 print_errmsg "ctsyschk failed on $LOCAL_NODE."
	         process_exit_code $rc
		 rc=$?
        fi
else
         nodes=${cmd_line##$basecmd}

	set -A total_nodes ""
	i=0
	node_cnt=0
	for each_node in ${nodes[@]}
	do
	    total_nodes[$i]="${each_node}"
	    i=$(($i+1))
	done

	 node_cnt=${#total_nodes[@]}
	 first_node=${total_nodes[0]}

	 i=0
	 for each_node in ${total_nodes[@]}
	 do
		if [[ $each_node == $first_node ]]; then
			continue
		fi
	        other_nodes[$i]="${each_node}"
	        i=$(($i+1))
	 done

	  if [[ $node_cnt == 1 ]]; then
		if [[ $Argcount == 0 ]]; then
			optI=1;
                        optP=1;
			Argcount=$((Argcount + 1))

			print_errmsg "Performing default checks as no input options specified." 
			process_opts
			rc=$?
                else
		    if [[ $optN == 1 ]]; then
		       print_errmsg "Option -n is a no-op for single node"
			    rc=${OTHER_ERROR}
#		    elif [[ $optM == 1 ]]; then
#		       print_errmsg "Option -m is a no-op for single node"
#			    rc=${OTHER_ERROR}
		    elif [[ $optS == 1 ]]; then
		       print_errmsg "Option -s is a no-op for single node"
			    rc=${OTHER_ERROR}
	            else
			 process_opts
			 rc=$?
		    fi
                fi
	  else
		  process_opts
		  rc=$?
	  fi
	  if [[ $rc == 0 ]]; then 
		 print_errmsg "ctsyschk is successful. "
	  else
		 process_exit_code $rc
		 rc=$?
		 print_errmsg "ctsyschk failed with $rc. "
	  fi
fi

rm -rf ${TMPDIR}
print_dbgmsg ${comment_str}
exit $rc
################
#   END MAIN   #
################