#!/bin/ksh
# IBM_PROLOG_BEGIN_TAG
# This is an automatically generated prolog.
#
#
#
# Licensed Materials - Property of IBM
#
# (C) COPYRIGHT International Business Machines Corp. 2018,2019
# All Rights Reserved
#
# US Government Users Restricted Rights - Use, duplication or
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp.
#
# IBM_PROLOG_END_TAG
#
# sccsid = "@(#)90 1.42 src/rsct/utils/ctsyschk.sh, common_utils, rsct_rady, rady2035a 8/14/18 06:24:12"
#
#
#ctsyschk -TV -h -n -k -s -l
# -C [p|m|a] // Cluster scope - p : PeerDomain m : Management Domain a : All
# -i [4|6|a] // interfaces - 4 : IPv4 6 : IPv6 a : All / Any
# -p [r|s|a] // port - r : rmc port s : HAGS & HATS port a -- All
# -U username //username for ssh
# -P password //password for ssh
# -t [tiebreaker_disk_info] //PVID or WWID of the disk
# -m // MTU size compatibility check amongst nodes
# -n // nodeid of all nodes to be verified for non uniqueness
# -k // key exchange with remote node needs to be verified
# -s // rmc session to remote node - authenticated / unauthenticated
# [node_name1 node_name2 ...]
#
#
#
#
#
# set -x
PATH=/opt/rsct/install/bin:/opt/rsct/bin:/usr/bin:/bin:/usr/sbin:/sbin
export PATH
BASEOS=`uname`
SPACE=" "
MINUS="-"
typeset Argcount=0
typeset CLUSTER_SCOPE=''
typeset LOCAL_NODE=""
typeset UNKNOWN_SCOPEOPT=0
typeset VALID_SCOPE=0
typeset UNKNOWN_COMMOPT=0
typeset UNKNOWN_PORTOPT=0
typeset UNKNOWN_SESSOPT=0
typeset VERBOSE=0
typeset DEFAULT_COMMOPT="a"
typeset DEFAULT_PORTOPT='a'
typeset COMMOPT=${DEFAULT_COMMOPT}
typeset PORTOPT=${DEFAULT_PORTOPT}
typeset optI=0
typeset optP=0
typeset optM=0
typeset optN=0
typeset optS=0
typeset optK=0
typeset optUSR=0
typeset optPSWD=0
typeset LOCAL_IPV4="127.0."
typeset LOCAL_IPV4_2="127.0.0.1/8"
typeset LINK_LOCAL_IPV4="169.254."
typeset LINK_LOCAL_IPV6="fe80:"
typeset EMPTY_ZERO_IPADDRV4="0.0.0.0"
typeset EMPTY_ZERO_IPADDRV6="::"
typeset VALID_IFS_V4_LST=""
typeset CONN_PATH_SUCCESS=0
typeset NODE_ID_SUCCESS=0
typeset PORT_CHECK_SUCCESS=0
typeset PROCESS_OPTS_SUCCESS=0
typeset MTU_COMPAT_SUCCESS=0
typeset PD_REQ_SUCCESS=0
typeset PD_REQ_FAIL=1
typeset NO_VALID_IP_FOUND=1
typeset NO_MATCHING_SUBNET_FOUND=2
typeset PORT_ERROR=3
typeset NODE_ID_NOT_UNIQUE=5
typeset MTU_CHECK_ERROR=6
typeset REMOTE_NODE_CONN_REFUSED=7
typeset OTHER_ERROR=8
typeset NO_VALID_IPV4_FOUND=11
typeset NO_VALID_IPV4_FOUND_RMT_NODE=12
typeset NO_VALID_IPV6_FOUND=13
#Port Availability check return codes in case of errors
typeset RMC_PORT_BOUND_BY_OTHER_RC=51
typeset TS_PORT_BOUND_BY_OTHER_RC=52
typeset GS_PORT_BOUND_BY_OTHER_RC=53
typeset RMC_NOT_LISTENING_RC=55
typeset TS_NOT_LISTENING_RC=56
typeset GS_NOT_LISTENING_RC=57
typeset RMC_PORT_HELD_BY_OTHER_RC=58
typeset TS_PORT_HELD_BY_OTHER_RC=59
typeset GS_PORT_HELD_BY_OTHER_RC=60
typeset FIREWALL_CHECK_FAILED_RC=62
typeset MTU_COMPAT_CHECK_FAILED=71
typeset NO_NODE_ID_FILE=111 #this is a warning
typeset MTU_NO_OP_4_IPV6=113 #warning that user is trying to check MTU size of IPv6 network
typeset UNSUPPORTED_OS_RC=114 #warning that user is running script on unsupported OS
typeset WRONG_OPT_FOR_PORT_CHECK_RC=115 # warning that user entered unsupported port option.
typeset NO_VALID_IPV4_FOUND_STR="No valid IPv4 interfaces found"
typeset NO_VALID_IPV4_FOUND_RMT_NODE_STR="No valid IPv4 interfaces found on remote node"
typeset NO_VALID_IPV6_FOUND_STR="No valid IPv6 interfaces found"
typeset NO_VALID_IP_FOUND_STR="No valid ip connectivity found on the node"
typeset NO_MATCHING_SUBNET_FOUND_STR="There are no matching subnets found between the nodes"
typeset REMOTE_NODE_CONN_REFUSED_STR="connection refused by remote node"
typeset NODE_ID_NOT_UNIQUE_STR="Duplicate node id found on the node"
typeset NO_NODE_ID_FILE_STR="Node ID file is not found on the node"
typeset RMC_PORT_BOUND_BY_OTHER_STR="RMC Port is bound by some other process. Please check /etc/services"
typeset TS_PORT_BOUND_BY_OTHER_STR="HATS Port is bound by some other process. Please check /etc/services"
typeset GS_PORT_BOUND_BY_OTHER_STR="HAGS Port is bound by some other process. Please check /etc/services"
typeset RMC_NOT_LISTENING_STR="RMC is not listening on port $CT_SYSCHK_RMC_PORT"
typeset TS_NOT_LISTENING_STR="Topology Services is not active on port $CT_SYSCHK_TS_PORT"
typeset GS_NOT_LISTENING_STR="Group Services is not active on port $CT_SYSCHK_TS_PORT"
typeset RMC_PORT_HELD_BY_OTHER_STR="RMC Port is being used by some other process"
typeset TS_PORT_HELD_BY_OTHER_STR="HATS Port is being used by some other process"
typeset GS_PORT_HELD_BY_OTHER_STR="HAGS Port is being used by some other process"
typeset UNSUPPORTED_OS_STR="This is an unsupported OS. Supported OS are AIX and Linux."
typeset WRONG_OPT_FOR_PORT_CHECK_STR="Unsupported option for port check!! Supported options are r (RMC), s (GS,TS), a (RMC, GS and TS)."
typeset FIREWALL_CHECK_FAILED_STR="Firewall check failed. A firewall rule is blocking communication on RMC, HATS or HAGS port. Please check firewall rules."
typeset MTU_NO_OP_4_IPV6_STR="MTU test is valid only for IPv4 interfaces"
typeset MTU_COMPAT_CHECK_FAILED_STR="MTU compatibility check failed"
LOCAL_NODE=`hostname`
LOCAL_NODE_SHORT_NAME=`hostname -s`
NODE_ID_FILE="/var/ct/cfg/ct_node_id"
LSHSC="/opt/hsc/bin/lshsc"
CT_SYSCHK_RMC_PORT=657
# If HAGS and HATS ports are not set by env. vars., assign default port-numbers
if [ -z "${CT_SYSCHK_TS_PORT}" ]; then
CT_SYSCHK_TS_PORT=12347
fi
if [ -z "${CT_SYSCHK_GS_PORT}" ]; then
CT_SYSCHK_GS_PORT=12348
fi
if [ -z "${CT_SYSCHK_SHELL}" ]; then
CT_SYSCHK_SHELL="ssh"
fi
# Fuctions start from here.
# Function to show usage (will be shown with -h option and in general)
function usage
{
[[ "$CTSYSCHK_VERBOSE" == "high" ]] && set -x
echo "Usage: ctsyschk [-U username] [-P password] [-C cluster_type ] [-i interface_type] [-p port ] [-m] [-n] [-s session_scope ] Node_Name1 Node_Name2"
exit_code=${OTHER_ERROR}
exit $exit_code
}
#Function to print debug messages (will be shown with -V option)
function print_dbgmsg
{
[[ "$CTSYSCHK_VERBOSE" == "high" ]] && set -x
if [[ $VERBOSE == 1 ]]
then
#echo ${basecmd} "($$)": "$1"
echo "$1"
fi
if [[ -n $CTSYSCHK_LOG_FILE ]]
then
echo ${basecmd} "($$)": "$1" >> ${CTSYSCHK_LOG_FILE}
fi
}
function print_errmsg
{
if [[ -n $CTSYSCHK_ERR_FILE ]]
then
echo ${basecmd} "($$)" : "$1" >> ${CTSYSCHK_ERR_FILE}
else
# echo ${basecmd} "($$)": "$1"
echo "$1"
fi
}
############################################################
# Function: REMOTE_EXEC
# This function is intended to run command on remote node
# either via expect script or directly via remote_shell.
#
# Environment variables:
# 1. CT_SYSCHK_SHELL (default remote_shell=> ssh)
#
# Inputs:
# 1. Nodename
# 2. Command
#
# Output:
# 1. command's output
#
# Return-code:
# 0 : Success
# Non-zero: Failure
############################################################
function REMOTE_EXEC {
remote_node=$1
shift
cmd=$*
if [ -z "${USERNAME}" ]; then
username=root
else
username=${USERNAME}
fi
if [ -z "${PASSWD}" ]; then
password=root123
else
password=${PASSWD}
fi
if [ -z "${CT_SYSCHK_SHELL}" ]; then
remote_shell=ssh
else
remote_shell=${CT_SYSCHK_SHELL}
fi
remote_exec_rc=0
command -v expect 1>/dev/null
if [[ $? -eq 0 ]];then
REMOTE_EXEC_VIA_EXPECT ${remote_node} ${cmd}
remote_exec_rc=$?
else
${remote_shell} ${remote_node} -l ${username} "export PATH=/sbin:/usr/sbin:$PATH; ${cmd}"
remote_exec_rc=$?
fi
return $remote_exec_rc
}
function REMOTE_EXEC_VIA_EXPECT {
if [ -z "${USERNAME}" ]; then
username=root
else
username=${USERNAME}
fi
if [ -z "${PASSWD}" ]; then
password=root123
else
password=${PASSWD}
fi
if [ -z "${CT_SYSCHK_SHELL}" ]; then
remote_shell=ssh
else
remote_shell=${CT_SYSCHK_SHELL}
fi
remote_node=$1
shift
cmd=$*
expect << SSH_EXPECT
log_user 0
# Current limitations:
# 1. CT_SYSCHK_SHELL should be set. otherwise it will take ssh by default.
set timeout 10
set output ""
set buf_captured 0
match_max 300000
# ==> Spawnning remote shell and getting output
spawn ${remote_shell} ${remote_node} -l ${username} "export PATH=/sbin:/usr/sbin:$PATH; ${cmd}"
expect {
"(yes/no)?*" {
send "yes\r"
expect "*?assword:" {
send "${password}\r"
}
}
"*?assword: "
{ send "${password}\r" }
"*ould not resolve hostname *" {
puts stderr "Invalid hostname. remote shell invocation failed!!\r"
exit 1
}
"*onnection refused*" {
puts stderr "Connection refused by .!!\r"
exit 1
}
"${remote_node}*NOT FOUND" {
puts stderr "Invalid hostname. rsh failed!!\r"
exit 1
}
"rshd*The remote user login is not correct" {
puts stderr "Invalid username. rsh failed!!\r"
exit 1
}
"Permission denied*" {
puts stderr "Permission denied for Host: ${remote_node}, User: ${username} .\r"
exit 1
}
timeout {
puts stderr "Timeout..."
exit 1
}
eof { set output \$expect_out(buffer)
set buf_captured 1}
}
if {\$buf_captured == 0} {
expect eof
}
# ==> By now, login process is complete. User either has got output or error.
set output \$expect_out(buffer)
puts stdout \$output
exit 0
SSH_EXPECT
} # End of REMOTE_EXEC
################################################################################
# Function: check_firewall_on_aix_internal
# This function takes cues from a firewall rule to identify whether the rule can
# block cluster communication on AIX nodes.
# Inputs:
# 1. Source IP Address
# 2. Source Port Comparison Operator (le/lt/ge/gt/eq/neq)
# 3. Source Port number
# 4. Destination IP Address
# 5. Destination Port Comparison Operator
# 6. Destination Port Number
# 7. Port option: r|s|a
# Output: None
# Return values:
# 0 : IP Filter rules does not affect cluster communication
# Non-zero : IP Filter rules may affect cluster communication
################################################################################
function check_firewall_on_aix_internal
{
src_ipaddr=$1 # if this is 0.0.0.0, don't use it
src_operator=$2
src_port_num=$3
dest_ipaddr=$4 # if this is 0.0.0.0, don't use it
dest_operator=$5
dest_port_num=$6
port_opt=$7
firewall_check_internal_rc=0
if [ $src_ipaddr = "0.0.0.0" ] && [ $dest_ipaddr = "0.0.0.0" ];then
return $firewall_check_internal_rc
fi
if [[ $src_ipaddr = "0.0.0.0" ]]; then
port_num=$dest_port_num
operator=$dest_operator
elif [[ $dest_ipaddr = "0.0.0.0" ]]; then
port_num=$src_port_num
operator=$src_operator
fi
if [ $src_ipaddr = "0.0.0.0" ] || [ $dest_ipaddr = "0.0.0.0" ];then
case "$port_opt" in
r)
if [ $operator = "eq" ] || [ $operator = "le" ] || [ $operator = "ge" ]; then
if [ ${port_num} -eq ${CT_SYSCHK_RMC_PORT} ];then
firewall_check_internal_rc=1
fi
elif [ $operator = "lt" ] && [ ${CT_SYSCHK_RMC_PORT} -lt ${port_num} ]; then
firewall_check_internal_rc=2
elif [ $operator = "gt" ] && [ ${CT_SYSCHK_RMC_PORT} -gt ${port_num} ]; then
firewall_check_internal_rc=3
elif [ $operator = "neq" -a ${CT_SYSCHK_RMC_PORT} -ne ${port_num} ] || [ $operator = "any" ]; then
#neq => only 1 port allowed. any => No port allowed.
firewall_check_internal_rc=4
else
firewall_check_internal_rc=0
fi
;;
s)
if [ $operator = "eq" ] || [ $operator = "le" ] || [ $operator = "ge" ]; then
if [ ${port_num} -eq ${CT_SYSCHK_GS_PORT} ] || [ ${port_num} -eq ${CT_SYSCHK_TS_PORT} ];then
firewall_check_internal_rc=1
fi
elif [ $operator = "lt" ] && [ ${CT_SYSCHK_GS_PORT} -lt ${port_num} || ${CT_SYSCHK_TS_PORT} -lt ${port_num} ]; then
firewall_check_internal_rc=2
elif [ $operator = "gt" ] && [ ${CT_SYSCHK_GS_PORT} -gt ${port_num} || ${CT_SYSCHK_TS_PORT} -gt ${port_num} ]; then
firewall_check_internal_rc=3
elif [ $operator = "neq" ] || [ $operator = "any" ]; then
#neq => only 1 port allowed. any => No port allowed. 3 ports are needed for RPD (RMC, HATS, HAGS).
firewall_check_internal_rc=4
else
firewall_check_internal_rc=0
fi
;;
a)
if [ $operator = "eq" ] || [ $operator = "le" ] || [ $operator = "ge" ];then
if [ ${port_num} -eq ${CT_SYSCHK_RMC_PORT} ] || [ ${port_num} -eq ${CT_SYSCHK_GS_PORT} ] || [ ${port_num} -eq ${CT_SYSCHK_TS_PORT} ];then
firewall_check_internal_rc=1
fi
elif [ $operator = "lt" ] &&
[ ${CT_SYSCHK_RMC_PORT} -lt ${port_num} || ${CT_SYSCHK_GS_PORT} -lt ${port_num} || ${CT_SYSCHK_TS_PORT} -lt ${port_num} ]; then
firewall_check_internal_rc=2
elif [ $operator = "gt" ] &&
${CT_SYSCHK_RMC_PORT} -gt ${port_num} || ${CT_SYSCHK_GS_PORT} -gt ${port_num} || ${CT_SYSCHK_TS_PORT} -gt ${port_num} ]; then
firewall_check_internal_rc=3
elif [ $operator = "neq" ] || [ $operator = "any" ]; then
#neq => only 1 port allowed. any => No port allowed. 3 ports are needed for RPD (RMC, HATS, HAGS).
firewall_check_internal_rc=4
else
firewall_check_internal_rc=0
fi
;;
*) firewall_check_internal_rc=$FIREWALL_CHECK_FAILED_RC
;;
esac
elif [ $src_ipaddr != "0.0.0.0" ] && [ $dest_ipaddr != "0.0.0.0" ];then
: <<'SRC_DEST_CHECK_IN_NODES'
If there is a special firewall from a particular source IP to particular destination IP,
we need to check first whether source and destination IP are not amongst any of the nodes' IPs.
SRC_DEST_CHECK_IN_NODES
src_or_dest_ip_on_node=0
for node_name in $nodes
do
REMOTE_EXEC ${node_name} "ifconfig -a inet|grep -w inet|tr '\n' ' '" >${TMPDIR}/IP_ADDR_OUT
egrep -w '$src_ipaddr|$dest_ipaddr' ${TMPDIR}/IP_ADDR_OUT
if [[ $? -eq 0 ]]; then
src_or_dest_ip_on_node=1
break
fi
done
rm -rf ${TMPDIR}/IP_ADDR_OUT
if [[ $src_or_dest_ip_on_node -ne 1 ]];then
return 0
fi
# Do, actual test now for RMC, HAGS and HATS ports.
#Right now, port_num has destination's port number.
operator=$dest_operator
port_num=$dest_port_num
if [ $port_num -ne 0 ];then
if [ $operator = "eq" || $operator = "le" || $operator = "ge" ] &&
[ ${port_num} -eq ${CT_SYSCHK_RMC_PORT} || ${port_num} -eq ${CT_SYSCHK_GS_PORT} || ${port_num} -eq ${CT_SYSCHK_TS_PORT} ];then
firewall_check_internal_rc=1
elif [ $operator = "lt" ] &&
[ ${CT_SYSCHK_RMC_PORT} -lt ${port_num} || ${CT_SYSCHK_GS_PORT} -lt ${port_num} || ${CT_SYSCHK_TS_PORT} -lt ${port_num} ]; then
firewall_check_internal_rc=2
elif [ $operator = "gt" ] &&
${CT_SYSCHK_RMC_PORT} -gt ${port_num} || ${CT_SYSCHK_GS_PORT} -gt ${port_num} || ${CT_SYSCHK_TS_PORT} -gt ${port_num} ]; then
firewall_check_internal_rc=3
elif [ $operator = "neq" ] || [ $operator = "any" ]; then
#neq => only 1 port allowed. any => No port allowed. 3 ports are needed for RPD (RMC, HATS, HAGS).
firewall_check_internal_rc=4
else
firewall_check_internal_rc=0
fi
fi
#Now, check whether the source port is blocked
operator=$src_operator
port_num=$src_port_num
if [[ $port_num -ne 0 ]] ;then
if [[ $operator = "eq" || $operator = "le" || $operator = "ge" ]] &&
[[ ${port_num} -eq ${CT_SYSCHK_RMC_PORT} || ${port_num} -eq ${CT_SYSCHK_GS_PORT} || ${port_num} -eq ${CT_SYSCHK_TS_PORT} ]];then
firewall_check_internal_rc=1
elif [[ $operator = "lt" ]] &&
[[ ${CT_SYSCHK_RMC_PORT} -lt ${port_num} || ${CT_SYSCHK_GS_PORT} -lt ${port_num} || ${CT_SYSCHK_TS_PORT} -lt ${port_num} ]]; then
firewall_check_internal_rc=2
elif [[ $operator = "gt" ]] &&
[[ ${CT_SYSCHK_RMC_PORT} -gt ${port_num} || ${CT_SYSCHK_GS_PORT} -gt ${port_num} || ${CT_SYSCHK_TS_PORT} -gt ${port_num} ]]; then
firewall_check_internal_rc=3
elif [[ $operator = "neq" ]] || [[ $operator = "any" ]]; then
#neq => only 1 port allowed. any => No port allowed. 3 ports are needed for RPD (RMC, HATS, HAGS).
firewall_check_internal_rc=4
else
firewall_check_internal_rc=0
fi
fi
else
firewall_check_internal_rc=0
fi
return $firewall_check_internal_rc
} # end of check_firewall_on_aix_internal
################################################################################
# Function: check_firewall_on_aix
# This function checks whether any source or destination ports are blocked on
# cluster nodes. This function is intended for AIX nodes.
# Inputs:
# 1. Port option: r|s|a
# Output: None
# Return values:
# 0 : IP Filter rules does not affect cluster communication
# Non-zero : IP Filter rules may affect cluster communication
################################################################################
function check_firewall_on_aix
{
firewall_rc=0
port_opt=$1
# Search for IP filter rules denying communication on ports
print_dbgmsg "Running lsfilt to check AIX firewall rules..."
rm -rf ${TMPDIR}/tmp_lsfilt_file
for each_node in $nodes
do
REMOTE_EXEC ${each_node} "lsfilt|grep -p deny" > ${TMPDIR}/tmp_lsfilt_file
while IFS= read -r line
do
# display line or do somthing on $line
case $line in
Rule*[0-9]*) rule_num_str=`echo $line`
;;
Rule*ction*) rule_action=`echo $line|cut -d: -f2`
;;
Source*Address*) src_addr=`echo $line|cut -d: -f2`
;;
Destination*Address*) dest_addr=`echo $line|cut -d: -f2`
;;
Source*Port*) src_port_str=`echo $line|cut -d: -f2`
src_operator=`echo ${src_port_str}|awk '{print $1}'`
src_port_num=`echo ${src_port_str}|awk '{print $2}'`
;;
Dest*Port*) dest_port_str=`echo $line|cut -d: -f2`
dest_operator=`echo ${dest_port_str}|awk '{print $1}'`;
dest_port_num=`echo ${dest_port_str}|awk '{print $2}'`;
check_firewall_on_aix_internal $src_addr $src_operator $src_port_num $dest_addr $dest_operator $dest_port_num $port_opt
firewall_rc=$?
if [[ $firewall_rc -ne 0 ]]; then
if [ $src_ipaddr != "0.0.0.0" ] || [ $dest_ipaddr != "0.0.0.0" ];then
print_errmsg "${each_node}: Warning: Firewall of $rule_num_str is blocking RSCT comm.!!"
return $firewall_rc
fi
firewall_rc=1
fi
;;
Interface*) interface=`echo $line|cut -d: -f2`
;;
*) ;;
esac
done < ${TMPDIR}/tmp_lsfilt_file
rm -rf ${TMPDIR}/tmp_lsfilt_file
done
rm -rf tmp_lsfilt_file
return $firewall_rc
} # end of check_firewall_on_aix
function validate_ipv6_addr
{
ipaddr_v6=$1
set invalid_ip=0
if [[ -z "${ipaddr_v6##$EMPTY_ZERO_IPADDRV6*}" ]] ;then
invalid_ip=1
fi
if [[ -z "${ipaddr_v6##$LINK_LOCAL_IPV6*}" ]] ;then
invalid_ip=1
fi
return $invalid_ip
}
function validate_ipv4_addr
{
ipaddr_v4=$1
invalid_ip=0
if [ -z "${ipaddr_v4##$LOCAL_IPV4*}" ] ;then
invalid_ip=1
fi
if [[ ${ipaddr_v4} == ${EMPTY_ZERO_IPADDRV4} ]]
then
invalid_ip=1
fi
if [ -z "${ipaddr_v4##$LINK_LOCAL_IPV4*}" ] ;then
invalid_ip=1
fi
return $invalid_ip
}
################################################################################
# Function: get_subnet_id_from_mask
# This function generates subnet id from the given host ip address and netmask
# On AIX nodes netmask will be in hexadecimal [ for e.g 0xffffff00] form it will
# be converted to IP address form in this function.
# However on Linux, the new "ip" command will provide the CIDR prefix instead of netmask
# hence on Linux an equivalent netmask in IP address form will be generated and provided
# to this get_subnet_id_form_mask function.
# The subnet / network id will be calculated as below.
#
# IPAddress netmask subnet
# ip1.ip2.ip3.ip4 nm1.nm2.nm3.nm4 ((ip1 & nm1).(ip2 & nm2).(ip3 & nm3).(ip4 & nm4))
#
################################################################################
function get_subnet_id_from_mask
{
ipaddr=$1
# echo "ipaddr= ${ipaddr}"
# echo "nmask= ${nmask}"
if [[ ${BASEOS} == "AIX" ]]; then
nmask=$2
printf "%d.%d.%d.%d\n" `echo ${nmask} | sed 's/../ 0x&/g'`>${TMPDIR}/nmfile
nmask_ip_fmt=`cat ${TMPDIR}/nmfile`
rm ${TMPDIR}/nmfile
elif [[ ${BASEOS} == "Linux" ]]; then
nmask_ip_fmt=$2
fi
OLDIFS=$IFS
IFS='.'
set -A ip_arry ${ipaddr}
set -A nm_arry ${nmask_ip_fmt}
j=0
while [[ $j -lt 4 ]]
do
subnt_id[$j]=$(( ${ip_arry[$j]} & ${nm_arry[$j]} ))
j=$(($j+1))
done
subnet_id="${subnt_id[0]}.${subnt_id[1]}.${subnt_id[2]}.${subnt_id[3]}"
IFS=$OLDIFS
echo "${subnet_id}"
}
function compare_subnets
{
# Create a file /tmp/matched_subnet_lst_file for containing list of matched subnets.
# This file will be used for MTU size compatibility check.
rm -rf ${TMPDIR}/matched_subnet_lst_file
rm -rf ${TMPDIR}/unmatched_subnet_lst_file
compatible_count=0
for each_lcl_nm in `cat ${TMPDIR}/${first_node}_subnets`
do
lcl_nm="$each_lcl_nm"
match_count=0
for each_node in ${other_nodes[@]}
do
rc=0
matched_nm=`cat ${TMPDIR}/${each_node}_subnets | grep ${lcl_nm}`
rc=$?
if [[ $rc == 0 ]]; then
match_count=$(( $match_count + 1 ))
fi
done
if [[ $match_count == ${#other_nodes[@]} ]]; then
echo ${lcl_nm} >> ${TMPDIR}/matched_subnet_lst_file
compatible_count=$(( $compatible_count + 1 ))
else
echo ${lcl_nm} >> ${TMPDIR}/unmatched_subnet_lst_file
fi
done
if [[ $compatible_count == 0 ]];then
return ${NO_MATCHING_SUBNET_FOUND}
fi
return 0
}
function get_valid_ipv6_list
{
ipv6_file="${TMPDIR}/${first_node}_ipv6_list"
if [[ ${LOCAL_NODE} == ${first_node} ]] || [[ ${LOCAL_NODE_SHORT_NAME} == ${first_node} ]]
then
if [[ ${BASEOS} == "AIX" ]]; then
ifconfig -a |grep inet6 > ${ipv6_file}
elif [[ ${BASEOS} == "Linux" ]]; then
ip -6 a |grep inet6 > ${ipv6_file}
fi
ipaddr6_list=`cat ${ipv6_file} | awk '{print $2}'`
else
if [[ ${BASEOS} == "AIX" ]]; then
REMOTE_EXEC ${first_node} "ifconfig -a |grep inet6" > ${ipv6_file}
rc=$?
elif [[ ${BASEOS} == "Linux" ]]; then
REMOTE_EXEC ${first_node} "ip -6 a |grep inet6" > ${ipv6_file}
fi
if [[ $rc != 0 ]]
then
return ${REMOTE_NODE_CONN_REFUSED}
fi
ipaddr6_list=`cat ${ipv6_file} | awk '{print $2}'`
fi
valid_ip_cnt=0
for ipaddr_v6 in ${ipaddr6_list[@]}
do
$(validate_ipv6_addr ${ipaddr_v6})
valid=$?
if [[ ${valid} != 1 ]]
then
valid_ipaddr_v6_list[$valid_ip_cnt]="$ipaddr_v6"
valid_ip_cnt=$(($valid_ip_cnt + 1))
fi
done
if [[ $valid_ip_cnt == 0 ]]
then
if [[ $node_cnt > 1 ]]; then
echo "IPv6 is not configured or not available on ${first_node}, will not check IPv6 on other nodes"
fi
if [[ $node_cnt == 1 ]]; then
echo "IPv6 is not configured or not available on ${first_node}"
fi
return ${NO_VALID_IPV6_FOUND}
fi
## Now check remote nodes
each_node=""
for each_node in ${other_nodes[@]}
do
valid_ip_cnt=0
if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]
then
ipv6_file="${TMPDIR}/${each_node}_ipv6_list"
if [[ ${BASEOS} == "AIX" ]]; then
REMOTE_EXEC ${each_node} "ifconfig -a | grep inet6" > ${ipv6_file}
rc=$?
elif [[ ${BASEOS} == "Linux" ]]; then
REMOTE_EXEC ${each_node} "ip -6 a |grep inet6" > ${ipv6_file}
rc=$?
fi
if [[ $rc != 0 ]]
then
return ${REMOTE_NODE_CONN_REFUSED}
fi
else
ipv6_file="${TMPDIR}/${LOCAL_NODE_SHORT_NAME}_ipv6_list"
if [[ ${BASEOS} == "AIX" ]]; then
ifconfig -a |grep inet6 > ${ipv6_file}
elif [[ ${BASEOS} == "Linux" ]]; then
ip -6 a |grep inet6 > ${ipv6_file}
fi
fi
rmt_ipaddr6_list=`cat ${ipv6_file} | awk '{print \$2}'`
valid_ip_cnt=0
for ipaddr_v6 in ${rmt_ipaddr6_list[@]}
do
echo "ipaddr_v6=" ${ipaddr_v6}
$(validate_ipv6_addr ${ipaddr_v6})
valid=$?
if [[ ${valid} != 1 ]]
then
rmt_valid_ipaddr_v6_list[$valid_ip_cnt]="$ipaddr_v6"
valid_ip_cnt=$(($valid_ip_cnt + 1))
fi
done
if [[ $valid_ip_cnt -eq 0 ]]
then
echo "No valid IPv6 interfaces on ${each_node}"
return ${NO_VALID_IPV6_FOUND}
fi
done
}
function get_nmlist_from_iplist_aix
{
typeset t_input="$@"
set -A input $t_input
set -A tmp_nm_list
ip_cnt=${#input}
tmpfile=${input[0]}
set ifs_invalid=0
set valid_ip_cnt=0
i=0
while [[ $i < $ip_cnt ]]
do
ipaddr_list[$i]=${input[$i+1]}
i=$(($i+1))
done
for ipaddr_v4 in ${ipaddr_list[@]}
do
ifs_invalid=0
$(validate_ipv4_addr "${ipaddr_v4}")
ifs_invalid=$?
if [[ ${ifs_invalid} != 1 ]]
then
valid_ipaddr_list[$valid_ip_cnt]="$ipaddr_v4"
raw_nmask=`cat ${tmpfile} |grep -w ${ipaddr_v4} | awk '{print $4}' | cut -b 3-10`
subnet_id=$(get_subnet_id_from_mask "${ipaddr_v4}" "${raw_nmask}")
tmp_nm_list[$valid_ip_cnt]=${subnet_id}
valid_ip_cnt=$(( $valid_ip_cnt + 1 ))
fi
done
echo ${tmp_nm_list[@]}
return ${valid_ip_cnt}
}
function check_subnet_compat_v4_aix
{
set valid_ip_cnt=0
set rc=0
## get valid ipv4 addrs & netmasks on first node treating it as base
filename="${TMPDIR}/${first_node}_iplist"
if [[ ${LOCAL_NODE} == ${first_node} ]] || [[ ${LOCAL_NODE_SHORT_NAME} == ${first_node} ]]
then
ifconfig -a inet |grep netmask > ${filename}
ipaddr_list=`cat ${filename} |grep inet | awk '{print $2}'`
else
REMOTE_EXEC ${first_node} "ifconfig -a inet |grep netmask" > ${filename}
rc=$?
if [[ $rc != 0 ]]
then
return ${REMOTE_NODE_CONN_REFUSED}
fi
ipaddr_list=`cat ${filename} |grep inet | awk '{print \$2}'`
fi
nm_list=$(get_nmlist_from_iplist_aix ${filename} ${ipaddr_list[@]})
valid_ip_cnt=$?
set -A base_nm_list ${nm_list[@]}
if [[ $valid_ip_cnt -eq 0 ]]
then
print_errmsg "IPv4 is not configured or not available on ${first_node}."
return ${NO_VALID_IPV4_FOUND}
else
# echo "Valid IPv4 interfaces available on $first_node"
for each_subnet in ${base_nm_list[@]}
do
echo ${each_subnet} >> ${TMPDIR}/${first_node}_subnets
done
fi
## Now check nodes other than first
each_node=""
for each_node in ${other_nodes[@]}
do
filename="${TMPDIR}/${each_node}_iplist"
if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]
then
REMOTE_EXEC ${each_node} "ifconfig -a inet |grep netmask" > ${filename}
rc=$?
if [[ $rc != 0 ]]
then
print_dbgmsg "Connection refused by remote node ${each_node}."
return ${REMOTE_NODE_CONN_REFUSED}
fi
nxt_ipaddr_list=`cat ${filename} |grep inet | awk '{print \$2}'`
else
ifconfig -a inet |grep netmask > ${filename}
nxt_ipaddr_list=`cat ${filename} |grep inet | awk '{print $2}'`
fi
valid_ip_cnt=0
nm_lst=$(get_nmlist_from_iplist_aix ${filename} ${nxt_ipaddr_list[@]})
valid_ip_cnt=$?
set -A nxt_nm_list ${nm_lst[@]}
if [[ $valid_ip_cnt -eq 0 ]]
then
print_errmsg "IPv4 is not configured or not available on ${each_node}."
return ${NO_VALID_IPV4_FOUND}
else
# echo "Valid IPv4 interfaces available on ${each_node}"
for each_subnet in ${nxt_nm_list[@]}
do
echo ${each_subnet} >> ${TMPDIR}/${each_node}_subnets
done
fi
done
if [[ $node_cnt > 1 ]] ; then
compare_subnets
rc=$?
if [[ $rc == 0 ]]; then
print_errmsg "Compatible subnets between nodes ${nodes[@]} are "
uniq ${TMPDIR}/matched_subnet_lst_file > ${TMPDIR}/matched_subnets
for matched_subnet in `cat ${TMPDIR}/matched_subnets`
do
echo $matched_subnet
done
fi
if [[ -s ${TMPDIR}/unmatched_subnet_lst_file ]];then
uniq ${TMPDIR}/unmatched_subnet_lst_file > ${TMPDIR}/unmatched_subnets
for unmatched_subnet in `cat ${TMPDIR}/unmatched_subnets`
do
echo "Subnet $unmatched_subnet is not found on ${other_nodes[@]} "
done
fi
fi
return $rc
}
function get_nmlist_from_iplist_linux
{
typeset t_input="$@"
set -A ipaddr_list "$@"
set -A tmp_nm_list
set ifs_invalid=0
set valid_ip_cnt=0
for ipaddr_v4 in ${ipaddr_list[@]}
do
ifs_invalid=0
ipaddr=`echo $ipaddr_v4 |awk 'BEGIN{FS="/"} {print $1}'`
prefix=`echo $ipaddr_v4 |awk 'BEGIN{FS="/"} {print $2}'`
$(validate_ipv4_addr "${ipaddr}")
ifs_invalid=$?
if [[ ${ifs_invalid} != 1 ]]
then
valid_ipaddr_list[$valid_ip_cnt]="${ipaddr}"
ntmask=$(get_netmask_from_prefix $prefix)
subnet_id=$(get_subnet_id_from_mask "${ipaddr}" "${ntmask}")
tmp_nm_list[$valid_ip_cnt]=${subnet_id}
valid_ip_cnt=$(( $valid_ip_cnt + 1 ))
fi
done
echo ${tmp_nm_list[@]}
return ${valid_ip_cnt}
}
function check_subnet_compat_v4_linux
{
set valid_ip_cnt=0
set rc=0
## get valid ipv4 addrs & netmasks on local_node first
filename="${TMPDIR}/${first_node}_iplist"
if [[ ${LOCAL_NODE} == ${first_node} ]] || [[ ${LOCAL_NODE_SHORT_NAME} == ${first_node} ]]
then
ip -4 a > ${filename}
ipaddr_list=`cat ${filename} |grep inet | awk '{print $2}'`
else
REMOTE_EXEC ${first_node} "ip -4 a " > ${filename}
rc=$?
if [[ $rc != 0 ]]
then
return ${REMOTE_NODE_CONN_REFUSED}
fi
ipaddr_list=`cat ${filename} |grep inet | awk '{print \$2}'`
fi
nm_list=$(get_nmlist_from_iplist_linux ${ipaddr_list[@]})
valid_ip_cnt=$?
set -A base_nm_list ${nm_list[@]}
if [[ $valid_ip_cnt -eq 0 ]]
then
return ${NO_VALID_IPV4_FOUND}
else
print_dbgmsg "Valid IPv4 interfaces available on $first_node."
for each_subnet in ${base_nm_list[@]}
do
echo ${each_subnet} >> ${TMPDIR}/${first_node}_subnets
done
fi
## Now check remaining nodes
each_nodes=""
for each_node in ${other_nodes[@]}
do
# echo $each_node
# echo "LOCAL_NODE=" ${LOCAL_NODE} ", LOCAL_NODE_SHORT_NAME=" ${LOCAL_NODE_SHORT_NAME}
filename="${TMPDIR}/${each_node}_iplist"
if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]
then
REMOTE_EXEC ${each_node} "ip -4 a" > ${filename}
rc=$?
if [[ $rc != 0 ]]
then
return ${REMOTE_NODE_CONN_REFUSED}
fi
else
ip -4 a > ${filename}
ipaddr_list=`cat ${filename} |grep inet | awk '{print $2}'`
fi
nxt_ipaddr_list=`cat ${filename} |grep inet | awk '{print \$2}'`
# print_dbgmsg "ip addr list for $each_node = ${nxt_ipaddr_list[@]}"
valid_ip_cnt=0
nm_lst=$(get_nmlist_from_iplist_linux ${nxt_ipaddr_list[@]})
valid_ip_cnt=$?
set -A nxt_nm_list ${nm_lst[@]}
# print_dbgmsg "Next node nmlist = ${nxt_nm_list[@]}"
if [[ $valid_ip_cnt -eq 0 ]]
then
return ${NO_VALID_IPV4_FOUND}
else
print_dbgmsg "Valid IPv4 interfaces available on ${each_node}."
for each_subnet in ${nxt_nm_list[@]}
do
echo ${each_subnet} >> ${TMPDIR}/${each_node}_subnets
done
fi
done
if [[ $node_cnt > 1 ]] ; then
compare_subnets
rc=$?
if [[ $rc == 0 ]]; then
print_errmsg "Compatible subnets between ${first_node} and ${other_nodes[@]} are"
uniq ${TMPDIR}/matched_subnet_lst_file > ${TMPDIR}/matched_subnets
for matched_subnet in `cat ${TMPDIR}/matched_subnets`
do
echo $matched_subnet
done
fi
if [[ -s ${TMPDIR}/unmatched_subnet_lst_file ]]; then
uniq ${TMPDIR}/unmatched_subnet_lst_file > ${TMPDIR}/unmatched_subnets
for unmatched_subnet in `cat ${TMPDIR}/unmatched_subnets`
do
echo "Subnet $unmatched_subnet is not found on ${other_nodes[@]} "
done
fi
fi
return $rc
}
################################################################################
# Function: get_netmask_from_prefix
# This function generates netmask for the equivalent cidr prefix and is invoked
# only on Linux nodes.
#
#
# prefix netmask
# 8 255.0.0.0
# 16 255.255.0.0
# 24 255.255.255.0
#
# This function will also take care of any other custom prefix say 21 , in that
# case netmask will be 255.255.248.0
#
# input : prefix
# output : netmask
#
################################################################################
function get_netmask_from_prefix
{
prefix=$1
# echo $prefix
#
quo=$(($prefix / 8 ))
rem=$(($prefix % 8))
# echo "quo = $quo"
# echo "rem = $rem"
a=2
b=$((8-$rem))
val=`echo "$a^$b" |bc`
nm_last_octet=$((256-$val))
if [[ $quo > 0 ]] then
nmask="255"
fi
i=1
while [[ $i < $quo ]]
do
nm=".255"
nmask=$nmask${nm}
i=$(($i+1))
done
if [[ $quo < 4 && $rem != 0 ]]
then
nmask="$nmask.${nm_last_octet}"
fi
cnt=0
OLDIFS=$IFS
IFS='.'
for word in $nmask
do
cnt=$(($cnt+1))
done
IFS=$OLDIFS
##echo $cnt
while [[ $cnt < 4 ]]
do
last_octet=".0"
nmask=$nmask$last_octet
cnt=$(($cnt+1))
done
echo "${nmask}"
}
function check_conn_path
{
typeset v4_rc=0
typeset v6_rc=0
conn_opt=$1
if [[ $conn_opt == '4' ]] ; then
print_errmsg "Performing Subnet availability (IPv4) checks for PeerDomain"
if [[ ${BASEOS} == "AIX" ]]; then
check_subnet_compat_v4_aix
rc=$?
elif [[ ${BASEOS} == "Linux" ]]; then
check_subnet_compat_v4_linux
rc=$?
fi
fi
if [[ $conn_opt == '6' ]]; then
print_errmsg "Looking for interfaces (IPv6) availability on nodes ${nodes[@]}"
get_valid_ipv6_list
rc=$?
fi
if [[ $conn_opt == 'a' ]]; then
print_errmsg "Performing Subnet availability (IPv4) checks for PeerDomain"
if [[ ${BASEOS} == "AIX" ]]; then
check_subnet_compat_v4_aix
v4_rc=$?
elif [[ ${BASEOS} == "Linux" ]]; then
check_subnet_compat_v4_linux
v4_rc=$?
fi
print_errmsg "Looking for interfaces (IPv6) availability on nodes ${nodes[@]}"
get_valid_ipv6_list
v6_rc=$?
if [[ $v4_rc != 0 && $v6_rc != 0 ]]
then
# print_errmsg "No valid connectivity found"
# return ${NO_VALID_IP_FOUND}
rc=$(($v4_rc || $v6_rc))
fi
fi
return $rc
}
function do_cleanup
{
# remove all the files that got created during ifconfig read
for each_node in $nodes
do
filename1="${TMPDIR}/${each_node}_iplist"
filename2="${TMPDIR}/${each_node}_ipv6_list"
filename3="${TMPDIR}/${each_node}_nmlist"
filename4="${TMPDIR}/${each_node}_subnets"
filename5="${TMPDIR}/matched_subnet_lst_file"
filename6="${TMPDIR}/unmatched_subnet_lst_file"
if [[ -f ${filename1} ]]; then
rm ${filename1}
fi
if [[ -f ${filename2} ]]; then
rm ${filename2}
fi
if [[ -f ${filename3} ]]; then
rm ${filename3}
fi
if [[ -f ${filename4} ]]; then
rm ${filename4}
fi
if [[ -f ${filename5} ]]; then
rm ${filename5}
fi
if [[ -f ${filename6} ]]; then
rm ${filename6}
fi
done
}
function check_port_available
{
rc=0
port_opt=$1
tmp_node_cnt=0
error_node_id=""
for each_node in $nodes
do
tmp_node_cnt=$(($tmp_node_cnt+1))
# print_dbgmsg "Checking connectivity with node $each_node using $CT_SYSCHK_SHELL..."
REMOTE_EXEC $each_node "hostname" 1>/dev/null
if [[ $? -ne 0 ]];then
return $REMOTE_NODE_CONN_REFUSED
fi
done
if [[ $tmp_node_cnt -eq 0 ]];then
nodes[0]=$LOCAL_NODE
fi
for each_node in $nodes
do
echo "Checking node $each_node for port availability..."
IS_RPD_UP=0
if [[ ${port_opt} != 'r' ]]; then
RPD_ONLINE=$((REMOTE_EXEC $each_node "lsrpdomain -xo 2>/dev/null")|tr -d '\r')
if [[ -n $RPD_ONLINE ]]; then
IS_RPD_UP=1
print_dbgmsg "RPD is online on ${each_node} . Over-riding CT_SYSCHK_TS_PORT and CT_SYSCHK_GS_PORT env. vars. if any."
CT_SYSCHK_TS_PORT=$(echo ${RPD_ONLINE}|awk '{print $5}')
CT_SYSCHK_GS_PORT=$(echo ${RPD_ONLINE}|awk '{print $6}')
fi
fi
# Check whether any ports are configured to be used by any other process
# via /etc/services entries
# etc_services_test_start
rm -rf ${TMPDIR}/etc_services_out
REMOTE_EXEC $each_node "cat /etc/services|egrep -w '${CT_SYSCHK_RMC_PORT}|${CT_SYSCHK_TS_PORT}|${CT_SYSCHK_GS_PORT}'" 1>${TMPDIR}/etc_services_out
RMC_PORT_BOUND_BY_OTHER=$(grep -w "${CT_SYSCHK_RMC_PORT}" ${TMPDIR}/etc_services_out|egrep 'udp|tcp'|grep -vw rmc|grep -v ^#)
TS_PORT_BOUND_BY_OTHER=$(grep -w "${CT_SYSCHK_TS_PORT}" ${TMPDIR}/etc_services_out|egrep 'udp|tcp'|grep -vw cthats|grep -v ^#)
GS_PORT_BOUND_BY_OTHER=$(grep -w "${CT_SYSCHK_GS_PORT}" ${TMPDIR}/etc_services_out|egrep 'udp|tcp'|grep -vw cthags|grep -v ^#)
rm -rf ${TMPDIR}/etc_services_out
case "$port_opt" in
r)
if [[ -n "${RMC_PORT_BOUND_BY_OTHER}" ]];then
# print_errmsg "$each_node : "
error_node_id=$each_node
return ${RMC_PORT_BOUND_BY_OTHER_RC}
fi
;;
s)
if [[ -n "${TS_PORT_BOUND_BY_OTHER}" ]];then
# print_errmsg "$each_node : "
error_node_id=$each_node
return ${TS_PORT_BOUND_BY_OTHER_RC}
fi
if [[ -n "${GS_PORT_BOUND_BY_OTHER}" ]];then
# print_errmsg "$each_node : "
error_node_id=$each_node
return ${GS_PORT_BOUND_BY_OTHER_RC}
fi
;;
a)
if [[ -n "${RMC_PORT_BOUND_BY_OTHER}" ]];then
# print_errmsg "$each_node : "
error_node_id=$each_node
return ${RMC_PORT_BOUND_BY_OTHER_RC}
fi
if [[ -n "${TS_PORT_BOUND_BY_OTHER}" ]];then
# print_errmsg "$each_node : "
error_node_id=$each_node
return ${TS_PORT_BOUND_BY_OTHER_RC}
fi
if [[ -n "${GS_PORT_BOUND_BY_OTHER}" ]];then
# print_errmsg "$each_node : "
error_node_id=$each_node
return ${GS_PORT_BOUND_BY_OTHER_RC}
fi
;;
esac
# etc_services_test_end
#Initialize variables with blank strings
CT_SYSCHK_RMC_PORT_TCP_OUT=""
CT_SYSCHK_GS_PORT_OUT=""
CT_SYSCHK_TS_PORT_OUT=""
CT_SYSCHK_GS_PORT_HELD_BY_OTHER=""
CT_SYSCHK_TS_PORT_HELD_BY_OTHER=""
unset OS_NAME
OS_NAME=$((REMOTE_EXEC $each_node "uname")|tr -d '\r'|tr -d '\n')
# On HMC, PATH variable cannot be set, so it will give output like this "OS_NAME='/bin/bash: PATH: readonly variableLinux'"
echo $OS_NAME|grep "readonly variableLinux" 1>/dev/null
if [[ $? -eq 0 ]];then
OS_NAME="Linux"
fi
if [[ ${OS_NAME} == "Linux" ]]; then
# Is it HMC? If yes, then RMC port check can be done with the help of lshmc --firewall command (via hscroot user)
LSHSC_EXISTS=$(REMOTE_EXEC $each_node "ls $LSHSC 2>/dev/null")
if [[ $CLUSTER_SCOPE == "m" ]] && [[ $LSHSC_EXISTS == $LSHSC ]]; then
# the node is HMC so run lshmc command to get firewall details
CT_SYSCHK_RMC_PORT_TCP_OUT=$((REMOTE_EXEC $each_node "export PATH=/hmcrbin/:/usr/hmcrbin:$PATH; lshmc --firewall|grep \"application=RMC.*ports=${CT_SYSCHK_RMC_PORT}:udp,tcp:${CT_SYSCHK_RMC_PORT}.*allowedhost=0.0.0.0/0.0.0.0\"|grep -v 255|grep -v ^$|tail -1")|tr -d '\r'|tr -d '\n')
else
CT_SYSCHK_RMC_PORT_TCP_OUT=$((REMOTE_EXEC $each_node "ss -anp|grep tcp.*${CT_SYSCHK_RMC_PORT}.*rmc")|tr -d '\r'|tr -d '\n')
fi
rm -rf ${TMPDIR}/SS_ANP_OUT
REMOTE_EXEC $each_node "ss -anp|egrep '${CT_SYSCHK_GS_PORT}|${CT_SYSCHK_TS_PORT}'" > ${TMPDIR}/SS_ANP_OUT
CT_SYSCHK_GS_PORT_OUT=$(grep udp.*${CT_SYSCHK_GS_PORT}.*hagsd ${TMPDIR}/SS_ANP_OUT)
CT_SYSCHK_GS_PORT_HELD_BY_OTHER=$(grep udp.*${CT_SYSCHK_GS_PORT} ${TMPDIR}/SS_ANP_OUT|grep -v hagsd|grep -v ^$|tail -1)
CT_SYSCHK_TS_PORT_OUT=$(grep udp.*${CT_SYSCHK_TS_PORT}.*hats_nim ${TMPDIR}/SS_ANP_OUT|grep -v ^$|tail -1)
CT_SYSCHK_TS_PORT_HELD_BY_OTHER=$(grep udp.*${CT_SYSCHK_TS_PORT} ${TMPDIR}/SS_ANP_OUT|grep -v hats_nim|grep -v ^$|tail -1)
rm -rf ${TMPDIR}/SS_ANP_OUT
elif [[ ${OS_NAME} == "AIX" ]]; then
#RMC port
RMC_TCP_SOCKADDR=$((REMOTE_EXEC $each_node "netstat -Aan | grep -w ${CT_SYSCHK_RMC_PORT} | grep tcp|grep LISTEN|head -1 | cut -f1 -d ' '")|tr -d '\r'|tr -d '\n')
if [[ -n $RMC_TCP_SOCKADDR ]];then
CT_SYSCHK_RMC_PORT_TCP_ON=$(REMOTE_EXEC $each_node "rmsock $RMC_TCP_SOCKADDR tcpcb")
CT_SYSCHK_RMC_PORT_TCP_OUT=$(echo $CT_SYSCHK_RMC_PORT_TCP_ON|grep "The socket.*is being held by proccess.*(rmcd)")
fi
#HAGS port
GS_UDP_SOCKADDR=$((REMOTE_EXEC $each_node "netstat -Aan | grep -w ${CT_SYSCHK_GS_PORT} | grep udp |grep -v ^$|tail -1| cut -f1 -d ' '")|tr -d '\r'|tr -d '\n')
if [[ -n $GS_UDP_SOCKADDR ]]; then
CT_SYSCHK_GS_PORT_UDP_ON=$(REMOTE_EXEC $each_node "rmsock $GS_UDP_SOCKADDR inpcb")
CT_SYSCHK_GS_PORT_OUT=$(echo $CT_SYSCHK_GS_PORT_UDP_ON|grep "The socket.*is being held by proccess.*(hagsd)")
CT_SYSCHK_GS_PORT_HELD_BY_OTHER=$(echo $CT_SYSCHK_GS_PORT_UDP_ON|grep "The socket.*is being held by proccess.*"|grep -vw hagsd|grep -v ^$|tail -1)
fi
#HATS port
TS_UDP_SOCKADDR=$((REMOTE_EXEC $each_node "netstat -Aan | grep -w ${CT_SYSCHK_TS_PORT} | grep udp |grep -v ^$|tail -1| cut -f1 -d ' '")|tr -d '\r'|tr -d '\n')
if [[ -n $TS_UDP_SOCKADDR ]]; then
CT_SYSCHK_TS_PORT_UDP_ON=$(REMOTE_EXEC $each_node "rmsock $TS_UDP_SOCKADDR inpcb")
CT_SYSCHK_TS_PORT_OUT=$(echo $CT_SYSCHK_TS_PORT_UDP_ON|grep "The socket.*is being held by proccess.*(hats_nim)")
CT_SYSCHK_TS_PORT_HELD_BY_OTHER=$(echo $CT_SYSCHK_TS_PORT_UDP_ON|grep "The socket.*is being held by proccess.*"|grep -vw hats_nim|grep -v ^$|tail -1)
fi
else
# print_errmsg "Unknown OS!!"
# print_errmsg "${each_node} : "
return ${UNSUPPORTED_OS_RC}
fi
case "$port_opt" in
r)
# If RMC is active, then check for port 657
if [ -n "${CT_SYSCHK_RMC_PORT_TCP_OUT}" ]; then
print_dbgmsg "SUCCESS: RMC is listening at port ${CT_SYSCHK_RMC_PORT} on ${each_node}."
rc=0
else
# print_errmsg "$each_node : "
error_node_id=$each_node
return ${RMC_NOT_LISTENING_RC}
fi
;;
s)
print_dbgmsg "Checking HAGS and HATS ports for availability..."
if [[ ${IS_RPD_UP} -eq 1 ]]; then
if [[ -n "${CT_SYSCHK_GS_PORT_OUT}" ]];then
print_dbgmsg "SUCCESS, Port ${CT_SYSCHK_GS_PORT} is being used by an online RPD on ${each_node} !!!"
else
# print_errmsg "Warning:Port ${CT_SYSCHK_GS_PORT} is not being used for Group Services!!"
# print_errmsg "$each_node : "
return ${GS_NOT_LISTENING_RC}
fi
elif [[ -n "${CT_SYSCHK_GS_PORT_HELD_BY_OTHER}" ]];then
# print_errmsg "Warning:Port ${CT_SYSCHK_GS_PORT} is not being used for Group Services!!"
#print_errmsg "$each_node : "
return ${GS_PORT_HELD_BY_OTHER_RC}
else
print_dbgmsg "SUCCESS: Port ${CT_SYSCHK_GS_PORT} is available on ${each_node}"
fi
if [[ ${IS_RPD_UP} -eq 1 ]]; then
if [[ -n "${CT_SYSCHK_TS_PORT_OUT}" ]];then
print_dbgmsg "SUCCESS, Port ${CT_SYSCHK_TS_PORT} is being used by an online RPD on ${each_node} !!!"
else
# print_errmsg "Warning:Port ${CT_SYSCHK_TS_PORT} is not being used for topology Services!!"
# print_errmsg "$each_node : "
return ${TS_NOT_LISTENING_RC}
fi
elif [[ -n "${CT_SYSCHK_TS_PORT_HELD_BY_OTHER}" ]];then
# print_errmsg "Warning:Port ${CT_SYSCHK_TS_PORT} is not being used for topology Services!!"
#print_errmsg "$each_node : "
return ${TS_PORT_HELD_BY_OTHER_RC}
else
print_dbgmsg "SUCCESS: Port ${CT_SYSCHK_TS_PORT} is available on ${each_node}"
fi
;;
a)
# If RMC is active, then check for port 657
if [ -n "${CT_SYSCHK_RMC_PORT_TCP_OUT}" ]; then
print_dbgmsg "SUCCESS: RMC is listening at port ${CT_SYSCHK_RMC_PORT} on ${each_node}."
else
# print_errmsg "RMC is not listening at port ${CT_SYSCHK_RMC_PORT}"
# print_errmsg "$each_node : "
error_node_id=${each_node}
return ${RMC_NOT_LISTENING_RC}
fi
if [[ ${IS_RPD_UP} -eq 1 ]]; then
if [[ -n "${CT_SYSCHK_GS_PORT_OUT}" ]];then
print_dbgmsg "SUCCESS, Port ${CT_SYSCHK_GS_PORT} is being used by an online RPD on ${each_node} !!!"
else
# print_errmsg "Warning:Port ${CT_SYSCHK_GS_PORT} is not being used for Group Services!!"
# print_errmsg "$each_node : "
return ${GS_PORT_HELD_BY_OTHER_RC}
fi
elif [[ -n "${CT_SYSCHK_GS_PORT_HELD_BY_OTHER}" ]];then
# print_errmsg "Warning:Port ${CT_SYSCHK_GS_PORT} is not being used for Group Services!!"
#print_errmsg "$each_node : "
return ${GS_PORT_HELD_BY_OTHER_RC}
else
print_dbgmsg "SUCCESS: Port ${CT_SYSCHK_GS_PORT} is available on ${each_node} "
fi
if [[ ${IS_RPD_UP} -eq 1 ]]; then
if [[ -n "${CT_SYSCHK_TS_PORT_OUT}" ]];then
print_dbgmsg "SUCCESS, Port ${CT_SYSCHK_TS_PORT} is being used by an online RPD on ${each_node} !!!"
else
# print_errmsg "Warning:Port ${CT_SYSCHK_TS_PORT} is not being used for topology Services!!"
# print_errmsg "$each_node : "
return ${TS_PORT_HELD_BY_OTHER_RC}
fi
elif [[ -n "${CT_SYSCHK_TS_PORT_HELD_BY_OTHER}" ]];then
# print_errmsg "Warning:Port ${CT_SYSCHK_TS_PORT} is not being used for topology Services!!"
#print_errmsg "$each_node : "
return ${TS_PORT_HELD_BY_OTHER_RC}
else
print_dbgmsg "SUCCESS: Port ${CT_SYSCHK_TS_PORT} is available on ${each_node}."
fi
;;
*) echo "Warning: WRONG OPTION FOR PORT CHECK !!"
return ${WRONG_OPT_FOR_PORT_CHECK_RC}
;;
esac
#######
## Check iptables command for blocked port on Linux on LPARs (NOT HMC)
# # iptables -L
# Chain INPUT (policy ACCEPT)
# target prot opt source destination
# all -- anywhere anywhere
#
# Chain FORWARD (policy ACCEPT)
# target prot opt source destination
# REJECT tcp -- anywhere anywhere tcp dpt:2222 reject-with tcp-reset
# REJECT tcp -- anywhere anywhere tcp spt:2221 reject-with tcp-reset
#
# Chain OUTPUT (policy ACCEPT)
# target prot opt source destination
firewall_check_rc=0
if [[ ${OS_NAME} == "Linux" ]]; then
if [[ $CLUSTER_SCOPE == "m" ]] && [[ $LSHSC_EXISTS == $LSHSC ]]; then
port_with_firewall=$((REMOTE_EXEC $each_node "export PATH=/hmcrbin/:/usr/hmcrbin:$PATH; lshmc --firewall|grep \"application=RMC.*ports=${CT_SYSCHK_RMC_PORT}:udp,tcp:${CT_SYSCHK_RMC_PORT}.*allowedhost=0.0.0.0/0.0.0.0\"|grep 255|grep -v ^$|tail -1")|tr -d '\r'|tr -d '\n')
if [[ -n "${port_with_firewall}" ]];then
return ${FIREWALL_CHECK_FAILED_RC}
fi
else
rm -rf ${TMPDIR}/IPTABLES_OUTFILE
REMOTE_EXEC ${each_node} "iptables -L 2>/dev/null"|egrep -i "reject|drop"|egrep "${CT_SYSCHK_GS_PORT}|${CT_SYSCHK_TS_PORT}|${CT_SYSCHK_RMC_PORT}|rmc"|awk '{print $7}' >${TMPDIR}/IPTABLES_OUTFILE
egrep -i "${CT_SYSCHK_GS_PORT}|${CT_SYSCHK_TS_PORT}|${CT_SYSCHK_RMC_PORT}|rmc" ${TMPDIR}/IPTABLES_OUTFILE
if [[ $? -eq 0 && -s ${TMPDIR}/IPTABLES_OUTFILE ]];then
for line in `cat ${TMPDIR}/IPTABLES_OUTFILE`
do
port_type=$(echo $line|cut -d':' -f1)
port_num=$(echo $line|cut -d':' -f2)
if [[ $port_type = "spt" ]];then
port_type="Source Port"
else
port_type="Destination Port"
echo $each_node
fi
return ${FIREWALL_CHECK_FAILED_RC}
done
fi
rm -rf ${TMPDIR}/IPTABLES_OUTFILE
firewall_check_rc=0
fi
elif [[ ${BASEOS} == "AIX" ]]; then
IsSecureAIX=$((REMOTE_EXEC $each_node "aixpert -t 2>/dev/null|grep 'Applied Profiles'|egrep 'HLS|MLS|LLS|DLS|SCBPS'")|tr -d '\r'|tr -d '\n')
if [[ -n $IsSecureAIX ]]; then
# Secure AIX test start
rm -rf ${TMPDIR}/LSFILT_OUTFILE
REMOTE_EXEC $each_node "lsfilt -O 2>/dev/null|grep permit" > ${TMPDIR}/LSFILT_OUTFILE
if [[ -s ${TMPDIR}/LSFILT_OUTFILE ]]; then
IS_RMC_PORT_PERMITTED=$(grep "eq|${CT_SYSCHK_RMC_PORT}|both|both" ${TMPDIR}/LSFILT_OUTFILE|grep "0.0.0.0|0.0.0.0")
IS_GS_PORT_PERMITTED=$(grep "eq|${CT_SYSCHK_GS_PORT}|both|both" ${TMPDIR}/LSFILT_OUTFILE|grep "0.0.0.0|0.0.0.0")
IS_TS_PORT_PERMITTED=$(grep "eq|${CT_SYSCHK_TS_PORT}|both|both" ${TMPDIR}/LSFILT_OUTFILE|grep "0.0.0.0|0.0.0.0")
else
IS_RMC_PORT_PERMITTED=""
IS_GS_PORT_PERMITTED=""
IS_TS_PORT_PERMITTED=""
fi
rm -rf ${TMPDIR}/LSFILT_OUTFILE
firewall_check_rc=0
case "$port_opt" in
r)
if [[ -z $IS_RMC_PORT_PERMITTED ]]; then
print_errmsg "$each_node: Warning: Please set filter rules to allow bi-directional RSCT communication through port: ${CT_SYSCHK_RMC_PORT}"
firewall_check_rc=1
fi
;;
s)
if [[ -z $IS_GS_PORT_PERMITTED ]] || [[ -z $IS_TS_PORT_PERMITTED ]]; then
print_errmsg "$each_node: Warning: Please set filter rules to allow bi-directional RSCT communication through ports: ${CT_SYSCHK_GS_PORT} and ${CT_SYSCHK_TS_PORT}!!"
firewall_check_rc=1
fi
;;
a)
if [[ -z $IS_RMC_PORT_PERMITTED ]] || [[ -z $IS_GS_PORT_PERMITTED ]] || [[ -z $IS_TS_PORT_PERMITTED ]]; then
print_errmsg "$each_node: Warning: Please set filter rules to allow bi-directional RSCT communication through ports: ${CT_SYSCHK_RMC_PORT}, ${CT_SYSCHK_GS_PORT} and ${CT_SYSCHK_TS_PORT}!!"
firewall_check_rc=1 # At-least one of the ports is blocked
fi
;;
esac
# Secure AIX test end
else
check_firewall_on_aix $port_opt
firewall_check_rc=$?
fi
if [[ $firewall_check_rc -ne 0 ]]; then
return ${FIREWALL_CHECK_FAILED_RC}
else
firewall_check_rc=0
fi
else
echo "OS: ${OS_NAME} not supported or recognized!!"
return ${UNSUPPORTED_OS_RC}
fi
done
return $rc
}
################################################################################
# Function: check_mtu_size
# This function checks whether specified MTU size is supported by RSCT for communication between
# cluster nodes
# Input: 1. None
# Output: None
# Return values:
# 0 : MTU size is supported by RSCT
# Non-zero : MTU size is not supported by RSCT
################################################################################
function check_mtu_size
{
print_errmsg "Performing MTU compatibility check."
tmp_node_cnt=0
for each_node in $nodes
do
tmp_node_cnt=$(($tmp_node_cnt+1))
# print_dbgmsg "Checking connectivity with node $each_node using $CT_SYSCHK_SHELL..."
REMOTE_EXEC $each_node "hostname" 1>/dev/null
if [[ $? -ne 0 ]];then
return $REMOTE_NODE_CONN_REFUSED
fi
done
if [[ $tmp_node_cnt -le 1 ]];then
print_errmsg "Performing MTU compatibility check - Done. Success"
return $MTU_COMPAT_SUCCESS
fi
# Assume return-code as success initially
rc=$MTU_COMPAT_SUCCESS
# File "${TMPDIR}/matched_subnet_lst_file" is filled with matching subnet entries during
# subnet compatibility check.
# If file does not exist, run check_conn_path function and get entries in file
# else just use the file for MTU compatibility check
mtu_sz_compat_file="${TMPDIR}/matched_subnets"
if [ ! -s ${mtu_sz_compat_file} ];then
if [ $COMMOPT == '4' ] || [ $COMMOPT == 'a' ]; then
# if there are matching subnet then only check for MTU size compatibility
check_conn_path 4
conn_path_rc_4_mtu=$?
if [[ $conn_path_rc_4_mtu -eq 0 ]]; then
if [[ ${BASEOS} == "AIX" ]]; then
check_subnet_compat_v4_aix
v4_rc=$?
elif [[ ${BASEOS} == "Linux" ]]; then
check_subnet_compat_v4_linux
v4_rc=$?
fi
else
print_errmsg "MTU test: No matching subnets!!";
rc=$MTU_COMPAT_CHECK_FAILED
fi
else
return $MTU_NO_OP_4_IPV6
fi
fi
if [ $rc -eq $MTU_NO_OP_4_IPV6 ] || [ $rc -eq $MTU_COMPAT_CHECK_FAILED ]; then
return $rc
fi
if [[ -s ${mtu_sz_compat_file} ]];then
uniq ${mtu_sz_compat_file} > ${TMPDIR}/mtu_compat_file
#Now ${TMPDIR}/mtu_compat_file contains unique compatible subnets. Remove old file
rm -rf ${mtu_sz_compat_file}
fi
first=1
MTUSZ=0 # Just initialize MTUSZ which is going to be compared with the other nodes' MTU
mtu_match_cnt=0 # Initially match count is 0
for each_node in $nodes
do
cmd_for_mtu="netstat -in|grep -vw sit|grep -vw lo0"
REMOTE_EXEC $each_node "$cmd_for_mtu" > ${TMPDIR}/mtu_internal_file_$each_node
done
#netmask number
nm_cnt=0
for line in `cat ${TMPDIR}/mtu_compat_file 2>/dev/null` # Start loop; for each netmask in the file,
do
nm_cnt=$(($nm_cnt+1))
# Make netmask to compare with netstat outputs
octet1=`echo $line|cut -d'.' -f1`
octet2=`echo $line|cut -d'.' -f2`
octet3=`echo $line|cut -d'.' -f3`
# If 3rd octet is also 0, try grepping only first 2 octets
if [[ $octet3 -eq 0 ]];then
if [[ $octet2 -eq 0 ]];then
nm_to_compare=$octet1\.
else
nm_to_compare=$octet1.$octet2\.
fi
else
nm_to_compare=$octet1.$octet2.$octet3\.
fi
first=1
unmatched_mtu_cnt=0
for each_node in $nodes
do
unset OS_NAME
OS_NAME=$((REMOTE_EXEC $each_node "uname")|tr -d '\r'|tr -d '\n')
# On HMC, PATH variable cannot be set, so it will give output like this "OS_NAME='/bin/bash: PATH: readonly variableLinux'"
echo $OS_NAME|grep "readonly variableLinux" 1>/dev/null
if [[ $? -eq 0 ]];then
OS_NAME="Linux"
fi
if [[ ${OS_NAME} == "AIX" ]]; then
mtu_sz=$(grep -w $nm_to_compare ${TMPDIR}/mtu_internal_file_$each_node|grep -v ^$|tail -1|awk '{print $2}')
elif [[ ${OS_NAME} == "Linux" ]]; then
# For Linux, additionally one more cmd is to be run because netstat -in does not give MTU
interface_name=$((REMOTE_EXEC $each_node "ip -4 a")|grep ${nm_to_compare}|grep -v ^$|tail -1|awk '{print $NF}'|tr -d '\r'|tr -d '\n')
if [[ ! -z $interface_name ]]; then
mtu_sz=$(grep -w $interface_name ${TMPDIR}/mtu_internal_file_$each_node|awk '{print $2}')
fi
else
mtu_cleanup
return $UNSUPPORTED_OS_RC
fi
# In first iteration, capture the MTU size of first node's interface
# which is being verified with other nodes' interfaces.
if [[ $first -eq 1 ]];then
MTUSZ=${mtu_sz}
first=0
continue
fi
if [[ $mtu_sz -eq $MTUSZ ]]; then
mtu_match_cnt=$(($mtu_match_cnt+1))
else
unmatched_mtu_cnt=$(($unmatched_mtu_cnt+1))
echo ${each_node}:MTU=$mtu_sz>> ${TMPDIR}/unmatched_mtu
echo ${each_node}:MTU=$mtu_sz>> ${TMPDIR}/${each_node}_mtu_mismatch
fi
done
if [[ $unmatched_mtu_cnt -gt 0 ]]; then
cat ${TMPDIR}/unmatched_mtu|tr '\n' ' ' > ${TMPDIR}/unmatched_mtu.nodes_$nm_cnt
rm -rf ${TMPDIR}/unmatched_mtu
echo "Warning: MTU mismatch for matching subnet: ${line} between node ${first_node}:MTU=$MTUSZ and `cat ${TMPDIR}/unmatched_mtu.nodes_$nm_cnt`"
fi
# Is every node's interface having same MTU ?
if [[ $mtu_match_cnt -eq $(($tmp_node_cnt-1)) ]]; then
rc=$MTU_COMPAT_SUCCESS
fi
done
# End loop; for each netmask in the file
#Now check if a node has entry in all the unmatched mtu nodes_$nm_cnt files, then the node is incompatible.
first=0
for each_node in $nodes
do
if [[ $first -eq 0 ]];then
first=$(($first+1))
mtu_first_node=$each_node
continue;
fi
if [[ -s ${TMPDIR}/${each_node}_mtu_mismatch ]]; then
unmatch_mtu_cnt_4_node=$(wc -l ${TMPDIR}/${each_node}_mtu_mismatch |awk '{print $1}')
if [[ $nm_cnt -eq $unmatch_mtu_cnt_4_node ]];then
print_errmsg "Error: ${mtu_first_node} and $each_node don't have any matching subnets with matching MTU sizes!!"
rc=$MTU_COMPAT_CHECK_FAILED
fi
fi
rm -rf ${TMPDIR}/unmatched_mtu.nodes_$nm_cnt
done
mtu_cleanup
if [[ $mtu_match_cnt -eq 0 ]];then
rc=$MTU_COMPAT_CHECK_FAILED
fi
return $rc
}
function mtu_cleanup {
rm -rf ${TMPDIR}/mtu_internal_file_*
rm -rf ${TMPDIR}/mtu_compat_file
}
function check_node_id
{
print_errmsg "Performing Node Id checks."
rm -rf ${TMPDIR}/node_id_*
for each_node in $nodes
do
rm -rf ${TMPDIR}/node_id_out
rm -rf ${TMPDIR}/node_id_err
REMOTE_EXEC ${each_node} "head -n 1 ${NODE_ID_FILE}" 1>${TMPDIR}/node_id_out 2>${TMPDIR}/node_id_err
# Possibilities: 1. ssh/rsh failed. 2. ssh passes but cmd fails.
if [ -s ${TMPDIR}/node_id_out ];then
echo "${each_node}:`cat ${TMPDIR}/node_id_out|tr -d '\r'|tr -d '\n'`" >> ${TMPDIR}/node_id_with_nodename_all
else
grep ${each_node} ${TMPDIR}/node_id_err
if [[ $? -eq 0 ]]; then
# meaning that error contains node name.
return ${REMOTE_NODE_CONN_REFUSED}
else
error_node_id=${each_node}
print_errmsg "${NODE_ID_FILE} does not exist on ${each_node}!!"
fi
fi
done
dup_node_id_found=0
if [ -s ${TMPDIR}/node_id_with_nodename_all ];then
cat ${TMPDIR}/node_id_with_nodename_all|cut -d':' -f2|sort > ${TMPDIR}/node_id_out_all
for entry in `uniq -d ${TMPDIR}/node_id_out_all`
do
dup_node_id_found=1
print_errmsg "Duplicate Node ID's found."
# grep $entry ${TMPDIR}/node_id_with_nodename_all|cut -d':' -f1|tr -s '\n' ' '
grep $entry ${TMPDIR}/node_id_with_nodename_all
done
fi
rm -rf ${TMPDIR}/node_id_*
if [[ ${dup_node_id_found} -eq 1 ]];then
# print_errmsg "The node ids are duplicated between nodes"
return ${NODE_ID_NOT_UNIQUE}
fi
# print_errmsg "Success. Node ID's are unique between ${nodes[@]}"
print_errmsg "Node Id checks - Done. All nodes have unique Node ID's."
return ${NODE_ID_SUCCESS}
}
function check_session_scope
{
UNAUTHENT_CMD="lsrsrc IBM.HostPublic PublicKey"
AUTHENT_CMD="lsrsrc IBM.Host"
if [[ $SESSOPT == 'u' ]]; then
for each_node in $nodes
do
if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]; then
OUT=`REMOTE_EXEC $each_node $UNAUTHENT_CMD`
rc=$?
if echo $OUT |grep -q "PublicKey = "
then
print_dbgmsg "Able to open unauthenticated session with $each_node."
fi
fi
done
fi
if [[ $SESSOPT == 'a' ]]; then
for each_node in $nodes
do
if [[ ${LOCAL_NODE} != ${each_node} ]] && [[ ${LOCAL_NODE_SHORT_NAME} != ${each_node} ]]; then
OUT=`REMOTE_EXEC $each_node $AUTHENT_CMD`
rc=$?
if echo $OUT |grep -q "Name "
then
print_dbgmsg "Able to open authenticated session with $each_node."
fi
fi
done
fi
return $rc
}
function check_peerdomain_req
{
rc=0
dflt_commopt='a'
print_errmsg "Checking PeerDomain pre-req's .."
# step-1 : check interfaces availability
check_conn_path "$dflt_commopt"
conn_chk_rc=$?
if [[ $conn_chk_rc != $CONN_PATH_SUCCESS ]]
then
return ${conn_chk_rc}
fi
if [[ -z $PORTOPT ]];then
PORTOPT='a'
fi
# step-2 : check port availability
print_errmsg "Performing port availability checks"
check_port_available $PORTOPT
port_chk_rc=$?
if [[ $port_chk_rc -ne $PORT_CHECK_SUCCESS ]];then
port_chk_failure_node=${each_node}
else
print_errmsg "RMC port (${CT_SYSCHK_RMC_PORT}) is available on all nodes."
print_errmsg "HAGS port (${CT_SYSCHK_GS_PORT}) is available on all nodes."
print_errmsg "HATS port (${CT_SYSCHK_TS_PORT}) is available on all nodes."
print_errmsg "Port availability checks.. Done."
fi
# step-3 : check node Id availability
check_node_id
node_chk_rc=$?
# Specific processing if basecmd contains -m (i.e. MTU size test)
# step-4 : check node Id availability
check_mtu_size
mtu_compat_rc=$?
# MTU size check end
if [[ ${mtu_compat_rc} -eq $MTU_COMPAT_SUCCESS ]];then
if [[ ${node_cnt} -ne 1 ]]; then
print_errmsg "Performing MTU compatibility check - Done. Interfaces on all nodes have same MTU size."
fi
else
print_errmsg "Performing MTU compatibility check - Done. MTU size is not compatible across all nodes."
fi
if [[ $conn_chk_rc == $CONN_PATH_SUCCESS ]] && [[ $port_chk_rc == $PORT_CHECK_SUCCESS ]] && [[ $node_chk_rc == $NODE_ID_SUCCESS ]] && [[ $mtu_compat_rc == $MTU_COMPAT_SUCCESS ]]
then
return $PD_REQ_SUCCESS
else
chk_rpd_req_rc=$(($conn_chk_rc | $port_chk_rc | $node_chk_rc | $mtu_compat_rc ))
if [[ $conn_chk_rc -ne $CONN_PATH_SUCCESS ]];then
rc=$conn_chk_rc
elif [[ $port_chk_rc -ne $PORT_CHECK_SUCCESS ]]; then
rc=$port_chk_rc
elif [[ $node_chk_rc -ne $NODE_ID_SUCCESS ]]; then
rc=$node_chk_rc
elif [[ $mtu_compat_rc -ne $MTU_COMPAT_SUCCESS ]]; then
rc=$mtu_compat_rc
else
rc=$chk_rpd_req_rc
fi
#process_exit_code $rc
return $rc
fi
}
function check_management_domain_req
{
print_errmsg "Performing port availability check for management domain cluster.."
PORTOPT='r'
check_port_available $PORTOPT
rc=$?
if [[ $rc -ne $PORT_CHECK_SUCCESS ]];then
# rc=${each_node}
print_errmsg "RMC port check failed on ${each_node}"
else
print_errmsg "RMC port (${CT_SYSCHK_RMC_PORT}) is available on all nodes."
print_errmsg "Port availability checks.. Done."
fi
return $rc
}
#####################################################################################################
# Name: process_opts #
# This function is called from main script and this function invokes the necessary checks related #
# to interfaces connectivity between the nodes , all checks related to peer domain or management #
# domain, port availability for hags/hats/rmc or firewall enabled etc.,. #
# This also validates the input options passed. #
# Return code: 0 if success, 1 otherwise #
#####################################################################################################
function process_opts
{
if [[ ${Argcount} == 0 ]]
then
echo "$0 : Invalid number of operands."
usage 1
fi
if [[ $UNKNOWN_SCOPEOPT == 1 ]] || [[ $UNKNOWN_PORTOPT == 1 ]] || [[ $UNKNOWN_SESSOPT == 1 ]] || [[ $UNKNOWN_COMMOPT == 1 ]]
then
print_errmsg "Invalid inputs specified"
exit 1
fi
if [[ $CLUSTER_SCOPE == 'p' ]]
then
check_peerdomain_req
rc=$?
if [[ $rc == 0 ]]; then
print_errmsg "Checking PeerDomain pre-req's - Done. Given nodes can be used for PeerDomain."
else
print_errmsg "Checking PeerDomain pre-req's - Done. PeerDomain prereq's failed. Check configuration details. "
fi
fi
if [[ $CLUSTER_SCOPE == 'm' ]]
then
check_management_domain_req
rc=$?
fi
if [[ $CLUSTER_SCOPE == 'a' ]]
then
check_peerdomain_req
rc1=$?
check_management_domain_req
rc2=$?
if [[ $rc1 == 0 && $rc2 == 0 ]]; then
return $PD_MD_REQ_SUCCESS
else
return $(($rc1 | $rc2))
fi
fi
#if [[ $CLUSTER_SCOPE != 'p' && $CLUSTER_SCOPE != 'm' && $CLUSTER_SCOPE != 'a' ]]
#then
if [[ ${optI} == 1 ]] && [[ ${VALID_SCOPE} == 0 ]] ; then
check_conn_path "$COMMOPT"
conn_rc=$?
rc=$conn_rc
if [[ $conn_rc -eq 0 ]]; then
if [[ $COMMOPT == '4' ]] || [[ $COMMOPT == '6' ]] ; then
print_errmsg "IPv${COMMOPT} interfaces are available on the nodes ${nodes[@]}."
else
print_errmsg "IPv4 / IPv6 interfaces are available on the nodes ${nodes[@]}."
fi
fi
fi
if [[ ${optP} == 1 ]] && [[ ${VALID_SCOPE} == 0 ]]; then
if [[ $UNKNOWN_PORTOPT -eq 1 ]];then
print_errmsg "Invalid Port Option"
process_exit_code $WRONG_OPT_FOR_PORT_CHECK_RC
else
check_port_available $PORTOPT
port_rc=$?
if [[ $port_rc != $PORT_CHECK_SUCCESS ]]; then
print_errmsg "Port availability check failed."
process_exit_code $port_rc
return $rc
fi
fi
fi
if [[ ${optN} == 1 ]] && [[ ${VALID_SCOPE} == 0 ]]; then
check_node_id
node_id_rc=$?
rc=${node_id_rc}
fi
if [[ ${optM} == 1 ]] && [[ ${VALID_SCOPE} == 0 ]]; then
# If no option is provided, just do it for all
typeset COMMOPT="a"
check_mtu_size
mtu_compat_rc=$?
if [[ ${mtu_compat_rc} -eq $MTU_COMPAT_SUCCESS ]];then
if [[ ${node_cnt} -ne 1 ]];then
print_errmsg "Performing MTU compatibility check - Done. Interfaces on all nodes have same MTU size."
fi
else
print_errmsg "Performing MTU compatibility check - Done. MTU size is not compatible across all nodes."
fi
if [[ $mtu_compat_rc -eq $REMOTE_NODE_CONN_REFUSED ]]; then
return $REMOTE_NODE_CONN_REFUSED
fi
fi
#fi
if [[ ${optS} == 1 ]]; then
check_session_scope
rc=$?
fi
do_cleanup
return $rc
}
function process_exit_code
{
exit_code=$1
case $exit_code in
$NO_VALID_IPV4_FOUND )
print_errmsg "${NO_VALID_IPV4_FOUND_STR}"
rc=${NO_VALID_IP_FOUND}
;;
$NO_VALID_IPV4_FOUND_RMT_NODE )
print_errmsg "${NO_VALID_IPV4_FOUND_RMT_NODE_STR}"
rc=${NO_VALID_IP_FOUND}
;;
$NO_VALID_IPV6_FOUND )
print_errmsg "${NO_VALID_IPV6_FOUND_STR}"
rc=${NO_VALID_IP_FOUND}
;;
$NO_VALID_IP_FOUND )
print_errmsg "${NO_VALID_IP_FOUND_STR}"
rc=${NO_VALID_IP_FOUND}
;;
$NO_MATCHING_SUBNET_FOUND )
print_errmsg "${NO_MATCHING_SUBNET_FOUND_STR}"
rc=${NO_VALID_IP_FOUND}
;;
$REMOTE_NODE_CONN_REFUSED )
print_errmsg "${REMOTE_NODE_CONN_REFUSED_STR}"
rc=${REMOTE_NODE_CONN_REFUSED}
;;
$NODE_ID_NOT_UNIQUE )
print_errmsg "${NODE_ID_NOT_UNIQUE_STR} ${error_node_id}"
rc=${NODE_ID_NOT_UNIQUE}
;;
$NO_NODE_ID_FILE )
print_errmsg "${NO_NODE_ID_FILE_STR} on ${error_node_id} "
;;
$RMC_NOT_LISTENING_RC )
print_errmsg "${RMC_NOT_LISTENING_STR} on ${error_node_id} "
rc=${PORT_ERROR}
;;
$TS_NOT_LISTENING_RC )
print_errmsg "${TS_NOT_LISTENING_STR}"
rc=${PORT_ERROR}
;;
$GS_NOT_LISTENING_RC )
print_errmsg "${GS_NOT_LISTENING_STR}"
rc=${PORT_ERROR}
;;
$RMC_PORT_BOUND_BY_OTHER_RC )
print_errmsg "${RMC_PORT_BOUND_BY_OTHER_STR} on ${error_node_id}"
rc=${PORT_ERROR}
;;
$TS_PORT_BOUND_BY_OTHER_RC )
print_errmsg "${TS_PORT_BOUND_BY_OTHER_STR} on ${error_node_id}"
rc=${PORT_ERROR}
;;
$GS_PORT_BOUND_BY_OTHER_RC )
print_errmsg "${GS_PORT_BOUND_BY_OTHER_STR} on ${error_node_id}"
rc=${PORT_ERROR}
;;
$RMC_PORT_HELD_BY_OTHER_RC )
print_errmsg "${RMC_PORT_HELD_BY_OTHER_STR} on ${error_node_id}"
rc=${PORT_ERROR}
;;
$TS_PORT_HELD_BY_OTHER_RC )
print_errmsg "${TS_PORT_HELD_BY_OTHER_STR} on ${error_node_id}"
rc=${PORT_ERROR}
;;
$GS_PORT_HELD_BY_OTHER_RC )
print_errmsg "${GS_PORT_HELD_BY_OTHER_STR} on ${error_node_id}"
rc=${PORT_ERROR}
;;
$WRONG_OPT_FOR_PORT_CHECK_RC )
print_errmsg "${WRONG_OPT_FOR_PORT_CHECK_STR}"
rc=${OTHER_ERROR}
;;
$FIREWALL_CHECK_FAILED_RC )
print_errmsg "${port_chk_failure_node}: ${FIREWALL_CHECK_FAILED_STR}"
rc=${PORT_ERROR}
;;
$INVALID_MTU_SZ_RC )
print_errmsg "${INVALID_MTU_SZ_STR}"
rc=${MTU_CHECK_ERROR}
;;
$MTU_NO_OP_4_IPV6 )
print_errmsg "${MTU_NO_OP_4_IPV6_STR}"
rc=${MTU_CHECK_ERROR}
;;
$MTU_COMPAT_CHECK_FAILED )
print_errmsg "${MTU_COMPAT_CHECK_FAILED_STR}"
rc=${MTU_CHECK_ERROR}
;;
$UNSUPPORTED_OS_RC )
print_errmsg "${UNSUPPORTED_OS_STR}"
rc=${OTHER_ERROR}
;;
esac
return $rc
}
################################################################################
# #
# MAIN FUNCTION #
# #
################################################################################
comment_str="########################################################################"
all_args=$@
# trap read debug
basecmd="$0"
TMPDIR=/tmp/CTSYSCHK_$$
OLDLOGDIR="/tmp/CTSYSCHK"
RMOLDDIRS=$(ls -d ${OLDLOGDIR}_* 2> /dev/null)
for i in ${RMOLDDIRS}
do
print_dbgmsg "Removing old DIRS in /tmp/CTSYSCHK*"
print_dbgmsg $i
pid=$(echo $i | sed 's:^'"${OLDLOGDIR}"\.'::g')
#echo $pid
if [[ -z $(ps -o pid= -p $pid 2> /dev/null) ]]; then
rm -rf $i
fi
done
mkdir -p ${TMPDIR}
OPTLIST=":i:p:s:C:U:P:mnuhV"
while getopts $OPTLIST opt
do
case $opt in
C )
CLUSTER_SCOPE=$OPTARG
case $CLUSTER_SCOPE in
p | m | a )
VALID_SCOPE=1
;;
*)
UNKNOWN_SCOPEOPT=1
usage
;;
esac
optC=1
basecmd=$basecmd$SPACE$MINUS$opt$SPACE$CLUSTER_SCOPE
;;
i )
COMMOPT=$OPTARG
case $COMMOPT in
4 | 6 | a )
;;
*)
UNKNOWN_COMMOPT=1
;;
esac
optI=1
basecmd=$basecmd$SPACE$MINUS$opt$SPACE$COMMOPT
;;
p )
PORTOPT=$OPTARG
case $PORTOPT in
r | s | a )
;;
*)
UNKNOWN_PORTOPT=1
;;
esac
optP=1
basecmd=$basecmd$SPACE$MINUS$opt$SPACE$PORTOPT
;;
U )
USERNAME=$OPTARG
optUSR=1
basecmd=$basecmd$SPACE$MINUS$opt$SPACE$USERNAME
;;
P )
PASSWD=$OPTARG
optPSWD=1
basecmd=$basecmd$SPACE$MINUS$opt$SPACE$PASSWD
;;
# t )
# echo "TB-disk test is a LIMITATION as of now."
# TB_DISK_INFO=$OPTARG
# basecmd=$basecmd$SPACE$MINUS$opt$SPACE$TB_DISK_INFO
# ;;
m )
optM=1
basecmd=$basecmd$SPACE$MINUS$opt
;;
n )
optN=1
basecmd=$basecmd$SPACE$MINUS$opt
;;
u )
optU=1
basecmd=$basecmd$SPACE$MINUS$opt
;;
# k )
# optK=1
# basecmd=$basecmd$SPACE$MINUS$opt
# ;;
s )
SESSOPT=$OPTARG
case $SESSOPT in
u | a )
;;
*)
UNKNOWN_SESSOPT=1
;;
esac
optS=1
basecmd=$basecmd$SPACE$MINUS$opt$SPACE$SESSOPT
;;
V )
VERBOSE=1
basecmd=$basecmd$SPACE$MINUS$opt
;;
h )
usage
;;
?|* )
echo "Unknown option: $opt "
usage
;;
esac
Argcount=$((Argcount + 1))
done
print_dbgmsg "${comment_str}"
typeset node_names=""
typeset each_node=""
#if [[ $Argcount == 0 ]]
#then
# echo "$0 : Invalid number of operands."
# usage 1
#fi
# Get the node names now
all_args=$@
local_node_present=0
ipargc="$#"
cmd_line="$0 $@"
set node_cnt=0
set -A nodes ""
set -A total_nodes ""
typeset error_node_id=""
if [[ ${ipargc} == 0 ]] ; then
cmd_line=${basecmd}
fi
if [[ "${cmd_line}" == "${basecmd}" ]]
then
if [[ $Argcount == 0 ]]
then
echo "$0 : Invalid number of operands."
usage 1
fi
print_dbgmsg "No nodenames specified, running on local node"
LOCAL_NODE=`hostname`
first_node=${LOCAL_NODE}
node_cnt=1
process_opts
rc=$?
if [[ $rc == 0 ]]
then
print_errmsg "ctsyschk is successful on $LOCAL_NODE."
else
print_errmsg "ctsyschk failed on $LOCAL_NODE."
process_exit_code $rc
rc=$?
fi
else
nodes=${cmd_line##$basecmd}
set -A total_nodes ""
i=0
node_cnt=0
for each_node in ${nodes[@]}
do
total_nodes[$i]="${each_node}"
i=$(($i+1))
done
node_cnt=${#total_nodes[@]}
first_node=${total_nodes[0]}
i=0
for each_node in ${total_nodes[@]}
do
if [[ $each_node == $first_node ]]; then
continue
fi
other_nodes[$i]="${each_node}"
i=$(($i+1))
done
if [[ $node_cnt == 1 ]]; then
if [[ $Argcount == 0 ]]; then
optI=1;
optP=1;
Argcount=$((Argcount + 1))
print_errmsg "Performing default checks as no input options specified."
process_opts
rc=$?
else
if [[ $optN == 1 ]]; then
print_errmsg "Option -n is a no-op for single node"
rc=${OTHER_ERROR}
# elif [[ $optM == 1 ]]; then
# print_errmsg "Option -m is a no-op for single node"
# rc=${OTHER_ERROR}
elif [[ $optS == 1 ]]; then
print_errmsg "Option -s is a no-op for single node"
rc=${OTHER_ERROR}
else
process_opts
rc=$?
fi
fi
else
process_opts
rc=$?
fi
if [[ $rc == 0 ]]; then
print_errmsg "ctsyschk is successful. "
else
process_exit_code $rc
rc=$?
print_errmsg "ctsyschk failed with $rc. "
fi
fi
rm -rf ${TMPDIR}
print_dbgmsg ${comment_str}
exit $rc
################
# END MAIN #
################