#!/bin/ksh93 # ALTRAN_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # Copyright (C) Altran ACT S.A.S. 2018,2019,2021. All rights reserved. # # ALTRAN_PROLOG_END_TAG # # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # 61haes_r721 src/43haes/usr/sbin/cluster/utilities/clmkcaa.sh 1.13 # # Licensed Materials - Property of IBM # # COPYRIGHT International Business Machines Corp. 2011 # All Rights Reserved # # US Government Users Restricted Rights - Use, duplication or # disclosure restricted by GSA ADP Schedule Contract with IBM Corp. # # IBM_PROLOG_END_TAG # @(#) 03d88c7 43haes/usr/sbin/cluster/utilities/clmkcaa.sh, 726, 2147A_aha726, Mar 22 2021 04:42 PM ################################################################### # # NAME: log # # FUNCTION: Writes all arguments to the LOGFILE # # EXECUTION ENVIRONMENT: Called from any other function in setup # # DATA STRUCTURES: # parameters: # global: # LOGFILE # # RETURNS: (int) # 0 = If the log entry was successful # 1 = If the log entry failed # # OUTPUT: The log entry (all arguments to the function) written to the # LOGFILE. # ################################################################### # function log { print -- "$(date) $*" >>$LOGFILE 2>&1 return $? } ####################################################################### # # NAME: ck_comm # # FUNCTION: # checks clcomd communication between nodes, inbound & outbound # # EXECUTION ENVIRONMENT: clverify # # DATA STRUCTURES: # global: list of nodes which can communicate with the local node # local_node name # # RETURNS: (int) # # OUTPUT: none # ###################################################################### function ck_comm { if [[ $DEBUG_MODE == "yes" || $VERBOSE_LOGGING == "high" ]] then set -x fi log "ck_comm: Called $(date)" typeset -i errors=0 nlist=$(clhandle -ac | cut -f2 -d":") for node in ${nlist} do log "ck_comm: node: ${node}\n" typeset -i try=0 noOutbound=1 noInbound=1 # : Check outbound communication # for (( try=0; try<$MAX_TRIES; try++ )) do if cl_rsh -n ${node} /usr/bin/hostname >>$LOGFILE 2>&1 then # : Outbound communications to node $node is working # noOutbound=0 break elif (( try < 4 )) then # : rest between each of the first few tries # sleep 1 fi done # try loop if (( noOutbound )) then log \ "ERROR: Unable to verify outbound clcomd communication to node: ${node}\n" dspmsg -s 30 scripts.cat 7 \ "ERROR: Unable to verify outbound clcomd communication to node: ${node}" ${node} dspmsg -s 30 scripts.cat 8 \ "Internode communication check using clcomd failed. Check the following: 1) /etc/cluster/rhosts has IP addresses for all nodes 2) clcomd subsystem is active (lssrc) 3) clcomd.log file.\n" (( errors++ )) fi # : Check inbound communication, See if that node can reach us # for (( try=0; try<$MAX_TRIES; try++ )) do if cl_rsh -n ${node} \ "/usr/es/sbin/cluster/utilities/cl_rsh -n ${local_node} /usr/bin/hostname" >>$LOGFILE 2>&1 then # : Inbound communication is working from node $node # noInbound=0 break elif (( try < 4 )) then sleep 1 fi done # try loop if (( noInbound )) then log \ "ERROR: Unable to verify inbound clcomd communication from node: ${node}\n" dspmsg -s 30 scripts.cat 9 \ "ERROR: Unable to verify inbound clcomd communication from node: ${node}" ${node} dspmsg -s 30 scripts.cat 8 \ "Internode communication check using clcomd failed. Check the following: 1) /etc/cluster/rhosts has IP addresses for all nodes 2) clcomd subsystem is active (lssrc) 3) clcomd.log file.\n" (( errors++ )) fi done # node loop return $errors } ####################################################################### # # NAME: mk_cluster # # FUNCTION: # Runs the CAA mkcluster command to create the CAA cluster. # # EXECUTION ENVIRONMENT: # # NOTES: # # RECOVERY OPERATION: # # DATA STRUCTURES: # parameters: # global: # # RETURNS: exit code from caa mkcluster # # OUTPUT: ###################################################################### # function mk_cluster { if [[ ${DEBUG_MODE} == yes || ${VERBOSE_LOGGING} == high ]] then set -x fi log "mk_cluster: Called $(date)\n" capabilities="site,ipv6" ipaddr="" node_list="" some_IPv6="" typeset -i num_nodes=0 PROGNAME="clmkcaa" # : Get cluster name # cname=$(/usr/es/sbin/cluster/utilities/clodmget -n -f name HACMPcluster) if [[ -z $cname ]] then # : We should not get here without a cluster name # dspmsg -s 27 scripts.cat 7 "ERROR: Missing cluster name or node name in subroutine create_CAA_config.\n" return 1 fi # : Get cluster id # cid=$(/usr/es/sbin/cluster/utilities/clodmget -n -f id HACMPcluster) # : get cluster repository disk # repos_pvid=$(/usr/es/sbin/cluster/utilities/clodmget -n -f repository HACMPsircol) if (( ${#repos_pvid} == 16 )) then repos_pvid=${repos_pvid}"0000000000000000" fi # : Change disk pvid to name for mkcluster # repos=$(/usr/es/sbin/cluster/utilities/clodmget -n -q "attribute = pvid and value = $repos_pvid" -f name CuAt) repos=${repos%%+([[:space:]])*} # : Must have a repository disk # if [[ -z $repos ]] then dspmsg -s 63 cluster.cat 40 "\nInvalid repository disk specified: pvid $repos_pvid could not be found in CuAt.\n" $repos_pvid dspmsg -s 27 scripts.cat 1 "ERROR: Cannot synchronize cluster changes without a cluster repository defined.\n\n" 1>&2 return 1 fi if [[ $(clodmget -f heartbeattype -n HACMPcluster) == U* ]] && \ [[ $capabilities == *unicast* ]] then # : This cluster uses unicast. # capabilities="${capabilities},unicast" # : Loop through all the nodes, building up a list of IP : addresses to be used. # odmget -q "object = COMMUNICATION_PATH" HACMPnode | \ egrep -w 'name =|value =|node_id =' | \ sed 's/.* = //;s/"//g' | \ paste - - - | \ while read node_name comm_path node_id do # : Find all the boot IP addresses on node $node_name which : are not in private networks # cle_list="" LC_ALL=C cllsif -c -S -i $node_name | cut -d':' -f2,5,7,15 | grep '^boot:public:' | \ while IFS=: read skip skip identifier net_type do if [[ $net_type == 'AF_INET' ]] then cle_list=${cle_list:+$cle_list","}"cle_ip=${identifier}" else # : Assume IPv6 if not AF_INET # cle_list=${cle_list:+$cle_list","}"cle_ip6=${identifier}" if [[ -z $some_IPv6 ]] then some_IPv6='ipv6' capabilities="${capabilities},ipv6" fi fi done # : At this point \$cle_list contains the boot IP addresses : for $node_name. Now add the node id. # if [[ $comm_path == @(+([0-9.])|+([0-9:])) ]] then # : If for some reason the communications path is a raw IP : address, turn it into a hostname for later CAA use # host $comm_path | read node_hostname rest if [[ -n $node_hostname ]] then odmget -q "object = COMMUNICATION_PATH and value = $comm_path" HACMPnode | \ sed 's/value = ".*"/value = "'$node_hostname'"/' | \ odmchange -q "object = COMMUNICATION_PATH and value = $comm_path" -o HACMPnode comm_path=$node_hostname fi fi if (( $node_id != 0 )) then node_list=${node_list:+$node_list","}"${comm_path}{cle_globid=${node_id},${cle_list}}" else node_list=${node_list:+$node_list","}"${comm_path}{${cle_list}}" fi (( num_nodes++ )) done else # : This cluster uses multicast, so start with the cluster IP address # ipaddr=$(/usr/es/sbin/cluster/utilities/clodmget -n -f ip_address HACMPsircol) if [[ -n $ipaddr ]] then ipaddr="-s $ipaddr" fi # : Get cluster nodes. Pass to mkcluster a comma separated list with node ids # ODMDIR=/etc/objrepos odmget -q "object = COMMUNICATION_PATH" HACMPnode | \ egrep -w "value =|node_id =" | \ sed 's/.* = //;s/"//g' | \ paste - - | \ while read comm_path node_id do if [[ $comm_path == [0-9.]* || $comm_path == *:* ]] then # : If for some reason the communications path is a raw IP : address, turn it into a hostname for later CAA use # host $comm_path | read node_hostname rest if [[ -n $node_hostname ]] then odmget -q "object = COMMUNICATION_PATH and value = $comm_path" HACMPnode | \ sed 's/value = ".*"/value = "'$node_hostname'"/' | \ odmchange -q "object = COMMUNICATION_PATH and value = $comm_path" -o HACMPnode comm_path=$node_hostname fi fi if (( $node_id != 0 )) then node_list=${node_list:+$node_list","}"${comm_path}{cle_globid=$node_id}" else node_list=${node_list:+$node_list","}"${comm_path}" fi (( num_nodes++ )) done fi # unicast/multicast # : Make sure we have all required arguments # if [[ -z $node_list ]] then # : We should not get here without a cluster name and at least one node defined # dspmsg -s 27 scripts.cat 7 "ERROR: Missing cluster name or node name in subroutine create_CAA_config.\n" 2>&1 return 1 fi log "mk_cluster: Multicast IP address is: ${mcast}\n" rm -rf /usr/es/sbin/cluster/etc/clmkcaa.$$ >/dev/null 2>&1 date > /usr/es/sbin/cluster/etc/clmkcaa.$$ # : Let the user know just how long CAA may take # typeset -i mkcltime=0 if (( ${num_nodes} <= 2 )) then mkcltime=2 elif (( $num_nodes <= 4 )) then mkcltime=3 elif (( $num_nodes <= 6 )) then mkcltime=4 elif (( $num_nodes <= 8 )) then mkcltime=6 elif (( $num_nodes <= 12 )) then mkcltime=20 elif (( $num_nodes <= 16 )) then mkcltime=30 else mkcltime=60 fi dspmsg -s 27 scripts.cat 8 "\n$PROGNAME: Configuring a $num_nodes node cluster in AIX may take up to $mkcltime minutes, Please wait.\n" $PROGNAME $num_nodes $mkcltime # : Create the SystemMirror cluster definition # print -- "$(date) ${PROGNAME}[$LINENO]: /usr/es/sbin/cluster/utilities/claddclstr -u" >> $LOGFILE /usr/es/sbin/cluster/utilities/claddclstr -u | sed "s/^/$(date) ${PROGNAME}[$LINENO]: /" >> $LOGFILE 2>&1 print -- "$(date) ${PROGNAME}[$LINENO]: return code from claddclstr: $rc " >>$LOGFILE # : Now the CAA cluster # print -- "$(date) ${PROGNAME}[$LINENO]: CLUSTER_OVERRIDE=yes ODMDIR=/etc/objrepos /usr/sbin/mkcluster \ -n $cname -r $repos $ipaddr -m ${node_list} ${capabilities}\n" | tee -a $LOGFILE mkcluster_out=$(CLUSTER_OVERRIDE="yes" ODMDIR=/etc/objrepos /usr/sbin/mkcluster -n $cname \ -r $repos $ipaddr -m "${node_list}" -c ${capabilities} 2>&1 ) rc=$? print -- "$(date) ${PROGNAME}[$LINENO]: ${mkcluster_out}" | tee -a $LOGFILE print -- "$(date) ${PROGNAME}[$LINENO]: return code from mkcluster: ${rc}" | tee -a $LOGFILE # : remove the lock # rm -rf /usr/es/sbin/cluster/etc/clmkcaa.$$ >/dev/null 2>&1 if (( $rc != 0 )) then grep '^caa.info' ${SYSLOG_CONF} | read skip caa_syslog skip dspmsg -s 27 scripts.cat 13 "ERROR: Problems encounted creating the cluster in AIX. See $CLUTIL_LOG and $caa_syslog for the output of the mkcluster command.\n" \ $CLUTIL_LOG $caa_syslog mkcluster return 1 fi # : Update the PowerHA timeout values with the CAA timeout values # typeset -i node_timeout=0 typeset -i node_down_delay=0 node_timeout=$(/usr/es/sbin/cluster/utilities/clodmget -n -f node_timeout HACMPcluster) node_down_delay=$(/usr/es/sbin/cluster/utilities/clodmget -n -f node_down_delay HACMPcluster) if (( $node_timeout == 0 )) then # : Update the default CAA value to our ODM. We store in seconds. # if nt=$(clctrl -tune -x node_timeout 2>&1) then node_timeout=${nt##*:} (( node_timeout = node_timeout / 1000 )) print "HACMPcluster: node_timeout=$node_timeout" | /usr/bin/odmchange -o HACMPcluster else print -- "$(date) ${PROGNAME}[$LINENO]: $nt" | tee -a $LOGFILE fi fi if (( $node_down_delay == 0 )) then # : Update the default CAA value to our ODM. We store in seconds. # if ndd=$(clctrl -tune -x node_down_delay 2>&1) then node_down_delay=${ndd##*:} (( node_down_delay = node_down_delay / 1000 )) print "HACMPcluster: node_down_delay=$node_down_delay" | /usr/bin/odmchange -o HACMPcluster else print -- "$(date) ${PROGNAME}[$LINENO]: $ndd" | tee -a $LOGFILE fi fi return $rc } # end of mk_cluster ################################################################### # Main ################################################################### PATH="$(/usr/es/sbin/cluster/utilities/cl_get_path all)" PROGNAME=${0##*/} if [[ $DEBUG_MODE == "yes" ]] || [[ $VERBOSE_LOGGING == "high" ]] then eval export $(/usr/es/sbin/cluster/utilities/cllsparam -x) set -x version='1.13' fi # : Defines and globals # typeset -i RC=0 export ODMDIR=/etc/es/objrepos typeset -i MAX_TRIES=10 MPING_PARAMS4="-v -c 5 -a 228.168.101.43" # for IPv4 MPING_PARAMS6="-v -c 5 -6" # IPv6 MPING_PARAMS=$MPING_PARAMS4 CDIR=${ODMDIR} LOGDIR=/var/hacmp/log LOGFILE=${LOGDIR}/$PROGNAME.log up_node_list="" # Get syslog configuration file typeset SYSLOG_CONF="" SYSLOG_CONF=$(clgetsyslog) # Use default configuration file for any kind of failures if (( $? != 0 )) then SYSLOG_CONF="/etc/syslog.conf" fi # : Create log directory # if [[ -f ${LOGDIR} ]] then dspmsg -s 33 scripts.cat 1 "${PROGNAME}: This tool expects to use the directory named ${LOGDIR},\n\ but there is already a file with than name\n" $PROGNAME $LOGDIR elif [[ -d ${LOGDIR} ]] then if [[ -f ${LOGFILE} ]] then dspmsg -s 33 scripts.cat 3 "${PROGNAME}: Saving existing ${LOGFILE} to ${LOGFILE}.bak\n" $PROGNAME $LOGFILE $LOGFILE mv -f ${LOGFILE} ${LOGFILE}.bak 2>/dev/null fi else if ! mkdir -p ${LOGDIR} 2>/dev/null then dspmsg -s 33 scripts.cat 2 "${PROGNAME}: This tool expects to use the directory named ${LOGDIR},\n\ but could not create it. Check path names and permissions.\n" $PROGNAME $LOGDIR exit 1 fi fi typeset local_node=$(get_local_nodename) if [[ -z $local_node ]] then dspmsg scripts.cat 9619 "$PROGNAME: Unable to discover the name of the local node.\n\ Please check the cluster configuration.\n" $PROGNAME exit 1 fi log "ckmkcaa: Local node name is: ${local_node}\n" # : Check clcomd function # dspmsg -s 33 scripts.cat 4 "${PROGNAME}: Verifying clcomd communication, please be patient.\n" if ! ck_comm then log "clmkcaa: Can't communicate with remote nodes using clcomd\n" dspmsg -s 33 scripts.cat 5 "${PROGNAME}: Internode communication check failed,\n\ check the clcomd.log file for more information.\n" $PROGNAME exit 1 fi dspmsg -s 33 scripts.cat 6 "${PROGNAME}: Creating CAA cluster, please wait.\n" $PROGNAME if ! mk_cluster then # : mk_cluster logs and echos error messages so we just exit here # exit 1 fi # : Capture final result # log "clmkcaa: lscluster output:\n" print "clmkcaa: lscluster output:\n" /usr/sbin/lscluster -m 2>&1 | tee -a ${LOGFILE} exit 0