#!/bin/ksh93 # ALTRAN_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # Copyright (C) Altran ACT S.A.S. 2020,2021. All rights reserved. # # ALTRAN_PROLOG_END_TAG # # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # 61haes_r714 src/43haes/usr/sbin/cluster/utilities/clexit.rc.sh 1.38.1.3 # # Licensed Materials - Property of IBM # # COPYRIGHT International Business Machines Corp. 1996,2013 # All Rights Reserved # # US Government Users Restricted Rights - Use, duplication or # disclosure restricted by GSA ADP Schedule Contract with IBM Corp. # # IBM_PROLOG_END_TAG # @(#) 7d4c34b 43haes/usr/sbin/cluster/utilities/clexit.rc.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM ############################################################################### # # Function : send_mail_notification # Description : used to send e-mail about failed subsystems # ############################################################################### send_mail_notification() { typeset PS4_FUNC="send_mail_notification" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x mail -s "${SUB}" root << EOT ${SSYS} exited unexpectedly. ${MSG} EOT return } ############################################################################### # # Function : set_log_path # Description : used to set the proper log path from the ODM # ############################################################################### set_log_path() { typeset PS4_FUNC="set_log_path" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x # determine the path to hacmp.out DESTDIR=$(clodmget -q"name=hacmp.out" -n -f value HACMPlogs) HACMP_OUT_FILE="$DESTDIR/hacmp.out" return } ############################################################################### # # Filename: clexit.rc # # This file is called when clstrmgr or clinfo exits abnormally. # ############################################################################### PROGNAME=${0##*/} export PATH="$(/usr/es/sbin/cluster/utilities/cl_get_path all)" if [[ "$VERBOSE_LOGGING" == "high" ]] then set -x version='%I%' fi # Single formal parameter is the subsystem name SSYS=$1 # these 2 are defined in clsnmp.h SMUX_FORCED=64 # exit code when clstrmgr has been forced down SMUX_SHUTDOWN=512 # exit code when clstrmgr stopped by aix shutdown # this flag is defined in ha_gs.h HA_GS_GROUP_SERVICES_HAS_DIED_HORRIBLY=512 #summary code when cthags has failed. # STOP_DAEMON is defined in clstrmgr.h STOP_DAEMON=6 # exit code when clstrmgr stopped by stopdaemon ipc request # from clstrmgr.h RD_EXIT_STATUS_PATH="/usr/es/sbin/cluster/.clstrmgr.exit" NODE_HALT_CONTROL_FILE="/usr/es/sbin/cluster/etc/ha_nodehalt.lock" CTHAGS_FAILED_STATUS_PATH="/usr/es/sbin/cluster/.cthags.exit" typeset state=0 case $SSYS in clstrmgrES) # # Clstrmgr will create a file with its exit code (if the exit # was intentional) # CM_EXIT=1 if [[ -s $RD_EXIT_STATUS_PATH ]] then CM_EXIT=$(cat $RD_EXIT_STATUS_PATH) fi # If exit code is STOP_DAEMON, only clstrmgr daemon will be # stopped and RSCT will not be stopped. if [[ $CM_EXIT == $STOP_DAEMON ]] then echo "$PROGNAME : Cluster Manager Daemon is stopped." elif [[ $CM_EXIT == 0 ]] then # # If the exit code is 0 (normal) we will restart clstrmgr here # errlogger "$PROGNAME : Normal termination of ${SSYS}. Restart now." set_log_path echo "$PROGNAME : Normal termination of ${SSYS}. Restart now." >>$HACMP_OUT_FILE for ((i=1; i<=5; i++)) do lssrc -ls ${SSYS} >/dev/null 2>&1 if [[ $? == 1 ]]; then state=0 break fi state=$i sleep 1 done if [[ $state == 0 ]]; then startsrc -s ${SSYS} >>$HACMP_OUT_FILE 2>&1 if [[ -s $CTHAGS_FAILED_STATUS_PATH ]] then CTHAGS_EXIT=$(cat $CTHAGS_FAILED_STATUS_PATH) if [[ $CTHAGS_EXIT == $HA_GS_GROUP_SERVICES_HAS_DIED_HORRIBLY ]] then sleep 5 echo "$PROGNAME : Cluster Manager Daemon restarted due to failure of cthags daemon when resources are in unmanage state.\n" >>$HACMP_OUT_FILE #Restarting cluster services in the background cl_restart_clstrmgr & fi fi fi elif [[ $CM_EXIT == $SMUX_SHUTDOWN ]] then # # Clstrmgr was stopped by aix shutdown command # errlogger "$PROGNAME : ${SSYS} terminated during AIX shutdown." set_log_path echo "$PROGNAME : ${SSYS} terminated during AIX shutdown." >>$HACMP_OUT_FILE # Remove the file created if the clstrmgr was stopped # intentionally since the node is going down. # The path is defined in (and must match) clstrmgr.h rm -f /var/hacmp/log/clstrmgrES_stopdaemon elif [[ $CM_EXIT == $SMUX_FORCED ]] then # # Clstrmgr was forced down and terminated normally. # Restart with special env to tell it it was reincarnated # after force down # errlogger "$PROGNAME : Normal termination of ${SSYS} after cluster service stopped with unmanage." set_log_path echo "$PROGNAME : Normal termination of ${SSYS} after cluster service stopped with unmanage." >>$HACMP_OUT_FILE startsrc -s ${SSYS} -e "FORCED_DOWN=TRUE" >>$HACMP_OUT_FILE 2>&1 else # # non-zero (abnormal) exit # # There is a chance the errlog and cl_log messages may not be flushed to # disk before the dump is initiated. If not, the errlog entry will be # available in the dump. errlogger "$PROGNAME : Unexpected termination of ${SSYS}" & cl_log 230 "$PROGNAME : Unexpected termination of ${SSYS}" $PROGNAME ${SSYS} & # send mail to the admin as a background process so we # dont hang waiting on mail SUB="Unexpected termination of clstrmgrES subsystem" MSG="The hacmp6000 inittab entry may have been removed." send_mail_notification & # An optional, user customizable script can be run here in # place of the default action. By default, # "/etc/cluster/hacmp.term" is shipped without execute # permission - if the user customizes it and adds execute # permission, call their script in place of the default action if [ -x "/etc/cluster/hacmp.term" ] then SUB="Invoking commands in /etc/cluster/hacmp.term" MSG="The hacmp6000 inittab entry may have been removed." send_mail_notification & /etc/cluster/hacmp.term elif [[ ! -s /usr/es/sbin/cluster/.shutdown ]] then # Because it is critical that we remove HA's inittab entry # to prevent a bouncing node, we run this in foreground. cl_rmitab hacmp6000 > /dev/null 2>&1 errlogger "$PROGNAME : The PowerHA SystemMirror 'hacmp6000' inittab entry has been removed, if present." ############################################### # Do not halt the node if cluster is not active ############################################### if [ ! -f $NODE_HALT_CONTROL_FILE ] then # Remove the file created if the clstrmgr was stopped # intentionally since the node is going down. # The path is defined in (and must match) clstrmgr.h rm -f /var/hacmp/log/clstrmgrES_stopdaemon cl_log 231 "$PROGNAME : Halting system immediately!!!" $PROGNAME & # Do a sync, then a short sleep to attempt to flush the messages # we just logged to disk, and allow background processes to complete. # Because the secondary node will start taking over the resources # very quickly, we can't wait indefinitely. This node must be halted # to avoid conflict over the resources. sync & sleep 2 # halt the node halt -q fi fi fi ;; clinfoES) cl_log 230 "$PROGNAME : Unexpected termination of ${SSYS}" $PROGNAME ${SSYS} # # if there is a core file, try to grab it # if what core | grep -q hacmp.clinfo then cp core /var/hacmp/log/clinfo.core.$$ fi # If somehow it got restarted, kill it stopsrc -c -s clinfoES ;; esac