# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
#  
#  
# Licensed Materials - Property of IBM 
#  
# (C) COPYRIGHT International Business Machines Corp. 2018,2021 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
#
# sccsid = "@(#)92   1.4   src/rsct/utils/cmds/rc.shutdown.sh, common_utils, rsct_rady, radys004a 12/1/21 11:21:41"


unalias -a
export LC_ALL=C
unset FAST_SHUTDOWN

# Output preservation for this script.  Currently we keep
# only the two most recent copies.

if [[ -f /tmp/rsct.rc.shutdown.out ]]
then
    rm -f /tmp/rsct.rc.shutdown.bak
    mv /tmp/rsct.rc.shutdown.out /tmp/rsct.rc.shutdown.bak
fi

exec 1>>/tmp/rsct.rc.shutdown.out
exec 2>&1

print "$0 called with parameters $@"
print "on $(date)"

# Currently the only AIX shutdown option we care about is the -F flag,
# which means the customer is doing a "fast shutdown" and we need to
# avoid any unnecessary delays.

while getopts :F OPT
do
    if [[ $OPT == "F" ]]; then
        print "Fast shutdown scenario"
        FAST_SHUTDOWN="YES"
    fi
done


# Determine the cluster type in place, using a command which does not go
# through any of the subsystems (RMC/ConfigRM), to avoid potential delays
# in RSCT query commands from slowing down the progress of this script.

CURRENT_DOMAIN=$(/opt/rsct/bin/ct_clusterinfo -c)
REALM_TYPE=$(/opt/rsct/bin/ct_clusterinfo -r)


# If a CAA domain is online, we need to take special action to avoid
# resource protection actions from disrupting the shutdown.

if [[ -n $CURRENT_DOMAIN ]] &&
   [[ $CURRENT_DOMAIN != "IW" ]] &&
   [[ $REALM_TYPE == "CAA" ]]
then

    print "$(date +%T) Handling shutdown for domain $CURRENT_DOMAIN type $REALM_TYPE"

    # Disabling these protections will avoid:
    #    ** HAGS reacting to ConfigRM or RMC exit
    #    ** HAGS DMS expiring after HAGS is stopped

    print "$(date +%T) Disabling cthags protections"
    /opt/rsct/bin/hags_disable_client_kill -s cthags
    /opt/rsct/bin/dms/stopdms -s cthags

    sleep 5  # Need a "buffer" to ensure cthags has time to process
             # the two prior commands before we stop ConfigRM.

    # Under the condition of a fast shutdown, we just stop ConfigRM.
    # This is minimally needed to avoid:
    #    ** ConfigRM reacting to HAGS exit
    #    ** Disablement of TieBreaker (below)
    # Otherwise, we perform a more complete shutdown of all
    # Resource Managers plus RMC using the rmcctrl command.

    if [[ -n $FAST_SHUTDOWN ]]
    then

        # ConfigRM's exit may be delayed depending on what else is going on.
        # But killing ConfigRM manually will result in SRC trying to restart
        # it, and there's no guarantee it won't be able to get back online
        # before the shutdown can complete.
        #
        # Our solution is first use "stopsrc -cs" so that srcmstr sends ConfigRM
        # SIGTERM and begins watching for it to stop.  Then, if ConfigRM doesn't
        # stop on its own within a second, send it a manual SIGKILL.  SRC shouldn't
        # try to restart ConfigRM, since it's already expecting it to stop.

        print "$(date +%T) Stopping ConfigRM"
        stopsrc -cs IBM.ConfigRM

        sleep 1
        LSSRC_CONFIGRM=$(lssrc -s IBM.ConfigRM | tail -1)
        CONFIGRM_STATE=${LSSRC_CONFIGRM##* }
        if [[ $CONFIGRM_STATE != "inoperative" ]]
        then
            CONFIGRM_PID=$(echo $LSSRC_CONFIGRM | awk '{print $3}')
            kill -9 $CONFIGRM_PID
            # If we had to kill it, sleep just a little longer before moving
            # on to TieBreaker operations.
            sleep 2
        fi

    else

        print "$(date +%T) Shutting down RMC layer"
        /opt/rsct/bin/rmcctrl -z

    fi

    # If this node is holding the TieBreaker, it *must* be released before
    # other nodes try to acquire it; failing to acquire the TieBreaker will
    # bring down the entire remaining subdomain.
    #
    # This must be done after ConfigRM is shut down.

    export CLUSTER_ID=$(/opt/rsct/bin/ct_clusterinfo -i)
    export CT_SR_FS_HOME=/var/ct/${CLUSTER_ID}/registry/local_tree
    TIEBREAKER_TYPE=$(/opt/rsct/bin/lssrtbl-api -s /IBM/PeerNode/Class::::OpQuorumTieBreaker)

    if [[ $TIEBREAKER_TYPE != "Success" ]] &&
       [[ $TIEBREAKER_TYPE != "Fail" ]] &&
       [[ $TIEBREAKER_TYPE != "Operator" ]]
    then
        print "$(date +%T) Tiebreaker type $TIEBREAKER_TYPE found"
        /opt/rsct/bin/ct_run_op_tiebreaker -u
    fi

    # If not undergoing fast shutdown, fully shut down cthags,
    # now that RMC/RM layer and TB are out of the way.

    if [[ -z $FAST_SHUTDOWN ]]
    then
        # First, an extra measure of protection to ensure cthags has been
        # able to bring the DMS offline before we stop it.

        DMSCOUNT=$(/opt/rsct/bin/haDMS/haDMS_query | awk '{print $1}')

        if [[ $DMSCOUNT != "0" ]]; then
            print "$(date +%T) Warning: DMS query indicates $DMSCOUNT active timers.  Attempting manual stop."
            /opt/rsct/bin/haDMS/haDMS_stop 3
            sleep 5
        fi

        print "$(date +%T) Stopping cthags"
        /opt/rsct/bin/cthagsctrl -k
    fi

fi  ## ($DOM_ID_TYPE == "CAA")


print "$(date +%T) $0 exiting"
exit 0