#!/bin/ksh93
#  ALTRAN_PROLOG_BEGIN_TAG
#  This is an automatically generated prolog.
#
#  Copyright (C) Altran ACT S.A.S. 2017,2021.  All rights reserved.
#
#  ALTRAN_PROLOG_END_TAG
#
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r721 src/43haes/usr/sbin/cluster/events/node_down_remote_complete.sh 1.2.11.3 
#  
# Licensed Materials - Property of IBM 
#  
# COPYRIGHT International Business Machines Corp. 1990,2016 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)  7d4c34b 43haes/usr/sbin/cluster/events/node_down_remote_complete.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM
#########################################################################
#
#   COMPONENT_NAME: EVENTS
#
#   FUNCTIONS: none
#
#########################################################################

#########################################################################
#                                                                       #
#       Name:           node_down_remote_complete                       #
#                                                                       #
#	Description:	This script is called after the 		#
#			node_down_remote script successfully completes.	#
#			The script starts application servers. 		#
#									#
#       Called by:      node_down_complete                              #
#                                                                       #
#       Calls to:       start_server                                    #
#									#
#       Arguments:      nodename [graceful | forced]                    #
#                                                                       #
#       Returns:        0       success                                 #
#                       1       failure                                 #
#                       2       bad argument                            #
#                                                                       #
#########################################################################

#########################################################################
# 
# Set status of resource in resource location DB.  Note that we only set this
# if NFS_resource is "TRANS" or nonexistent,  because this means that 
# we are actually taking the resource. 
#
#########################################################################
set_resource_status()
{
    typeset PS4_FUNC="set_resource_status"
    [[ "$VERBOSE_LOGGING" == "high" ]] && set -x
    set +u
    eval TEMPNFS=\$"NFS_$GROUPNAME"
    set -u
    if [ "$TEMPNFS" = "" -o "$TEMPNFS" = "TRANS" ]
    then
	if [ "$EMULATE" = "EMUL" ]
	then
		cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n"
		echo	"clchdaemons -d clstrmgr_scripts -t resource_locator -n \""$LOCALNODENAME"\" -o \""$GROUPNAME"\" -v \"$1\"\n"
	else
		clchdaemons -d clstrmgr_scripts -t resource_locator -n "$LOCALNODENAME" -o "$GROUPNAME" -v "$1"
		if [ $? -ne 0 ]
		then
			cl_log 655 "$PROGNAME: Problem with resource location database in HACMPdaemons ODM." $PROGNAME
			STATUS=1
		fi
	fi
        #
        # Resource Manager Updates
        #
        if [ "$1" = "UP" ]
        then
            if [ "$PRINCIPAL_ACTION" = "ACQUIRE" ]
            then
                cl_RMupdate rg_up $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            elif [ "$PRINCIPAL_ACTION" = "SUSTAIN" ]
            then
                cl_RMupdate rg_up $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            elif [ "$AUXILLIARY_ACTION" = "ACQUIRE_SECONDARY" ]
            then
                cl_RMupdate rg_up_secondary $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            elif  [ "$AUXILLIARY_ACTION" = "RELEASE_SECONDARY" ]
            then
                cl_RMupdate rg_down $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            fi
        else
            if [ "$PRINCIPAL_ACTION" = "ACQUIRE" ]
            then    
                cl_RMupdate rg_error $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            elif [ "$AUXILLIARY_ACTION" = "ACQUIRE_SECONDARY" ]
            then
                cl_RMupdate rg_error_secondary $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            else
                cl_RMupdate rg_error $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            fi
        fi
    fi
}

##############################################################################
#
# Main Starts Here
#
##############################################################################

typeset PROGNAME=${0##*/}
export PATH="$(/usr/es/sbin/cluster/utilities/cl_get_path all)"

if [[ $VERBOSE_LOGGING == "high" ]]; then
    eval export $(cllsparam -x)
    set -x
    version='%I%'
fi

STATUS=0

if [ ! -n "$EMULATE" ]
then
        EMULATE="REAL"
fi

if [ $# -lt 1 ]
then
        cl_echo 1034 "Usage: $PROGNAME nodename [graceful | forced]\n"  $PROGNAME
        exit 2
fi

set -u

if [ $# -eq 1 ]
then
	#
	# Remote node not down gracefully,
	# start application server and possible NFS mout to a new node.
	#
  export NODENAME=$1
	if [ -n "$MOUNT_FILESYSTEM" ] && [ "$NODENAME" != "$LOCALNODENAME" ]
        then
                NW_NFSMOUNT_LABEL=""

                if [ -n "$NFS_NETWORK" ]
                then
                    ALL_NFSMOUNT_LABEL=$(odmget -q "group=$GROUPNAME AND name=SERVICE_LABEL" HACMPresource | grep value | awk '{print $3}' | sed 's/"//g')
                    for label in $ALL_NFSMOUNT_LABEL; do
                        IN_NETWORK=$(cllsif -cS 2> /dev/null | grep :$NFS_NETWORK: | cut -d: -f1 | grep -x ${label} )
                        if [ -n "$IN_NETWORK" ]
                        then
                            NW_NFSMOUNT_LABEL="$NW_NFSMOUNT_LABEL $label"
                        fi
                    done

                fi

		#
		# Select active service label
		#
		# First, make sure to flush the NFSMOUNT_LABEL if it exists.
		# Only one of these nonzero.
		#
		if [ -n "$NW_NFSMOUNT_LABEL$NFSMOUNT_LABEL$TAKEOVER_LABEL$SERVICE_LABEL" ]
		then
		    if [ "$EMULATE" = "EMUL" ]
		    then
			cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n"
			echo arp -d $NFSMOUNT_LABEL
		    else
			# make sure to flush the arp cache for this label.
			arp -d $NW_NFSMOUNT_LABEL $NFSMOUNT_LABEL $TAKEOVER_LABEL $SERVICE_LABEL
		    fi
		fi

		NFSHOST=""

		for host in $NW_NFSMOUNT_LABEL $NFSMOUNT_LABEL $TAKEOVER_LABEL $SERVICE_LABEL
		do
			ping $host 1024 1 >/dev/null
			if [ $? = 0 ]
			then
				NFSHOST=$host
				break
			fi
		done

		if [ -n "$NFSHOST" ] ; then
			cl_activate_nfs 1 $NFSHOST "$MOUNT_FILESYSTEM"
			if [ $? -ne 0 ] ; then
				STATUS=1
			fi
		else
			cl_log 653 "$PROGNAME: NFS Mounting failed. \
			No reachable service interfaces found on node \
			$NFS_HOST" $PROGNAME $NFS_HOST
		fi
	fi

    if [ -n "$APPLICATIONS" ] ; then
        clmanageroha -o acquire -s -l "${APPLICATIONS// /,}" 3>&2
        if (( $? > 0 )) ; then
		STATUS=2
	    else
		if [ "$EMULATE" = "EMUL" ] ; then
			cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n" 
                	echo "clcallev start_server $APPLICATIONS\n"
		else
                	clcallev start_server "$APPLICATIONS"
			if [ $? -ne 0 -a $STATUS -eq 0 ]; then
			    STATUS=2
			fi
		fi
	    fi
        fi

	# 
	# Set final resource status
	#
	if [ $STATUS -ne 0 ] ; then
	    set_resource_status "ERROR"
	else
	    set_resource_status "UP"
	fi

# else
	#
	# Remote node graceful or forced down, do nothing.
	#
fi

if [ $STATUS -eq 2 ] ; then
    if [[ -z $CLUSTER_MAJOR || -z $CLUSTER_MINOR ]] ; then
	echo "Resource Group Allocation Failure disabled on $LOCALNODENAME for $GROUPNAME, version not found."
        exit 1  # terminal error, we don't know what version we are running
    fi
    if [ $CLUSTER_MAJOR -gt 44 -o $CLUSTER_MAJOR -eq 44 -a $CLUSTER_MINOR -eq 1 ] ; then
	echo "Resource Group Allocation Failure enabled on $LOCALNODENAME for $GROUPNAME."
        exit 0  # non-terminal exit status for RG acquisition failure
    else
	echo "Resource Group Allocation Failure disabled on $LOCALNODENAME for $GROUPNAME, PowerHA SystemMirror 4.4.0 or earlier version exists."
        exit 1  # terminal error, HACMP 4.4.0 or earlier release
    fi
else
    exit $STATUS
fi