#!/bin/ksh
#  ALTRAN_PROLOG_BEGIN_TAG
#  This is an automatically generated prolog.
#
#  Copyright (C) Altran ACT S.A.S. 2017,2021.  All rights reserved.
#
#  ALTRAN_PROLOG_END_TAG
#
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r714 src/43haes/usr/sbin/cluster/events/node_down_remote.sh 1.2.2.69 
#  
# Licensed Materials - Property of IBM 
#  
# COPYRIGHT International Business Machines Corp. 1990,2007 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)  7d4c34b 43haes/usr/sbin/cluster/events/node_down_remote.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM

#########################################################################
#
#   COMPONENT_NAME: EVENTS
#
#   FUNCTIONS: service
#
#########################################################################

#########################################################################
#                                                                       #
#       Name:           node_down_remote                                #
#                                                                       #
#	Description:	This script is called when a remote node	#
#			leaves the cluster.  The script unmounts	#
#			network filesystems  If not down gracefully,    #
#			the script will also take over resources like	#
#			fs, vg, disk, service address, AIX Connections  #
#			realm/service pairs, etc.		        #
#									#
#       Called by:      node_down                                       #
#                                                                       #
#       Calls to:       acquire_takeover_addr, acquire_service_addr,    #
#                       get_disk_vg_fs, get_aconn_rs, cl_start_snalink  #
#                                                                       #
#       Arguments:      nodename [graceful | forced]             	#
#                                                                       #
#       Returns:        0       success                                 #
#                       1       failure                                 #
#			2	bad argument				#
#                                                                       #
#########################################################################

get_fileysystems()
{
	typeset PS4_FUNC="get_fileysystems"
	[[ "$VERBOSE_LOGGING" = "high" ]] && set -x
	if [ $STATUS -eq 1 ]
	then
		set_resource_status "ERROR"
		exit 1
	fi

# Initialize OEM variables.
    set +u
    OEM_FILESYSTEM=${OEM_FILESYSTEM:-""}
    OEM_VOLUME_GROUP=${OEM_VOLUME_GROUP:-""}
    set -u
        clcallev get_disk_vg_fs "$FILESYSTEM" "$VOLUME_GROUP" "$DISK" "$OEM_FILESYSTEM" "$OEM_VOLUME_GROUP"
	rc=$?
	if [ $rc -eq 7 ] ; then
            STATUS=2
            REP_RES_FATAL=1
	    return
        fi

        if [ $rc -ne 0 -a $STATUS -eq 0 ] ; then
            STATUS=2
	    return
        fi

	if [ -n "$EXPORT_FILESYSTEM$EXPORT_FILESYSTEM_V4" ]
	then

    	    #
    	    # Export filesystem(s) to all other cluster nodes .
    	    # Before doing that, build a root label list, which
    	    # contains all nodes' service labels.
    	    #

            SERVICE=$(cllsif -cS | awk -F : '$2 == "boot" || $2 == "service" || $2 == "persistent"' | grep -v :serial: | cut -d: -f1 | sort -u)
            for service in $SERVICE
            do
            	   if [ -n "$ROOT" ]
            	   then
                	ROOT="$ROOT:$service"
            	   else
                	ROOT="$service"
            	   fi
    	    done
	    if [ "$EMULATE" = "EMUL" ]
	    then
		cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n"
		echo "stopsrc -s rpc.lockd\n"
	    else
		#stop NFS daemons
		if [ -n "$TAKEOVER_LABEL$SERVICE_LABEL" ]
		then
		    touch /tmp/.RPCLOCKDSTOPPED
		    stopsrc -s rpc.lockd
		fi
	    fi
	    
	    if [ "$EMULATE" = "EMUL" ]
	    then 
		cl_echo 3020 "NOTICE >>>> The following command was not executed <<<<\n" 
	    	echo "cl_export_fs '$ROOT' '$EXPORT_FILESYSTEM' '$EXPORT_FILESYSTEM_V4'\n"
	    else
	    	cl_export_fs "$ROOT" "$EXPORT_FILESYSTEM" "$EXPORT_FILESYSTEM_V4"
		if [ $? -ne 0 -a $STATUS -eq 0 ] ; then
		    STATUS=2
		fi
	    fi
	fi
}

###############################################################################
# 
# Set status of resource in resource location DB.  Note that we only set this
# if NFS_resource is "TRANS" or nonexistent,  because this means that we 
# are actually taking the resource. 
#
###############################################################################
set_resource_status()
{
    typeset PS4_FUNC="set_resource_status"
    [[ "$VERBOSE_LOGGING" = "high" ]] && set -x
    set +u
    eval TEMPNFS=\$"NFS_$GROUPNAME"
    set -u
    if [ "$TEMPNFS" = "" -o "$TEMPNFS" = "TRANS" ]
    then
	if [ "$EMULATE" = "EMUL" ]
	then
		cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n"
		echo	"clchdaemons -d clstrmgr_scripts -t resource_locator -n \""$LOCALNODENAME"\" -o \""$GROUPNAME"\" -v \"$1\"\n"
	else
		clchdaemons -d clstrmgr_scripts -t resource_locator -n "$LOCALNODENAME" -o "$GROUPNAME" -v "$1"
		if [ $? -ne 0 ]
		then
			cl_log 655 "$PROGNAME: Problem with resource location database in HACMPdaemons ODM.\n" $PROGNAME
			STATUS=1
		fi
	fi
        #
        # Resource Manager Updates
        #
        if [ "$1" = "ACQUIRING" ]
        then
            if [ "$PRINCIPAL_ACTION" = "ACQUIRE" ]
            then
                cl_RMupdate acquiring $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1 

            elif [ "$FOLLOWER_ACTION" = "ACQUIRE_SECONDARY" ]
            then
                cl_RMupdate acquiring_secondary $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            elif [ "$FOLLOWER_ACTION" = "RELEASE_SECONDARY" ]
            then
                cl_RMupdate releasing_secondary $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            fi
        else
            if [ "$FOLLOWER_ACTION" = "ACQUIRE_SECONDARY" ]
            then
                cl_RMupdate rg_error_secondary $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            elif [ "$FOLLOWER_ACTION" = "PRIMARY_BECOMES_SECONDARY" ]
            then
                cl_RMupdate rg_error_secondary $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
	    else
                cl_RMupdate rg_error $GROUPNAME $PROGNAME
                [ $? -ne 0 ] && STATUS=1
            fi
        fi
    fi
}

###############################################################################
#
# Start of main
#
###############################################################################

typeset PROGNAME=${0##*/}
export PATH="$(/usr/es/sbin/cluster/utilities/cl_get_path all)"

if [[ $VERBOSE_LOGGING == "high" ]]; then
    eval export $(cllsparam -x)
    set -x
    version='%I%'
fi

STATUS=0
ARGS=$1
TYPE=$2
ROOT=""
REP_RES_FATAL=0

if [ ! -n "$EMULATE" ]
then
        EMULATE="REAL"
fi

if [ $# -lt 1 ]
then
	cl_echo 1034 "Usage: $PROGNAME nodename [graceful | forced]\n" $PROGNAME
	exit 2
fi

set -u

CROSSMOUNT=0
export CROSSMOUNT

#
# Unmount network filesystems (NFS_filesystems) if the remote node
# was not forced down.  In the case of a forced down node, the cluster
# administrator is responsible for handling the resources belonging
# to that node since thay are no longer managed by HACMP.
#
case "$TYPE" in
     forced)	# Do nothing.
     		;;

     graceful)	# => graceful w/OUT takeover
		#
		if [ -n "$MOUNT_FILESYSTEM" ]
		   then
			# Regardless of whether MOUNT_FILESYSTEM contains
			# a semicolon -- i.e. 
			# regardless of whether MOUNT_FILESYSTEM is
			# NFS-mounted new-style or old-style,
			# dismount it on this node.
			#
			cl_deactivate_nfs "$MOUNT_FILESYSTEM"
			if [ $? -ne 0 ]
			   then
			       STATUS=1
			   fi
		   fi
     		;;

     *)		# => graceful WITH takeover, or crash
		#
		if [ -n "$MOUNT_FILESYSTEM" ]
		   then
			# If the following finds a semicolon, it returns zero.
			# If it does not find a semicolon, it returns one.
			#
			echo $MOUNT_FILESYSTEM | grep -q "\;/"
			RC=$?
			if [ $RC = 1 ]
			   then
				# No semicolon in MOUNT_FILESYSTEM.
				#
				cl_deactivate_nfs "$MOUNT_FILESYSTEM"
				if [ $? -ne 0 ]
				   then
				       STATUS=1
				   fi
			   else
				# Semicolon found in MOUNT_FILESYSTEM.
				#
				CROSSMOUNT=1
			fi
		   fi
     		;;
     esac

if [ $UPDATESTATD = 0 ]
then
    # if this is a two node cluster and exported filesystems exist, then when
    # the cluster topology is stable notify rpc.statd of the changes
    if [ 2 -eq $(odmget HACMPnode | grep 'name =' | sort | uniq | wc -l ) ]
    then
	RESOURCE_GROUPS=$(odmget HACMPgroup | grep 'group =' | awk '{print $3}' | sed 's/"//g')
	for group in $RESOURCE_GROUPS
	do
	    EXPORTLIST=$(odmget -q "group=$group AND name=EXPORT_FILESYSTEM" HACMPresource \
		| grep value | awk '{print $3}' | sed 's/"//g')
	    if [ -n "$EXPORTLIST" ]
	    then
		if [ "$EMULATE" = "EMUL" ]
		then
		    cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n"
		    echo "cl_update_statd\n"
		else
		    cl_update_statd
		    if [ $? -ne 0 ] ; then
			cl_log 1074 "$PROGNAME: Failure occurred while processing cl_update_statd.\n" $PROGNAME
			STATUS=1
		    fi
		fi
		break
	    fi
	done
    fi
fi

if [ $# -eq 1 ] ; then
    #
    # Remote node not graceful or forced down.
    # Acquire the service (shared) addr, take over its service address, 
    # fs, disk and vg, and re-export its filesystem.
    #
    # First, indicate that the resource is in the process of coming up by 
    # placing it into state "ACQUIRING".  This will persist until the resource 
    # comes completely up or there is an error.
    #
    set_resource_status "ACQUIRING"

    if [[ "$PRINCIPAL_ACTION" = "ACQUIRE" ]]
    then    
        #
        # Call replicated resource set-primary method associated
        # with any replicated resource defined in the resource group
        # we arecurrently processing. 
        #
        call_replicated_methods  "set_primary" ""
    fi


    # Start the WPAR.  Due to the fact that WPAR enablement/disablement is
    # done in a lazy fashion, the actual state of WPAR activity will not
    # necessarily match our ODM state.  Consequently, we can't simply look
    # at the WPAR_NAME environment variable.
    #
    # The command clstart_wpar will check if the resource group actually has
    # a WPAR so we don't need to check for that here.
    clstart_wpar

    if (( $? != 0 ))
    then
        echo "Failed to start the WPAR associated with resource group '${GROUPNAME}'"
        (( $STATUS == 0 )) && STATUS=2
    fi

    # Get filesystems if takeover of filesystem before ipaddr is set
    # to true. This way filesystems will be ready as soon as address
    # will be taken over. This removes the error of "Missing filesystem"
    #
    if [[ "$FS_BEFORE_IPADDR" = "true" ]] ; then
	get_fileysystems;
	[[ $REP_RES_FATAL = 1 ]] && exit $REP_RES_FATAL
    fi

    if [ -n "$SERVICE_LABEL" ] ; then
	clcallev acquire_service_addr "$SERVICE_LABEL"
	if [ $? -ne 0 -a $STATUS -eq 0 ] ; then
	    STATUS=2
        fi
    fi

    if [ -n "$TAKEOVER_LABEL" ]
    then
	clcallev acquire_takeover_addr "$TAKEOVER_LABEL"
	if [ $? -ne 0 -a $STATUS -eq 0 ] ; then
	# 
	# If we cannot acquire the proper service address,
	# do not attempt to acquire associated resources.
	#
	    STATUS=2
	fi
    fi


    # Get filesystems here if takeover of filesystem before ipaddr is
    # not set to true.
    #
    if [[ "$FS_BEFORE_IPADDR" != "true" ]]
    then
        get_fileysystems;
	[[ $REP_RES_FATAL = 1 ]] && exit $REP_RES_FATAL
    fi
     

    #
    # Start tape resources
    #
    if [ -n "$SHARED_TAPE_RESOURCES" ]
    then
	if [ "$EMULATE" = "EMUL" ]
	then 
		cl_echo 3020 "NOTICE >>>> The following command was not executed <<<<\n" 
		echo "cl_tape_resource_get_multi $SHARED_TAPE_RESOURCES\n"
	else
		cl_tape_resource_get_multi "$SHARED_TAPE_RESOURCES"
		if [ $? -ne 0 -a $STATUS -eq 0 ] ; then
		    STATUS=2
		fi
	fi
    fi

    #
    # Start AIX Connections services
    #
    if [ -n "$AIX_CONNECTIONS_SERVICES" ]
    then
	if [ "$EMULATE" = "EMUL" ]
	then 
		cl_echo 3020 "NOTICE >>>> The following command was not executed <<<<\n" 
		echo "clcallev get_aconn_rs $AIX_CONNECTIONS_SERVICES\n"
	else
		clcallev get_aconn_rs "$AIX_CONNECTIONS_SERVICES"
		if [ $? -ne 0 -a $STATUS -eq 0 ] ; then
		    STATUS=2
		fi
	fi
    fi

    #
    # Start AIX SMB/Fast Connect services
    #
    if [ -n "$AIX_FAST_CONNECT_SERVICES" ]
    then
	if [ "$EMULATE" = "EMUL" ]
	then 
		cl_echo 3020 "NOTICE >>>> The following command was not executed <<<<\n" 
		echo "get_fast_connect_rs \n"
	else
		get_fast_connect_rs
		if [ $? -ne 0 -a $STATUS -eq 0 ] ; then
		    STATUS=2
		fi
	fi
    fi

# start commlink processing
if [[ -n $COMMUNICATION_LINKS ]]
then
    	if [ "$EMULATE" = "EMUL" ]
    	then
			cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n"
			echo "cl_start_commlinks\n"
    	else
			cl_start_commlinks "$COMMUNICATION_LINKS"
			if [ $? -ne 0 -a $STATUS -eq 0 ] ; then
			    STATUS=2
			fi
	fi
fi


# else  remote node down "graceful" or "forced".
	#
        # remote node down graceful or forced, do nothing .
	#
fi

if [ $STATUS -ne 0 ]
then
    set_resource_status "ERROR"
fi

if [ $STATUS -eq 2 ]
then
	exit 0
else
	exit $STATUS
fi
