#!/bin/ksh93
#  ALTRAN_PROLOG_BEGIN_TAG
#  This is an automatically generated prolog.
#
#  Copyright (C) Altran ACT S.A.S. 2017,2018,2019,2021.  All rights reserved.
#
#  ALTRAN_PROLOG_END_TAG
#
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r721 src/43haes/usr/sbin/cluster/events/rg_move_complete.sh 1.46 
#  
# Licensed Materials - Property of IBM 
#  
# COPYRIGHT International Business Machines Corp. 1999,2016 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG
# @(#)  7d4c34b 43haes/usr/sbin/cluster/events/rg_move_complete.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM

#########################################################################
#                                                                       #
#       Name:           rg_move_complete                                #
#                                                                       #
#       Description:    This script is called at the end of the         #
#                       rg_move recovery program.  Inspired by          #
#                       reconfig_resource_complete, it calls both       #
#                       node_up_remote_complete and                     #
#                       node_up_local_complete to remount NFS           #
#                       filesystems and run applications, respectively. #
#                                                                       #
#       Called by:      cluster manager                                 #
#                                                                       #
#       Calls to:       node_up_local_complete, node_up_remote_complete #
#                                                                       #
#       Arguments:      nodename, RGid, [destination_nodename]          #
#                                                                       #
#       Returns:        0       success                                 #
#                       1       failure                                 #
#                       2       bad argument                            #
#                                                                       #
#########################################################################

#########################################################################
# Does a union of SIBLING_ACQUIRING_GROUPS and SIBLING_RELEASING_GROUPS
# and removes any duplicate group names
#########################################################################
listSiblingGroups () {
    typeset PS4_FUNC="listSiblingGroups"
    [[ $VERBOSE_LOGGING == "high" ]] && set -x

    if [[ -n $(clodmget -f type HACMPrresmethods) ]]
    then 
        return ""
    fi

    for group in $SIBLING_ACQUIRING_GROUPS $SIBLING_RELEASING_GROUPS
    do
        echo $group
    done | sort -u
    
    return
}


#########################################################################
#
# Main Starts Here
#
#########################################################################

typeset PROGNAME=${0##*/}
export PATH="$(/usr/es/sbin/cluster/utilities/cl_get_path all)"
eval export $(cllsparam -x)

#
: Set the Run-Time Parameter values and export them to all successive scripts.
#
LOCAL_NODE=$(get_local_nodename)
set -a
    eval $(cllsparam -n $LOCAL_NODE)
set +a

: The following lines create informative trace output
[[ $VERBOSE_LOGGING == "high" ]] && {
    set -x
    version='%I%'
}

#
# This will be the exit status seen by the Cluster Manager.
#
integer STATUS=0

if [[ -z $EMULATE ]]
then
    EMULATE="REAL"
fi

set -u

if (( $# < 2 || $# > 3 ))
then
    cl_echo 6163 "Usage: $PROGNAME nodename RG_id [destination]\n" "$PROGNAME"
    exit 2
fi

export NODENAME=$1
RGID=$2
if (( $# == 3 ))
then
    RGDESTINATION=$3
else
    RGDESTINATION=""
fi

#
: serial number for this event is $EVENT_SERIAL_NUMBER
#

#
: Interpret resource group ID into a resource group name.
#
eval RGNAME=$(clodmget -q"id=$RGID" -f group -n HACMPgroup)
GROUPNAME="$RGNAME"

integer UPDATESTATD=0
integer NFSSTOPPED=0
integer LIMIT=60 WAIT=1 TRY=0

#
: If this is a two node cluster and exported filesystems exist, then
: when the cluster topology is stable notify rpc.statd of the changes.
#
if (( 2 == $(clnodename | wc -l) ))
then
    RESOURCE_GROUPS=$(clodmget -f group -n HACMPgroup)
    for group in $RESOURCE_GROUPS
    do
        EXPORTLIST=$(clodmget -q "group=$group AND name=EXPORT_FILESYSTEM" \
                                -f value -n HACMPresource)
        if [[ -n $EXPORTLIST && $group == $RGNAME ]]
        then
            UPDATESTATD=1
            if [[ $EMULATE == "EMUL" ]]
            then
                cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n"
                echo "cl_update_statd\n"
            else
                cl_update_statd
                if (( $? != 0 )); then
                    cl_log 1074 "$PROGNAME: Failure occurred while processing cl_update_statd.\n" $PROGNAME
                    STATUS=1
                fi
            fi
            break
        fi
    done
fi

#
: Set the RESOURCE_GROUPS environment variable with the names
: of all resource groups participating in this event, and export
: them to all successive scripts.
#
set -a
    clsetenvgrp_output=$(clsetenvgrp $NODENAME $PROGNAME $RGNAME $RGDESTINATION)
    RC=$?
    eval "$clsetenvgrp_output"
set +a
if (( $RC != 0 ))
then
    STATUS=1
fi
   

#
: For each participating resource group, serially process the resources.
#
for group in $RESOURCE_GROUPS
do
    #
    : All sub-scripts must inherit the same environment
    : values which were set by this script via set -a.
    : e.g.: all sub-scripts must inherit VERBOSE_LOGGING value.
    :
    : Although clsetenvres parses its second argument, looking for
    : the keywords "up," "down" and "complete", we will pass
    : the name of this script which only has a "complete" in it.
    : This is analogous to what reconfig_resource_complete does,
    : and it works.
    #
    set -a
        eval $(clsetenvres $group $PROGNAME)
    set +a
    export GROUPNAME=$group

    if [[ -n "$EXPORT_FILESYSTEM$EXPORT_FILESYSTEM_V4" ]]
    then
        NFSSTOPPED=1
    fi

    #
    : Call node_up_remote_complete AND, if necessary,
    : node_up_local_complete.  Note that we run this
    : latter script when this node is NOT the one from
    : which the resource group moved.
    :
    : This looks strange indeed, but only because the names
    : of the scripts reflect how they were originally called,
    : rather than what they actually do.  This sequence is
    : also used by reconfig_resource_complete.
    :
    : In fact, the script "node_up_remote_complete" remounts
    : NFS if needed, and "node_up_local_complete" runs any
    : applications configured in the resource groups it is
    : passed.  I have indicated more informative alternative
    : names for each command, below:
    :
    : Emulation is handled by these scripts, so I need not check here.
    :
    : NFS_remount $NODENAME, for a resource group
    : implicitly passed in the environment.
    #
    clcallev node_up_remote_complete $NODENAME
    if (( $? != 0 ))
    then
        cl_log 650 "$PROGNAME: Failure occurred while processing Resource Group $group. Manual intervention required.\n" "$PROGNAME" "$group"
        STATUS=1
    fi

    if [[ $PRINCIPAL_ACTION == "ACQUIRE" ]]
    then
        #
        : run_rg_applications, for a resource group
        : implicitly passed in the environment.
        #
        clcallev node_up_local_complete
    fi
    if (( $? != 0 ))
    then
        cl_log 650 "$PROGNAME: Failure occurred while processing Resource Group $group. Manual intervention required.\n" "$PROGNAME" "$group"
        STATUS=1
    fi
done

if (( $UPDATESTATD == 1 ))
then
    if [[ $EMULATE == "EMUL" ]]
    then
        cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n"
        echo "stopsrc -s rpc.lockd\n"
        echo "startsrc -s rpc.lockd\n"
    else
        stopsrc -s rpc.lockd
        rcstopsrc=$?
        if (( $rcstopsrc != 0 ))
        then
            : rc_stopsrc.rpc.lockd == $rcstopsrc
        fi

        for (( TRY=0; $TRY<$LIMIT; TRY++ ))
        do
            LC_ALL=C lssrc -s rpc.lockd | tail -1 | read name subsystem pid state
            if [[ ! -z $state ]];
            then
                sleep $WAIT
            else
                break
            fi
        done

        if [[ ! -z $state ]];
        then
            : Friendly stop has not worked. Trying a more forceful method.
            stopsrc -cs rpc.lockd

            : Await rpc.lockd shut down.
            : RG with NFS resources cannot be handled without rpc.lockd.

            cl_echo 9999 "Warning: rpc.lockd is running, waiting for it to stop.\n\n"

            for (( ; ; ))
            do
                LC_ALL=C lssrc -s rpc.lockd | tail -1 | read name subsystem pid state
                if [[ ! -z $state ]];
                then
                    sleep $WAIT
                else
                    break
                fi
            done
        fi

            #
            : Sure that rpc.lockd stopped. Restart it.
            #
            startsrc -s rpc.lockd
            rcstartsrc=$?
            if (( $rcstartsrc != 0 ))
            then
                : rc_startrc.rpc.lockd == $rcstartsrc
            fi
    fi

else
    : \$UPDATESTATD == $UPDATESTATD
    if (( $NFSSTOPPED == 1 ))
    then
        if [[ $EMULATE == "EMUL" ]]
        then
            cl_echo 3020 "NOTICE >>>> The following command was not executed <<<< \n"
            echo "startsrc -s rpc.lockd\n"
        else
            for (( TRY=0; $TRY<$LIMIT; TRY++ ))
            do
                LC_ALL=C lssrc -s rpc.lockd | tail -1 | read name subsystem pid state
                if [[ ! -z $state ]];
                then
                    sleep $WAIT
                else
                    break
                fi
            done

            if [[ ! -z $state ]]
            then
                : Friendly stop has not worked. Trying a more forceful method.
                stopsrc -cs rpc.lockd
                : Await rpc.lockd shut down.
                : RG with NFS resources cannot be handled without rpc.lockd.

                cl_echo 9999 "Warning: rpc.lockd is running, waiting for it to stop.\n\n"

                for (( ; ; ))
                do
                    LC_ALL=C lssrc -s rpc.lockd | tail -1 | read name subsystem pid state
                    if [[ ! -z $state ]];
                    then
                        sleep $WAIT
                    else
                        break
                    fi
                done
            fi

                #
                : Sure that rpc.lockd stopped. Restart it.
                #
                startsrc -s rpc.lockd
                rcstartsrc=$?
                if (( $rcstartsrc != 0 ))
                then
                    : rc_startrc.rpc.lockd == $rcstartsrc
                fi
        fi
    fi
fi

#
: If the resource group in this rg_move is now homeless,
: then we need to put it into an error state.
#
active_node=0
for GROUP in $HOMELESS_GROUPS
do
  # This code is used only for SAP RG's, In one scenario SAP ERS RG's are bounce
  # between ONLINE and ERROR state, to avoid this we are moving ERS RG
  # to OFFLINE if both SCS and ERS are trying to come online on the same node and
  # only one is node active in the cluster.

  #Check if this RG is ERS RG or not,if is it not ERS RG update RG state to error and continue
  application_id=$(clodmget -n -q "name=RESOURCE_GROUP and value=$GROUP"  -f application_id HACMPsa_metadata)
  
  if [[ -n $application_id && $application_id == @(*ERS*) ]]
  then
      # set active_node if any node is active other than local node
      for node in $(clodmget -n -f nodes -q "group =$GROUP" HACMPgroup)
      do
        if [[ $node == $LOCAL_NODE ]]
        then
            continue
        else
            STATE=$(LC_ALL=C cl_rsh -n $node "/usr/es/sbin/cluster/utilities/cldare -a")
            if [[ $STATE == "ACTIVE" ]]
            then
                active_node=1
                break
            fi
        fi
      done

      if (( $active_node != 1 ))
      then
          # yes current RG is ERS, now get the corresponding misc data
          CS=$(clodmget -n -q "group=$GROUP and name=MISC_DATA" -f value HACMPresource) 
          if [[ -n $CS ]]
          then
              CS_IPs=$(echo $CS | cut -f2 -d,)
              CS_IP1=$(echo $CS_IPs | cut -f1 -d_)
              CS_IP2=$(echo $CS_IPs | cut -f2 -d_)

              if [[ -n $CS_IP2 ]]
              then
                  CS_IP="${CS_IP1}|${CS_IP2}"
              else
                  CS_IP="${CS_IP1}"
              fi

              # Check SCS ServiceIP is online or not ? if it is online then ERS should go to OFFLINE
              if [[ -n $(ifconfig -a | egrep -w "${CS_IP}") ]]
              then
                  cl_RMupdate rg_down $GROUP $PROGNAME
                  continue
              fi
          fi
      fi      
  fi
  cl_RMupdate rg_error $GROUP $PROGNAME
  
done

#
: If the resource group in this rg_move is now homeless_secondary,
: then we need to put it into an errorsecondary state.
#
for GROUP in $HOMELESS_FOLLOWER_GROUPS
do
    cl_RMupdate rg_error_secondary $GROUP $PROGNAME
done 

#
: Set an error state for concurrent groups that have
: been brought offline on this node by rg_move.
#
for GROUP in $ERRSTATE_GROUPS
do
    if [[ $GROUP == $RGNAME ]]
    then
        cl_RMupdate rg_error $RGNAME $PROGNAME
    fi
done

# AM_SYNC_CALLED_BY determines if the cl_sync_vgs is called 
# as part of rg_move_complete event.This is used for logging
# VG sync in clavailability.log in cl_sync_vgs only if it is
# called as part of rg_move event. 

export AM_SYNC_CALLED_BY="RG_MOVE_COMPLETE"
process_resources
STATUS=$?
: The exit status of process_resources is: $STATUS
#
# unsetting AM_SYNC_CALLED_BY from caller's environment
# as we don't require it after this point in execution
#
unset AM_SYNC_CALLED_BY
if [[ $RG_DEPENDENCIES == "TRUE" ]]
then
    for siblingGroup in $SIBLING_ACQUIRING_GROUPS
    do
        set -a
            eval $(clsetenvres $siblingGroup $PROGNAME)
        set +a
        set -a
            eval $(clsetrepenv $siblingGroup)
        set +a
        export GROUPNAME=$siblingGroup

        METHODS=$(cl_rrmethods2call join_cleanup)
        for method in $METHODS
        do
            if [[ -x $method ]]
            then
                if ! $method $PROGNAME $NODENAME
                then
                    STATUS=1
                fi
            fi
        done
    done
fi

# Call the rg_move replicated resource method for all groups
# participating in this event

if [[ -z $RESOURCE_GROUPS ]]
then
      RESOURCE_GROUPS=$RGNAME
fi


for group in $RESOURCE_GROUPS
do
    export GROUPNAME=$group

    METHODS=$(cl_rrmethods2call postrg_move)

     for method in $METHODS
     do
         if [[ -x $method ]]
         then
             if ! $method $PROGNAME $NODENAME
             then
                 STATUS=1
             fi
         fi
     done
done

# Refresh clcomd.
refresh -s clcomd
 
exit $STATUS