#!/bin/ksh
#  ALTRAN_PROLOG_BEGIN_TAG
#  This is an automatically generated prolog.
#
#  Copyright (C) Altran ACT S.A.S. 2017,2018,2019,2021.  All rights reserved.
#
#  ALTRAN_PROLOG_END_TAG
#
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r714 src/43haes/usr/sbin/cluster/events/start_server.sh 1.4.1.31 
#  
# Licensed Materials - Property of IBM 
#  
# COPYRIGHT International Business Machines Corp. 1990,2012 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)  7d4c34b 43haes/usr/sbin/cluster/events/start_server.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM
#########################################################################
#
#   COMPONENT_NAME: EVENTS
#
#   FUNCTIONS: none
#
#########################################################################

#########################################################################
#									#
#	Name:		start_server					#
#									#
#	Description:	This script is called to start the 		#
#			application controller(s) when the local 		#
#			node is completely up or a remote node 		#
#			has finished leaving the cluster.		#
#									#
#	Called by:	node_up_local_complete,				#
#			node_down_remote_complete			#
#									#
#	Calls to:	None                   				#
#									#
#	Arguments: 	[controller-names...]				#
#									#
#	Returns:	0	success					#
#			1	failure					#
#									#
#########################################################################

start_and_monitor_server()
{
    typeset PS4_FUNC="start_and_monitor_server"
    [[ "$VERBOSE_LOGGING" = "high" ]] && set -x

    RETURN_STATUS=0
   
    server=$1
    
    # first check if the application is already running


    echo "Checking whether $server is already running...\n"  

        if [[ -n ${WPARNAME} ]]
        then
            cl_app_startup_monitor -s $server -a -w ${WPARNAME}
        else
            cl_app_startup_monitor -s $server -a
        fi
  
        RETURN_STATUS=$?
        : exit status of cl_app_startup_monitor is: $RETURN_STATUS

        if [[ $RETURN_STATUS  == 0 ]] ; # cl_app_startup_monitor monitor is RUNNING
        then
            echo "The application monitor detected a running instance of $server. PowerHA SystemMirror will not start another instance of $server on this node. \n" 
            echo "$1 $RETURN_STATUS" > $TMP_FILE.$server
            return
        fi

   # The application is not running or we could  not determine if its running	
   # or we are in emulation mode
   echo "Application monitor(s) indicate that $server is not active. Continuing with application startup.\n"
   
   
    #
    # First get the start server script.
    # Then prepare the pattern which we will write to an app. server
    # status file to keep track of the running server (we only need
    # to start the app. server once on one node.)
    #
    START=$(cllsserv -cn $server | cut -d':' -f2)
    START_SCRIPT=$(echo $START | cut -d' ' -f1)
    START_MODE=$(cllsserv -cn $server | cut -d':' -f4)
    [[ -z "$START_MODE" ]] && START_MODE="background"

    PATTERN="$LOCALNODENAME $server"
    RETURN_STATUS=0
    
    # Format for consumption by cl_am utility
    amlog_trace $AM_APP_START_BEGIN "Starting application controller in $START_MODE|$server"
    
    #
    # If the script is executable, start the server,
    # If not executable, echo error message.
    #
    if [[ ( -n ${WPARNAME} && -x ${WPARDIR}/${START_SCRIPT} ) || \
          ( -z ${WPARNAME} && -x $START_SCRIPT ) ]]
    then
        if [ "$START_MODE" == "background" ]
        then
            echo "Running application controller start script for $server in the background at $(LC_ALL=C date).\n"
            ODMDIR=$DCD ${EXEC} $START &
	    # there is no check of exit code for background startup
        else
            echo "Running application controller start script for $server in the foreground at $(LC_ALL=C date).\n"
            ODMDIR=$DCD ${EXEC} $START
            RETURN_STATUS=$?
            echo "Finished running application controller start script for $server at $(LC_ALL=C date), exit status was $RETURN_STATUS.\n"
        fi
    else
       cl_log 310 "$START_SCRIPT does not exist or is not executable." $START_SCRIPT
       RETURN_STATUS=1
    fi

    if [[ $RETURN_STATUS != 0 ]]
    then
        if [[ $PROC_RES = true ]]
        then
            RETURN_STATUS=11  #shows that the resource manager got updated
        fi
    
        # update the resource manager with the results
        cl_RMupdate resource_error $server $PROGNAME
        echo "$1 $RETURN_STATUS" > $TMP_FILE.$server
        return $RETURN_STATUS
    fi


    if [[ -n ${WPARNAME} ]]
    then
            cl_app_startup_monitor -s $server -w ${WPARNAME}
    else
            cl_app_startup_monitor -s $server
    fi

    RETURN_STATUS=$?
    : exit status of cl_app_startup_monitor is: $RETURN_STATUS
    if [[ $RETURN_STATUS != 0 ]]
    then
            if [[ $PROC_RES = true ]]; then
                RETURN_STATUS=11  #shows that the resource manager got updated
            fi
            cl_RMupdate resource_error $server $PROGNAME
            echo "ERROR: Application Startup did not succeed."
    fi

    # write STATUS into file
    echo "$1 $RETURN_STATUS" > $TMP_FILE.$server
    
    # Format for consumption by cl_am utility
    START_MODE=$(cllsserv -cn $server | cut -d':' -f4)
    if [[ $START_MODE == 'foreground' ]]
    then
        if [[ $RETURN_STATUS != 0 ]]
        then
            amlog_err $AM_APP_START_FAILURE "Starting application controller in foreground|$server"
        else    
            amlog_trace $AM_APP_START_END "Starting application controller in foreground|$server"
            
            # Start the process CPU and Memory logging for cl_availability
            MACTIVE=$(clodmget -q "name = $server" -n -f cpu_usage_monitor HACMPserver)
            
            if [[ $MACTIVE == "yes" ]]
            then
                MBINARY=$(clodmget -q "name = $server" -n -f process_to_monitor_cpu HACMPserver)
                MINTERVAL=$(clodmget -q "name = $server" -n -f cpu_usage_monitor_interval HACMPserver)
                nohup cl_AMmonitor -b $MBINARY -i $MINTERVAL -s $server&
            fi
        fi
    fi
    return $RETURN_STATUS
}



##############################################################
#
# MAIN starts here
#
##############################################################

PROGNAME=${0##*/}
export PATH="$(/usr/es/sbin/cluster/utilities/cl_get_path all)"

# Including Availability metrics library file
. /usr/es/lib/ksh93/availability/cl_amlib


if [[ $VERBOSE_LOGGING == high ]]
then
    set -x
    version='%I%'
fi


export TMP_FILE="/var/hacmp/log/.start_server.$$"
export DCD=/etc/es/objrepos
export ACD=/usr/es/sbin/cluster/etc/objrepos/active

rm -f $TMP_FILE

STATUS=0

# Usage: start_server [server-names...]

PROC_RES=false

# if JOB_TYPE is set, and it doesn't equal to "GROUP", then
# we are processing for process_resources
if [[ ${JOB_TYPE:-0} != 0 && $JOB_TYPE != "GROUP" ]]; then
   PROC_RES=true
fi

set -u
typeset WPARNAME EXEC WPARDIR
export  WPARNAME EXEC WPARDIR

EXEC=""
WPARNAME=""
WPARDIR=""

# update the resource manager with this operation
ALLSERVERS="All_servers"
ALLNOERRSERV="All_nonerror_servers"
cl_RMupdate resource_acquiring $ALLSERVERS $PROGNAME

WPARNAME=$(clwparname ${GROUPNAME})

if (( $? == 0 )) && [[ -n ${WPARNAME} ]]
then
    WPARDIR=$(clwparroot ${GROUPNAME})

    if [[ $? != 0 ]]
    then
        echo "$0 ERROR: Failed to get the WPAR directory for ${WPARNAME}"
        exit 1
    fi

    EXEC="clwparexec ${WPARNAME}"
fi

for server in $APPLICATIONS
do 
  start_and_monitor_server $server &
done

wait #wait for everybody to finish

# Format for consumption by cl_am utility
for server in $APPLICATIONS
do
    START_MODE=$(cllsserv -cn $server | cut -d':' -f4)
    if [ "$START_MODE" == "background" ]
    then
        SUCCESS=$(cat $TMP_FILE.$server | cut -f2 -d" ")
        if [[ $SUCCESS != 0 ]]; 
        then
            amlog_err $AM_APP_START_FAILURE "Starting application controller in background|$server"
        else
            amlog_trace $AM_APP_START_END "Starting application controller in background|$server"
            # Start the process CPU and Memory logging for cl_availability
            MACTIVE=$(clodmget -q "name = $server" -n -f cpu_usage_monitor HACMPserver)
             
            if [[ $MACTIVE == "yes" ]]
            then
                MBINARY=$(clodmget -q "name = $server" -n -f process_to_monitor_cpu HACMPserver)
                MINTERVAL=$(clodmget -q "name = $server" -n -f cpu_usage_monitor_interval HACMPserver)
                nohup cl_AMmonitor -b $MBINARY -i $MINTERVAL -s $server&
            fi
        fi
    fi
done

# see if everybody was successful - results are written to a file 
# in the form "<controller> <exit code>"
for server in $APPLICATIONS
do
      SUCCESS=$(cat $TMP_FILE.$server | cut -f2 -d" ")
      # check for status file corruption
      if [[ $SUCCESS !=  +([0-9]) ]]; then
          # status file malformed
          SUCCESS=11
      elif (( $SUCCESS != 0 )); then
        break
      fi
done

#Remove temporary files in any case
if [[ $SUCCESS == 0 ]]; then
    for server in $APPLICATIONS
    do
	    rm -f $TMP_FILE.$server
    done
    cl_RMupdate resource_up $ALLNOERRSERV $PROGNAME
else
    echo "One or more application controllers failed to start."
    echo "See $TMP_FILE.<controller name> for individual exit status."
fi

exit $SUCCESS