#  ALTRAN_PROLOG_BEGIN_TAG                                                    
#  This is an automatically generated prolog.                                  
#                                                                              
#  Copyright (C) Altran ACT S.A.S. 2017,2019,2020,2021.  All rights reserved.  
#                                                                              
#  ALTRAN_PROLOG_END_TAG                                                      
#                                                                              
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r721 src/43haes/usr/sbin/cluster/sa/oracle/sbin/DBStatus.sh 1.18 
#  
# Licensed Materials - Property of IBM 
#  
# Restricted Materials of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2006,2016 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)  7d4c34b 43haes/usr/sbin/cluster/sa/oracle/sbin/DBStatus.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM

#----------------------------------------------------------------------------
# Global Definitions:
#----------------------------------------------------------------------------

PS_CMD="/usr/bin/ps"

# If we were running on the global WPAR we could use the following "enhanced"
# "ps."
#typeset PS_CMD="/usr/bin/ps -@ <wpar-name>"

#----------------------------------------------------------------------------
# Functions:
#   osaProcStatus
#   osaDBGetDefProcs
#   osaSQLStatus
#   osaListenerStatus
#   osaSpecListenerStatus

#----------------------------------------------------------------------------
# Function: 
#   osaProcStatus
#
# Purpose:
#   Notifies if a process is alive or not.
#
# Arguments: 
#       (1) names: process names in an array form  
#
# Returns:
#       0 if alive
#       >0 otherwise
#
function osaProcStatus {
    if [[ $VERBOSE_LOGGING == "high" ]]
    then
	PS4_FUNC=osaProcStatus
	set -x
    fi
    typeset appname=$1
    typeset -A realprocs
    osaLoadAllProcs realprocs;
    typeset -A pnames
    osaDBGetManProcs pnames $appname

    # Now compare mandatory process list with current process list
    typeset -i nfound=0   # number of mandatory processes not found
    typeset -l p
    for p in ${pnames[*]}; do
        if [[ -z ${realprocs[$p]} ]]; then
	    #
            : Mandatory DB Process \""$p"\" NOT found.
	    #
            user_msg 50 11 $p
            (( nfound = $nfound + 1 ))
        fi
    done
    return $nfound
}

#----------------------------------------------------------------------------
# Function: 
#   osaDBGetManProcs
#
# Purpose:
#   The following function returns a list of mandatory processes
#   for a given ORACLE_SID.
#
# Arguments:
#   (1) hash array of processes
#
# Returns:
#   n/a
#
function osaDBGetManProcs {
    if [[ $VERBOSE_LOGGING == "high" ]]
    then
	PS4_FUNC=osaDBGetManProcs
	set -x
    fi
    typeset  -n pnames=$1
    typeset appname=$2
    set -a
    DBVERSION=$(osaDBGetVersion)
    set +a
    export DBVERSION
    osaDBGetDefProcs pnames
}

#----------------------------------------------------------------------------
# Function:
#   osaDBGetDefProcs
#
# Purpose:
#   Returns a list of manditory processes based on the database
#   version
#
# Arguments:
#   (1) array to return list of processes
#
# Returns:
#   n/a
#
function osaDBGetDefProcs {
    if [[ $VERBOSE_LOGGING == "high" ]]
    then
        PS4_FUNC=osaDBGetDefProcs
        set -x
    fi
    typeset -n pnames=$1
    log_msg "DB Version: "$DBVERSION
    if [[ "${DBVERSION/\.*/}" == "18"* || "${DBVERSION/\.*/}" == "19"* ]]; then
        log_msg "Loading Oracle $DBVERSION process list DBW LGWR CKPT SMON PMON RECO PMAN LREG MMON MMNL"
        pnames=( [DBW]=DBW [LGWR]=LGWR [CKPT]=CKPT [SMON]=SMON [PMON]=PMON [RECO]=RECO [PMAN]=PMAN [LREG]=LREG [MMON]=MMON [MMNL]=MMNL) 
    elif [[ "${DBVERSION/\.*/}" == "12"* ]]; then
        log_msg "Loading Oracle $DBVERSION process list DBW LGWR CKPT SMON PMON MMAN RECO LREG MMON"
        pnames=( [DBW]=DBW [LGWR]=LGWR [CKPT]=CKPT [SMON]=SMON [PMON]=PMON [MMAN]=MMAN [RECO]=RECO [LREG]=LREG [MMON]=MMON)
    elif [[ "${DBVERSION/\.*/}" == "10" || "${DBVERSION/\.*/}" == "11" ]]; then
        log_msg "Loading Oracle $DBVERSION process list DBW LGWR CKPT SMON PMON MMAN RECO"
        pnames=( [DBW]=DBW [LGWR]=LGWR [CKPT]=CKPT [SMON]=SMON [PMON]=PMON [MMAN]=MMAN [RECO]=RECO)
    elif [[ "${DBVERSION/\.*/}" == "9" ]]; then
        log_msg "Loading Oracle $DBVERSION process list DBW LGWR CKPT SMON PMON RECO"
        pnames=( [DBW]=DBW [LGWR]=LGWR [CKPT]=CKPT [SMON]=SMON [PMON]=PMON [RECO]=RECO)
    else
        # unknown version
        # minimal list for any DB
        log_msg "WARNING: Loading Unsupported Oracle version Process list DBW LGWR CKPT SMON PMON. This is likely the result of not having the DB up and running when the smart assistant executed."
        pnames=( [DBW]=DBW [LGWR]=LGWR [CKPT]=CKPT [SMON]=SMON [PMON]=PMON)
    fi
}

#----------------------------------------------------------------------------
# Function:
#   osaLoadAllProcs
#
# Purpose:
#   Run ps and obtain a list of running processes
#
# Arguments:
#   (1) list of running processes
#
# Returns:
#   n/a
#
function osaLoadAllProcs {
    if [[ $VERBOSE_LOGGING == "high" ]]
    then
	PS4_FUNC=osaLoadAllProcs
	set -x
    fi
    typeset -n realprocs=$1

   # $PS_CMD -u $ORACLE_USER -o args | {
    $PS_CMD -e -o args | grep ora_ | awk '{print $1}' | {
        while read cmd; do
            typeset -l match match1 sid=$ORACLE_SID
            match=${cmd##"ora_"}            # match for ora_<proc-name>_sid
            match1=${match%%"_"$sid} # match against ORACLE_SID
            if [[ $match != $cmd  ]] && [[ $match1 != $match ]]; then
                match2=${match1//+([0-9])/}  # remove `multiplicity'
                realprocs[$match2]=1
            fi
        done
    }
}


#----------------------------------------------------------------------------
# Function: 
#   osaSQLStatus
#
# Purpose: 
#   Runs `SELECT instance_status from V$INSTANCE;'.
#
# Arguments:
#   Assumes $INSTANCE identifes instance
#
# Returns:
#   0 if instance is `open'
#   1 or 10 otherwise
#
function osaSQLStatus {
    [[ -z $(whence clodmget) ]] && PATH=$PATH:/usr/es/sbin/cluster/utilities
    if [[ $VERBOSE_LOGGING == 'high' ]]
    then
	PS4_FUNC=osaSQLStatus
	set -x
    fi
    typeset stat=$(LC_ALL=C osaSQLPlusRun 'select status from V$INSTANCE;' 2>&1)
    [[ $VERBOSE_LOGGING == 'high' ]] && print -- "$stat"
    if print -- "$stat" | grep -q -w OPEN
    then
        return 0;
    else
	#
	:   Log the failure status - it might be interesting
	#
	clutils_log_dir=$(clodmget -q "name = clutils.log" -f value -n HACMPlogs)
	clutils_log_dir=${clutils_log_dir:-/var/hacmp/log}
	print "$(date) oraSQLStatus[$LINENO]: Connect to Oracle Data Base $INSTANCE failed" >> $clutils_log_dir/clutils.log
	print "$(date) oraSQLStatus[$LINENO]: LC_ALL=C osaSQLPlusRun 'select status from V$INSTANCE'\n$stat" >> $clutils_log_dir/clutils.log
	log_msg "osaSQLStatus[$LINENO]: Connect to Oracle Data Base $INSTANCE failed"
	log_msg "oraSQLStatus[$LINENO]: LC_ALL=C osaSQLPlusRun 'select status from V$INSTANCE'\n$stat"
	#
        :   check to see if we got "ORACLE not available" message
	#
	if [[ $stat == *(?)'ORACLE not available'*(?) ]]
	then
            return 10
        else
            return 1;
        fi
    fi
}

#----------------------------------------------------------------------------
# Function:
#   osaListenerStatus
#
# Purpose:
#   Runs `lsnrctl status <listener-name>' against all listeners for
#   given SID.
#
# Arguments:
#   n/a 
#
# Returns:
#   0 if instance is `alive'
#   >=1 otherwise
#
function osaListenerStatus {
    if [[ -z $(whence clodmget) ]]
    then
	export PATH=$PATH:/usr/es/sbin/cluster/utilities
	eval export $(cllsparam -n $(clodmget -f nodename HACMPcluster))
    fi
    if [[ $VERBOSE_LOGGING == 'high' ]]
    then
	PS4_FUNC=osaListenerStatus
	PS4_TIMER=true
	set -x
    fi
    typeset -n lsnrstatus=$1
    typeset -i i=0
    typeset -A listeners
    typeset -i stat=0
    typeset lis
    integer LSMAXTIMEOUT		#   Seconds to wait for obtaining Listener status
    integer default_retries=15		#   Default number of retries
    integer margin=5			#   Seconds less than monitor_interval 
					#   available for retry

    clutils_log_dir=$(clodmget -q "name = clutils.log" -f value -n HACMPlogs)
    clutils_log_dir=${clutils_log_dir:-/var/hacmp/log}
    LSNSTAT=${clutils_log_dir}/lsnrctl.stat
    export LSNSTAT

    #
    :	Listener timeout - waiting for command to process - in seconds
    :	based on the monitor interval
    #
    if [[ -z $MONITOR_INTERVAL ]]
    then
	LSMAXTIMEOUT=15
    else
	LSMAXTIMEOUT=$MONITOR_INTERVAL
	if (( $LSMAXTIMEOUT < $default_retries + $margin ))
	then
	    LSMAXTIMEOUT=$default_retries
	else
	    (( LSMAXTIMEOUT-=$margin ))
	fi
    fi

    #
    :	For all the listeners defined for this SID, $ORACLE_SID,  
    :	get the unique listeners and find their status individually.
    #
    for (( i=0 ; $i < ${#allDBListeners[*]} ; i++ ))
    do
        set -A x ${allDBListeners[$i]}
        if [[ ${x[0]} == "$ORACLE_SID" ]]; then
            typeset lisn=${x[1]}
            listeners["$lisn"]=1
        fi
    done

    for lis in ${!listeners[*]}
    do
	#
        :   Background the listener status command, then wait for the proc to exit
	#
        (print "${ORACLE_HOME}/bin/lsnrctl status $lis" > $LSNSTAT.${lis}
	${ORACLE_HOME}/bin/lsnrctl status $lis >> $LSNSTAT.${lis} 2>&1  
	echo "RC=$?" >> $LSNSTAT.${lis}) &
        PID=$!
	if ! ps $PID
	then
	    #
	    :	Failed immediately!
	    #
	    logger -t "PowerHA SystemMirror for AIX" "Could not start ${ORACLE_HOME}/bin/lsnrctl status $lis "
	fi
	#
        :   Wait $LSMAXTIMEOUT seconds before killing the listener status
        :   command.  A timeout indicates the listener is not running.
	#
        typeset -i lstat=1  #listener status initialized to 1
        typeset -i count
	for (( count=0 ; $count<$LSMAXTIMEOUT ; count++ ))
	do
            sleep 1
            if ! ps $PID >/dev/null 
	    then
		#
		:   If the listener has stopped, break out of the loop
		#
                break
            fi
        done

	#
        :   Determine if the lsnrctl process is still running. 
	:   If so, assume it failed due to hang.
	#
        if ps $PID >/dev/null
	then
	    #
	    :	Kill hung lsnrctl process
	    #
	    logger -t "PowerHA SystemMirror for AIX" "Killing hung listener ${lis} process $PID"
            kill $PID
	fi

	if [[ -s $LSNSTAT.${lis} ]]
	then
	    #
	    :	Pick up any return code
	    #
	    last_line=$(tail -1 $LSNSTAT.${lis})
	    if [[ $last_line == 'RC='*(?) ]]
	    then
		print -- "$last_line" | IFS='=' read skip lstat
	    fi
	fi

	#
        :   See if the listener status was available.
	#
        if (( $lstat != 0 ))
	then
	    #
	    :	Display and save the listener status
	    #
	    cat $LSNSTAT.${lis}
	    if [[ ! -s $LSNSTAT.${lis}.first ]]
	    then
		[[ -s $LSNSTAT.${lis} ]] && cp -f $LSNSTAT.${lis} $LSNSTAT.${lis}.first
	    fi
	    [[ -s $LSNSTAT.${lis} ]] && cp -f $LSNSTAT.${lis} $LSNSTAT.${lis}.last
	    lsnrstatus["$lis"]=1
            osaError "Listener "$lis" is down.  See $LSNSTAT.${lis}.last"
        else
	    lsnrstatus["$lis"]=0
            log_msg "Listener " $lis" is alive."
        fi
	(( stat += $lstat ))		#   accumulate failure indications
    done
    return $stat
}

#----------------------------------------------------------------------------
# Function:
#   osaSpecListenerStatus
#
# Purpose:
#   Runs `lsnrctl status <listener-name>' against specified listener.
#
# Arguments:
#   (1) listener name
#   (2) An associative array "lsnrstatus" to store status of listener. 
#
# Returns:
#   0 if listener thread is `alive'
#   >=1 otherwise
#
function osaSpecListenerStatus {

    eval export $(cllsparam -n $(clodmget -f nodename HACMPcluster))

    if [[ $VERBOSE_LOGGING == 'high' ]]
    then
        PS4_FUNC=osaSpecListenerStatus
        PS4_TIMER=true
        set -x
    fi
 
    typeset lis=$1
    typeset -n lsnrstatus=$2
    integer LSMAXTIMEOUT                #   Seconds to wait for obtaining Listener status
    integer default_retries=15          #   Default number of retries
    integer margin=5                    #   Seconds less than monitor_interval
                                        #   available for retry

    clutils_log_dir=$(clodmget -q "name = clutils.log" -f value -n HACMPlogs)
    clutils_log_dir=${clutils_log_dir:-/var/hacmp/log}
    LSNSTAT=${clutils_log_dir}/lsnrctl.stat
    export LSNSTAT

    #
    :   Listener timeout - waiting for command to process - in seconds
    :   based on the monitor interval
    #

    if [[ -z $MONITOR_INTERVAL ]]
    then
        LSMAXTIMEOUT=15
    else
        LSMAXTIMEOUT=$MONITOR_INTERVAL
        if (( $LSMAXTIMEOUT < $default_retries + $margin ))
        then
            LSMAXTIMEOUT=$default_retries
        else
            (( LSMAXTIMEOUT-=$margin ))
        fi
    fi

    #
    :   Background the listener status command, then wait for the proc to exit
    #
    (print "${ORACLE_HOME}/bin/lsnrctl status $lis" > $LSNSTAT.${lis}
    ${ORACLE_HOME}/bin/lsnrctl status $lis >> $LSNSTAT.${lis} 2>&1
    echo "RC=$?" >> $LSNSTAT.${lis}) &
    PID=$!
    if ! ps $PID
    then
        #
        :   Failed immediately!
        #
        logger -t "PowerHA SystemMirror for AIX" "Could not start ${ORACLE_HOME}/bin/lsnrctl status $lis "
    fi
    #
    :   Wait $LSMAXTIMEOUT seconds before killing the listener status
    :   command.  A timeout indicates the listener is not running.
    #
    typeset -i lstat=1  #listener status initialised to 1  
    typeset -i count
    for (( count=0 ; $count<$LSMAXTIMEOUT ; count++ ))
    do
        sleep 1
        if ! ps $PID >/dev/null
        then
            #
            :   If the listener has stopped, break out of the loop
            #
            break
        fi
    done

    #
    :   Determine if the lsnrctl process is still running.
    :   If so, assume it failed due to hang.
    #
    if ps $PID >/dev/null
    then
        #
        :   Kill hung lsnrctl process
        #
        logger -t "PowerHA SystemMirror for AIX" "Killing hung listener ${lis} process $PID"
        kill -9 $PID
    fi

    if [[ -s $LSNSTAT.${lis} ]]
    then
        #
        :   Pick up any return code
        #
        last_line=$(tail -1 $LSNSTAT.${lis})
        if [[ $last_line == 'RC='*(?) ]]
        then
            print -- "$last_line" | IFS='=' read skip lstat
        fi
    fi

    #
    :   See if the listener status was available.
    #
    if (( $lstat != 0 ))
    then
        #
        :   Display and save the listener status
        #
        cat $LSNSTAT.${lis}
        if [[ ! -s $LSNSTAT.${lis}.first ]]
        then
            [[ -s $LSNSTAT.${lis} ]] && cp -f $LSNSTAT.${lis} $LSNSTAT.${lis}.first
        fi
        [[ -s $LSNSTAT.${lis} ]] && cp -f $LSNSTAT.${lis} $LSNSTAT.${lis}.last
        lsnrstatus["$lis"]=1
        osaError "Listener "$lis" is down.  See $LSNSTAT.${lis}.last"
    else
        lsnrstatus["$lis"]=0
        log_msg "Listener " $lis" is alive."
    fi
    return $lstat
}