#!/bin/ksh93
#  ALTRAN_PROLOG_BEGIN_TAG
#  This is an automatically generated prolog.
#
#  Copyright (C) Altran ACT S.A.S. 2017,2018,2019,2020,2021,2022.  All rights reserved.
#
#  ALTRAN_PROLOG_END_TAG
#
###############################################################################
# @(#)  5881272 43haes/usr/sbin/cluster/events/utils/clsessionroha.sh, 61aha_r726, 2205A_aha726, May 16 2022 12:15 PM
###############################################################################

#########################################################################
#
#   COMPONENT_NAME: hacmp.events
#
#   CLASS:
#
#       roha_session
#
#   FUNCTIONS:
#
#       roha_session_log
#       roha_session_open
#       roha_session_close
#       roha_session_set_running_apps
#       roha_session_set_other
#       roha_session_set_synchronous
#       roha_session_update_cluster_manager
#       roha_session_report_error
#       roha_session_report_success
#       roha_session_read_odm_dynresop
#       roha_session_write_odm_dynresop
#       roha_session_print_odm_dynresop
#       roha_session_read_odm_rohaparam
#       roha_session_write_odm_rohaparam
#       roha_session_print_odm_rohaparam
#       roha_session_wait_on_async_release
#
#########################################################################

#================================================
# The following, commented line enforces coding
# standards when this file is edited via vim.
#================================================
# vim:tabstop=4:shiftwidth=4:expandtab:smarttab
#================================================

#=======================================================
# R O H A _ S E S S I O N
#=======================================================
typeset roha_session=(
    typeset -i id=0                         # unique id
    typeset operation=""                    # type of operation to perform
    typeset -i compute_only=0               # if set, no acquisition/release will be performed
    typeset -i systemmirror_mode=0          # if set, PowerHA SystemMirror mode, otherwise isolation mode (default).
    typeset -i synchronous=0                # if set, release will be synchronous
    typeset optimal_apps=""                 # ACs to be brought online/offline
    typeset running_apps=""                 # ACs currently running
    typeset preferred_hmc_list=""           # list of HMC(s) set on this partition,
                                            #   this current hmc list is persisted into ODM, and
                                            #   will be preferably used by clhmccmd.
    typeset preferred_nova_list=""          # list of nova(s) set on this partition,
                                            #   this current nova list is persisted into ODM, and
                                            #   will be preferably used by clnovacmd.
    typeset preferred_partition=""          # partition on which the code is running,
                                            #   this current partition is persisted into ODM, and
                                            #   will be preferably used by clhmccmd/clnovacmd.
    typeset preferred_managed_system=""     # managed system on which the current partition runs,
                                            #   this managed system is persisted into ODM, and
                                            #   will be preferably used by clhmccmd/clnovacmd.
    typeset preferred_enterprise_pool=""    # enterprise pool on which the current partition runs,
                                            #   this enterprise pool is persisted into ODM, and
                                            #   will be preferably used by clhmccmd.
    typeset other_lpar=""                   # lpar hosting target node (release) or source node (acquisition)
    typeset other_cec=""                    # cec  hosting target node (release) or source node (acquisition)
)
typeset -i online_rgs_skip=0
#=============================================================================
#
# Name:        roha_session_log
#
# Description: Display a string to STDOUT. This string is used as header for
#  ROHA log items in hacmp.out file
#
# Inputs:      string
#
# Outputs:     None
#
# Globals:     None
#
# Returns:     None
#
#=============================================================================
function roha_session_log
{
    print "$1" | \
    while read line ; do
        print "[$ROHA_TAG:${roha_session.id}:($SECONDS)] $line"
    done
} # End of "roha_session_log()"

#=============================================================================
#
# Name:        roha_session_open
#
# Description: Open a session. Actions done here are: parsing options,
#              performing checks, initializing variables and updating ODM.
#
# Inputs:      $* - list of arguments
#
# Outputs:     None
#
# Globals:     LOCALNODENAME    environment variable
#              GROUPNAME        environment variable
#
# Returns:     RC_SUCCESS
#              RC_FAILURE
#
# Exits:       RC_SUCCESS if missing parameter and prints usage
#              RC_SUCCESS if no ROHA configured
#
#=============================================================================
function roha_session_open
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    #=======================================================
    # Set session id
    #=======================================================
    roha_session.id=$$
    roha_session_log "Open session ${roha_session.id} at $(LC_ALL=C date)"

    #=======================================================
    # Get options
    #=======================================================
    while getopts ":cso:l:t" opt ; do
        case $opt in
        c) # compute_only mode
            roha_session.compute_only=1
            ;;
        s) # PowerHA SystemMiror mode
            roha_session.systemmirror_mode=1
            ;;
        o) # operation
            roha_session.operation=$OPTARG
            ;;
        l) # optimal_apps
            roha_session.optimal_apps="${OPTARG//[:,]/ }"
            ;;
        t) # skip the online rgs and this option not visable for user
            online_rgs_skip=1
            ;;
        \?) # else
            usage
            exit $RC_SUCCESS
        esac
    done

    #=======================================================
    # Check operation is present and correct
    #=======================================================
    if [[ ${roha_session.operation} != @(acquire|release|adjust) ]] ; then
        usage
        return $RC_PARAM_ERROR
    fi

    typeset -i no_roha_apps=0
    typeset -i need_explicit_res_rel=0

    #=======================================================
    # To check if any AC (argument) with ROHA, we check if
    #   the concatenation of the complete list of ROHA
    #   and the argument list have some duplicate. If not,
    #   so none of the AC in argument list have ROHA.
    #   - To concatenate the two lists, we first have the one
    #     returned by clmgr q roha which is line-separated
    #     and the argument one. For the argument one, we first
    #     convert it to a line separated list. Then we
    #     remove any prospective duplicated arguments (sort -u).
    #   - On the concatenated list, we only keep the duplicated
    #     items (sort then uniq -d)
    # In case of Automatic Release After Failure (ARAF)
    #   (no apps passed as argument), we still need to check
    #   if there is any ROHA.
    #=======================================================
    if [[ -n ${roha_session.optimal_apps} && -z $(echo "$(clmgr q roha)\n$(echo ${roha_session.optimal_apps// /'\n'} | sort -u)" | sort | uniq -d) ]] \
    || [[ -z ${roha_session.optimal_apps} && -z $(clmgr q roha) ]] ; then
        roha_session_log "INFO: No ROHA configured on applications.\n"
        no_roha_apps=1
    fi
    #=======================================================
    # Read tunables from /etc/environment
    #=======================================================
    read_tunables
    echo "$TUNABLE_ROHA_SKIP_NODE" | grep -q $LOCALNODENAME
    if (( $? == 0 )); then
        roha_session_log "INFO: TUNABLE_ROHA_SKIP_NODE is set to local node,hence skipping ROHA operations\n"
        exit 0
    fi
 
    if (( $no_roha_apps == 1 )) ; then
        if (( $(roha_session_read_odm_dynresop "DLPAR_MEM") == 0.00 )) \
        && (( $(roha_session_read_odm_dynresop "DLPAR_PROCS") == 0 )) \
        && (( $(roha_session_read_odm_dynresop "DLPAR_PROC_UNITS") == 0.00 )) ; then
            roha_session_log "INFO: Nothing to be done.\n"
            exit $RC_SUCCESS
        else
            roha_session_log "INFO: Need to release resources those are acquired as part of ROHA operations.\n"
            need_explicit_res_rel=1
        fi
    fi
    
    typeset hmc_list=
    typeset nova_list=
    
    #================================================================
    # Get NovaLink list and HMC list for the local node
    #================================================================
    nova_list=$(cl_get_nova_list 2>/dev/null)
    hmc_list=$(cl_get_hmc_list 2>/dev/null)

    if (( $need_explicit_res_rel == 1 )) && [[ -z $nova_list ]] && [[ -z $hmc_list ]] ; then
        roha_session_log "INFO: Could not proceed for release of ROHA resources as there is no NovaLink or HMC configured.\n"
        exit $RC_SUCCESS
    fi

    #=======================================================
    # Check if a previous instance of the script is running.
    # If running, wait till config_too_long timeout
    #=======================================================
    ps -eo 'pid,args' | egrep -vw "$$|grep" | grep -qw "$PROGNAME"
    if (( $? == 0 )) ; then
        typeset -i max_time=$(clmgr -cS -a MAX_EVENT_TIME qu cluster)
        (( max_time *= 2 ))
        typeset -i timeOut=max_time
        roha_session_log "$(dspmsg -s $ROHA_SET $ROHA_MSGS 523 \
        "%1\$s: ERROR: Another %1\$s script is currently running. Will wait max till %2\$d sec.\n" \
        "$PROGNAME" "$max_time")"
        while (( $timeOut > 0 )); do
            ps -eo 'pid,args' | egrep -vw "$$|grep" | grep -qw "$PROGNAME"
            if (( $? == 0 )) ; then
               (( timeOut -= 2 ))
                if (( $timeOut > 0 )); then
                    sleep 2
                else
                    roha_session_log "$(dspmsg -s $ROHA_SET $ROHA_MSGS 524 \
                    "%2\$s: ERROR: Waited for %1\$d sec, another %2\$s script is still running. Try again later on.\n" \
                    "$max_time" "$PROGNAME")"
                    return $RC_PARAM_ERROR
                fi
            else
                break
            fi
        done
    fi

    #=======================================================
    # Determine current running applications
    #=======================================================
    roha_session_set_running_apps

  if (( CONN_TYPE != 2 )); then
    attrs="local_lpar_name:local_sys_name"

    #================================================================
    # First try on NovaLink If it is configured, otherwise try on HMC
    #================================================================
    if [[ -n $nova_list ]]; then
        roha_session.preferred_nova_list=$nova_list
        out_str=$(clnovacmd -o query -a $attrs)
        if (( $? > 0 )) ; then
            roha_session_log "INFO: Unable to communicate with Novalink.\n"
            return $RC_PARAM_ERROR
        fi
    else
        out_str=$($CLHMCCMD_CMD -o query -a $attrs)
        if (( $? > 0 )) ; then
            roha_session_log "INFO: Unable to communicate with HMC.\n"
            return $RC_PARAM_ERROR
        fi
    fi
    print "$out_str" | IFS=: read roha_session.preferred_partition \
        roha_session.preferred_managed_system    

    if [[ -n $hmc_list ]]; then
        roha_session.preferred_hmc_list=$hmc_list
        attrs="local_codpool_name"
        out_str=$($CLHMCCMD_CMD -o query -a $attrs)
        if (( $? > 0 )) ; then
            roha_session_log "INFO: Unable to get Local EPCOD pool name from HMC.\n"
        else
            roha_session.preferred_enterprise_pool=$out_str
        fi        
    fi
  fi

    #=======================================================
    # Set source or target lpar in case of move or takeover.
    #=======================================================
    roha_session_set_other

    #=======================================================
    # Check if synchronous release is needed (source and
    # target node are on the same managed system)
    #=======================================================
    roha_session_set_synchronous

    #=======================================================
    # Save data in HACMPdynresop ODM
    #=======================================================
    roha_session_write_odm_dynresop TIMESTAMP          "$(LC_ALL=C date)"
    roha_session_write_odm_dynresop MANAGED_SYSTEM     "${roha_session.preferred_managed_system}"
    roha_session_write_odm_dynresop ENTERPRISE_POOL    "${roha_session.preferred_enterprise_pool}"
    roha_session_write_odm_dynresop PREFERRED_HMC_LIST "${roha_session.preferred_hmc_list}"
    roha_session_write_odm_dynresop PREFERRED_NOVA_LIST "${roha_session.preferred_nova_list}"

    #=======================================================
    # Print ODM
    #=======================================================
    roha_session_print_odm_rohaparam
    roha_session_print_odm_dynresop

    #=======================================================
    # Print Session specific information
    #=======================================================
    roha_session_log "$(printf "+------------------+---------------------------------+\n")"
    roha_session_log "$(printf "| Session info     |   Value                         |\n")"
    roha_session_log "$(printf "+------------------+---------------------------------+\n")"
    roha_session_log "$(printf "| Operation        |   %                    27.27s   |\n" ${roha_session.operation})"
    roha_session_log "$(printf "| Compute only     |                           %3d   |\n" ${roha_session.compute_only})"
    roha_session_log "$(printf "| SystemMirror mode|                           %3d   |\n" ${roha_session.systemmirror_mode})"
    if (( ${roha_session.compute_only} == 0 )) ; then
        roha_session_log "$(printf "| Synchronous      |   %                    27.27s   |\n" ${roha_session.synchronous})"
    fi
    roha_session_log "$(printf "| Handled Apps     | %s\n" "${roha_session.optimal_apps}")"
    roha_session_log "$(printf "| Running Apps     | %s\n" "${roha_session.running_apps}")"
    roha_session_log "$(printf "+------------------+---------------------------------+\n")"

    return $RC_SUCCESS
} # End of "roha_session_open()"

#=============================================================================
#
# Name:        roha_session_close
#
# Description: Close a session. Actions done here are: updating ODM.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     None
#
# Returns:     None
#
#=============================================================================
function roha_session_close
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    #=======================================================
    # Write final data in HACMPdynresop ODM
    #=======================================================
    roha_session_write_odm_dynresop TIMESTAMP             "$(LC_ALL=C date)"
    roha_session_write_odm_dynresop MANAGED_SYSTEM        ""
    roha_session_write_odm_dynresop ENTERPRISE_POOL       ""
    roha_session_write_odm_dynresop PREFERRED_HMC_LIST    ""
    roha_session_write_odm_dynresop PREFERRED_NOVA_LIST   ""
    if [[ -z ${roha_session.optimal_apps} && -z ${roha_session.running_apps} \
        && ${roha_session.compute_only} == 0 ]] ; then
        #=======================================================
        # Clear HACMPdynresop on automatic release.
        #=======================================================
        roha_session_write_odm_dynresop DLPAR_MEM        ""
        roha_session_write_odm_dynresop DLPAR_PROCS      ""
        roha_session_write_odm_dynresop DLPAR_PROC_UNITS ""
        roha_session_write_odm_dynresop ONOFF_MEM        ""
        roha_session_write_odm_dynresop ONOFF_CPU        ""
        roha_session_write_odm_dynresop CODPOOL_MEM      ""
        roha_session_write_odm_dynresop CODPOOL_CPU      ""
        roha_session_write_odm_dynresop MAX_SPP_DIFF     "" 
    fi

    roha_session_print_odm_dynresop
    roha_session_log "Close session ${roha_session.id} at $(LC_ALL=C date)"

    return $RC_SUCCESS
} # End of "roha_session_close()"

#=============================================================================
#
# Name:        roha_session_set_running_apps
#
# Description: Compute a list of application controllers currently running on
#              the node.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     LOCALNODENAME    environment variable
#              GROUPNAME        environment variable
#              EVENT_TYPE       environment variable
#
# Returns:     RC_SUCCESS
#              RC_FAILURE
#
#=============================================================================
function roha_session_set_running_apps
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset acquiring_rgs=""
    typeset env_rgs=""
    typeset online_rgs=""
    typeset online_apps=""
    typeset async_apps=""
    typeset wait_apps=""

    #=======================================================
    : Get a list of resource groups which current state is
    : ACQUIRING or ONLINE. They are considered as running
    : for the current acquisition/release.
    : Also consider resource groups which current state is
    : ACQUIRE_SECONDARY or ONLINE_SECONDARY.
    #=======================================================
    out=$($CLRGINFO_CMD -s)
    if (( $? > 0 )) || [[ -z $out ]] ; then
        return $RC_FAILURE
    fi
    acquiring_rgs=$(print "$out" | awk -F: '($3 == "'$LOCALNODENAME'" && $2 == "ACQUIRING") {print $1}')
    acquiring_rgs=${acquiring_rgs:+$acquiring_rgs }$(print "$out" | awk -F: '($3 == "'$LOCALNODENAME'" && $2 == "ACQUIRING_SECONDARY") {print $1}')
    #======================================================================================
    : During nodeup rc.init starts the automatic release of resources PowerHA SystemMirror
    : In this case a check added if online_rgs_skip is "1",then skip fetching the list of resource groups which current state is online.
    #======================================================================================
    if (( $online_rgs_skip != 1 )) ; then
        online_rgs=$(print "$out" | awk -F: '($3 == "'$LOCALNODENAME'" && $2 == "ONLINE") {print $1}')
        online_rgs=${online_rgs:+$online_rgs }$(print "$out" | awk -F: '($3 == "'$LOCALNODENAME'" && $2 == "ONLINE_SECONDARY") {print $1}')
    fi
    #=======================================================
    : If DARE reconfig, avoid double counting resource
    : groups being currently processed.
    #=======================================================
    if [[ $EVENT_TYPE == reconfig* ]] ; then
        if [[ -n $GROUPNAME ]]; then
            out=$($CLRGINFO_CMD -s $GROUPNAME)
            if (( $? > 0 )) || [[ -z $out ]] ; then
                return $RC_FAILURE
            fi
            env_rgs=$(print "$out" | awk -F: '($3 == "'$LOCALNODENAME'" && ($2 == "ONLINE" || $2 == "ONLINE_SECONDARY" || $2 == "ACQUIRING" || $2 == "ACQUIRING_SECONDARY")) {print $1}')
        fi
        online_rgs=$(print "$online_rgs $env_rgs" | tr ' ' '\n' | sort -u)
    fi

    #=======================================================
    : Remove Resource Group from online_rgs if that entry
    : exist in acquiring_rgs list
    #=======================================================
    online_rgs=${online_rgs//+([[:space:]])/ }  # Replace newlines with spaces
    online_rgs=" $online_rgs "                  # Add space boundaries in case rg is first or last
    for rg in $acquiring_rgs ; do
        online_rgs=${online_rgs/ $rg / }            # Remove just that exact rg
    done

    #=======================================================
    : Get a list of application controllers for these
    : resource groups.
    #=======================================================
    for rg in $acquiring_rgs ; do
        online_apps=${online_apps:+"${online_apps} "}$($CLODMGET_CMD -q "group=$rg and name=APPLICATIONS" -nf value HACMPresource)
    done
    for rg in $online_rgs ; do
        online_apps=${online_apps:+"${online_apps} "}$($CLODMGET_CMD -q "group=$rg and name=APPLICATIONS" -nf value HACMPresource)
    done

    #=======================================================
    : Remove from that list application controllers that
    : are currently being brought online/offline.
    #=======================================================
    for app in $online_apps ; do
        if [[ ${roha_session.optimal_apps} != @(?(* )$app?( *)) ]] ; then
            roha_session.running_apps=${roha_session.running_apps:+${roha_session.running_apps} }$app
        fi
    done

    #========================================================
    : Remove all the application controllers that are 
    : currently being brought online from Async release list. 
    #========================================================
    if [[ ${roha_session.operation} == "acquire" ]] ; then
        wait_apps=$(ODMDIR=/etc/es/objrepos clodmget -q "key=WAIT_APPS" -nf value HACMPdynresop 2>/dev/null)
        for app in $wait_apps ; do
            if [[ ${roha_session.optimal_apps} != @(?(* )$app?( *)) ]] ; then
                async_apps=${async_apps:+${async_apps} }$app
            fi
        done
        if [[ -n $async_apps ]] ; then
            roha_session_write_odm_dynresop WAIT_APPS  "${async_apps}"
        fi
    fi

    return $RC_SUCCESS
} # End of "roha_session_set_running_apps()"

#=============================================================================
#
# Name:        roha_session_set_other
#
# Description: Determine the source lpar and source cec in case of acquisition
#              Determine the target lpar and target cec in case of release
#              Depending on the number of nodes in the cluster, computation is
#               different.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     LOCALNODENAME      environment variable
#              GROUPNAME          environment variable
#              GROUP_<RG>_<NODE>  environment variable
#              RESGRP_<RG>_<NODE> environment variable
#
# Returns:     RC_SUCCESS in all cases
#
#=============================================================================
function roha_session_set_other
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS
    typeset -i op_rc=$RC_SUCCESS

    typeset -i nb_of_nodes=$(clhandle -a | wc -l)

    if (( $nb_of_nodes == 1 )) ; then
        #=======================================================
        # On 1-node cluster, there is no source nor target lpar.
        #=======================================================
        roha_session.other_lpar=""
        roha_session.other_cec=""
    elif (( $nb_of_nodes >= 2 )) ; then
        #=======================================================
        # At least 2-node cluster needs special computation.
        #=======================================================
        for RG in $GROUPNAME ; do
            for NODE in $($CLODMGET_CMD -nf name HACMPnode | sort -u) ; do
                if [[ $NODE != $LOCALNODENAME ]] ; then
                    #=======================================================================
                    : In case of acquisition after a move or a takeover, the
                    :  source node is pointed out through a env variable which value is set to
                    :  ISUPPREEVENT. The env variable name contains the node name and the rg name.
                    : In case of release during a move or a takeover, the
                    :  target node is pointed out through a env variable which value is set to
                    :  WILLBEUPPOSTEVENT. The env variable name contains the node name and the rg name.
                    #=======================================================================
                    typeset NODE_RG_STATE=$(eval "echo \${GROUP_${RG}_${NODE//-/$HA_DASH_CHAR}}")

                    #=======================================================================
                    # If this node is trying to do an acquisition, after a move
                    #  or a takeover, we are actually looking for the source node
                    #  - which is given by the node on which resource group was ONLINE.
                    #=======================================================================

                    #=======================================================================
                    # If this node is trying to do a release, before a move
                    #  or a takeover, we are actually looking for the target node
                    #  - which is given by the node on which resource group will be ONLINE
                    #=======================================================================

                    if [[ ${roha_session.operation} == "acquire" && ${NODE_RG_STATE} == "ISUPPREEVENT" ]] \
                        || [[ ${roha_session.operation} == "release" && ${NODE_RG_STATE} == "WILLBEUPPOSTEVENT" ]]; then
                        #=======================================================================
                        # nodename/lparname match and nodename/cecname match are stored into
                        #  HACMPdynresop ODM
                        # we use cl_dynresop to retrieve match between lparname and nodename
                        #  into HACMPdynresop ODM
                        #=======================================================================
                        roha_session.other_lpar=$(LC_ALL=C cl_dynresop -n $NODE -L)
                        op_rc=$?
                        if (( op_rc == RC_SUCCESS)) ; then
                            typeset sys_name=""
                            typeset other_node_nova_list=
                            other_node_nova_list=$(cl_get_nova_list -n $NODE 2>/dev/null)
                            nova_rc=$?
                            if (( $nova_rc == 0 )) && [[ -n $other_node_nova_list ]]; then
                                sys_name=$(clnovacmd  -o query -a sys_name -L ${roha_session.other_lpar})
                            else
                                sys_name=$($CLHMCCMD_CMD  -o query -a sys_name -L ${roha_session.other_lpar})
                            fi
                            op_rc=$?
                            if (( op_rc == RC_SUCCESS)) ; then
                                if (( $nova_rc == 0 )) && [[ -n $other_node_nova_list ]]; then
                                    clnovacmd -o query -a sys_name -M ${sys_name} -H "$other_node_nova_list" | IFS=: read roha_session.other_cec skip
                                else
                                    $CLHMCCMD_CMD -o query -a sys_name -M ${sys_name} | IFS=: read roha_session.other_cec skip
                                fi
                                op_rc=$?
                            fi
                        fi
                        rc=$op_rc
                        break 2
                    fi
                fi
            done
        done
    fi

    #=======================================================================
    # Failing to get information on the other node should not prevent
    # execution of the operation. If we fail at getting full info, just
    # consider we don't have it at all.
    #=======================================================================
    if (( $rc != $RC_SUCCESS )) ; then
        roha_session.other_lpar=""
        roha_session.other_cec=""
        rc=$RC_SUCCESS
    fi

    return $rc
} # End of "roha_session_set_other()"

#=============================================================================
#
# Name:        roha_session_set_synchronous
#
# Description: Determine the mode of acquisition/release by looking if source
#              lpar and the target lpar are on the same managed system or not.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     None
#
# Returns:     RC_SUCCESS
#              RC_FAILURE
#
#=============================================================================
function roha_session_set_synchronous
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    if [[ ${roha_session.operation} == "release" ]] ; then

        if [[ $(roha_session_read_odm_rohaparam "force_sync_release") == 1 ]] ; then
            roha_session.synchronous=1
        elif [[ -n ${roha_session.other_cec} && -n ${roha_session.preferred_managed_system} ]] ; then
            if [[ ${roha_session.other_cec} == ${roha_session.preferred_managed_system} ]]; then
                roha_session.synchronous=1
            else
                roha_session.synchronous=0
            fi
        else
            roha_session.synchronous=1
        fi

    else # [[ ${roha_session.operation} == "acquire" ]]

        roha_session.synchronous=1

    fi

    return $RC_SUCCESS
} # End of "roha_session_set_synchronous()"

#=============================================================================
#
# Name:        roha_session_update_cluster_manager
#
# Description: Call cl_RMupdate to notify the cluster except if asynchronous.
#
# Inputs:      event        First parameter to pass to cl_RMupdate
#              param        Second parameter to pass to cl_RMupdate
#              script       Third parameter to pass to cl_RMupdate
#
# Outputs:     None
#
# Returns:     RC_SUCCESS
#
#=============================================================================
function roha_session_update_cluster_manager
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    #=======================================================
    : Call cl_RMupdate $1 $2 $3 only in synchronous mode,
    :  and only if normal mode -not compute only mode-,
    :  and only if systemmirror mode.
    #=======================================================
    if [[ ${roha_session.synchronous} == 1
                && ${roha_session.compute_only} == 0
                && ${roha_session.systemmirror_mode} == 1 ]] ; then
        $CLRMUPDATE_CMD "$1" "$2" "$3"
    fi

    return $RC_SUCCESS
} # End of "roha_session_update_cluster_manager()"

#=============================================================================
#
# Name:        roha_session_report_error
#
# Description: Notify cluster manager and all application controllers. The
#              astute reader will note that putting all these application
#              controllers into error state is rather harsh, since it is
#              possible that resources sufficient for some of them were
#              acquired. Similarly, any resources acquired for application
#              controllers above should probably be released here.
#              Unfortunately, the logic of this routine does not allow
#              discerning what, if anything, was acquired.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     PROGNAME
#
# Returns:     RC_SUCCESS
#
#=============================================================================
function roha_session_report_error
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    #=======================================================
    : Set Applications in error state only when acquiring.
    #=======================================================
    if [[ ${roha_session.operation} == "acquire" ]] ; then
        for app in ${roha_session.optimal_apps} ; do
            roha_session_update_cluster_manager resource_error $app $PROGNAME
        done
    fi

    #=======================================================
    : Notify Event Summary.
    #=======================================================
    tmp=$(dspmsg -s $ROHA_SET $ROHA_MSGS 102 "An error occurred while performing %1\$s operation.\n" "${roha_session.operation}")
    roha_session_update_cluster_manager event_sum_info "$tmp" $PROGNAME

    roha_session_log "$tmp"

    return $RC_SUCCESS
} # End of "roha_session_report_error()"

#=============================================================================
#
# Name:        roha_session_report_success
#
# Description: Display success message and update event summary.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     PROGNAME
#              roha_session
#              roha_query
#              roha_identify
#
# Returns:     RC_SUCCESS
#
#=============================================================================
function roha_session_report_success
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset string= tmp=
    string=${string:+$string}"\tThe following resources were %1\$d for application controllers %2\$d.\n"

    if (( CONN_TYPE == 2 )); then
        string=${string:+$string}"\tDLPAR memory: %3\$d GB.\n"
    else
        string=${string:+$string}"\tDLPAR memory: %3\$d GB     \tOn/Off CoD memory: %4\$d GB    \tEnterprise Pool memory: %5\$d GB.\n"
    fi

    if [[ ${roha_query.lpar.cpu.proc_mode} == "pvm" ]] ; then
        string=${string:+$string}"\tDLPAR processor: %4\$d CPU(s).\n"
        tmp=$(dspmsg -s $ROHA_SET $ROHA_MSGS 545 "$string" "${roha_session.operation}d" "${roha_session.optimal_apps}" ${roha_compute.mem} ${roha_compute.pu})

    elif [[ ${roha_query.lpar.cpu.proc_mode} == "shared" ]] ; then
        string=${string:+$string}"\tDLPAR processor: %6\$d PU/%7\$d VP\tOn/Off CoD processor: %8\$d CPU(s) \tEnterprise Pool processor: %9\$d CPU(s)\n"
        tmp=$(dspmsg -s $ROHA_SET $ROHA_MSGS 100 "$string" "${roha_session.operation}d" "${roha_session.optimal_apps}" ${roha_identify.dlpar.mem} ${roha_identify.onoff.mem} ${roha_identify.codpool.mem} ${roha_identify.dlpar.pu} ${roha_identify.dlpar.vp} ${roha_identify.onoff.cpu} ${roha_identify.codpool.cpu})

    else # [[ ${roha_query.lpar.cpu.proc_mode} == "ded" ]]
        string=${string:+$string}"\tDLPAR processor: %6\$d CPU(s)     \tOn/Off CoD processor: %7\$d CPU(s) \tEnterprise Pool processor: %8\$d CPU(s)\n"
        tmp=$(dspmsg -s $ROHA_SET $ROHA_MSGS 101 "$string" "${roha_session.operation}d" "${roha_session.optimal_apps}" ${roha_identify.dlpar.mem} ${roha_identify.onoff.mem} ${roha_identify.codpool.mem} ${roha_identify.dlpar.cpu} ${roha_identify.onoff.cpu} ${roha_identify.codpool.cpu})
    fi

    roha_session_update_cluster_manager event_sum_info "$tmp" $PROGNAME

    roha_session_log "$tmp"

    return $RC_SUCCESS
} # End of "roha_session_report_success()"

#=============================================================================
#
# Name:        roha_session_read_odm_dynresop
#
# Description: Display the field value of the matched criterium read from
#              HACMPdynresop ODM from DCD.
#
# Inputs:      key         The key criterium to search from ODM.
#
# Outputs:     None
#
# Returns:     None
#
#=============================================================================
function roha_session_read_odm_dynresop
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    out=$(ODMDIR=/etc/es/objrepos $CLODMGET_CMD -q "key=$1" -nf value HACMPdynresop)
    print -- ${out:-0}
} # End of "roha_session_read_odm_dynresop()"

#=============================================================================
#
# Name:        roha_session_write_odm_dynresop
#
# Description: Write value HACMPdynresop ODM stanza matching the key criterium
#              from DCD.
#
# Inputs:      key         The key criterium to search from ODM.
#              value       The value to write to ODM.
#
# Outputs:     None
#
# Returns:     None
#
#=============================================================================
function roha_session_write_odm_dynresop
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    if [[ -n $(ODMDIR=/etc/es/objrepos $CLODMGET_CMD -q "key=$1" -f value HACMPdynresop) ]] ; then
        print -- "HACMPdynresop:\\nvalue=\"$2\"" | ODMDIR=/etc/es/objrepos odmchange -o HACMPdynresop -q "key=$1"
    else
        print -- "HACMPdynresop:\\nkey=\"$1\"\\nvalue=\"$2\"" | ODMDIR=/etc/es/objrepos odmadd
    fi
} # End of "roha_session_write_odm_dynresop()"

#=============================================================================
#
# Name:        roha_session_print_odm_dynresop
#
# Description: Display all HACMPdynresop fields from ODM.
#
# Inputs:      None
#
# Outputs:     None
#
# Returns:     None
#
#=============================================================================
function roha_session_print_odm_dynresop
{
    roha_session_log "$(print "===== HACMPdynresop ODM ====")"
    roha_session_log "$(print "TIMESTAMP             = $(roha_session_read_odm_dynresop "TIMESTAMP")")"
  if (( CONN_TYPE != 2 )); then
    roha_session_log "$(print "MANAGED_SYSTEM        = $(roha_session_read_odm_dynresop "MANAGED_SYSTEM")")"
    roha_session_log "$(print "ENTERPRISE_POOL       = $(roha_session_read_odm_dynresop "ENTERPRISE_POOL")")"
    roha_session_log "$(print "PREFERRED_HMC_LIST    = $(roha_session_read_odm_dynresop "PREFERRED_HMC_LIST")")"
    roha_session_log "$(print "PREFERRED_NOVA_LIST   = $(roha_session_read_odm_dynresop "PREFERRED_NOVA_LIST")")"
  fi
    roha_session_log "$(print "DLPAR_MEM (GB)        = $(roha_session_read_odm_dynresop "DLPAR_MEM")")"
    roha_session_log "$(print "DLPAR_PROC_UNITS      = $(roha_session_read_odm_dynresop "DLPAR_PROC_UNITS")")"
  if (( CONN_TYPE != 2 )); then
    roha_session_log "$(print "DLPAR_PROCS           = $(roha_session_read_odm_dynresop "DLPAR_PROCS")")"
    roha_session_log "$(print "CODPOOL_MEM           = $(roha_session_read_odm_dynresop "CODPOOL_MEM")")"
    roha_session_log "$(print "CODPOOL_CPU           = $(roha_session_read_odm_dynresop "CODPOOL_CPU")")"
    roha_session_log "$(print "ONOFF_MEM             = $(roha_session_read_odm_dynresop "ONOFF_MEM")")"
    roha_session_log "$(print "ONOFF_CPU             = $(roha_session_read_odm_dynresop "ONOFF_CPU")")"
    roha_session_log "$(print "EFFECTIVE_HMC_VERSION = $(roha_session_read_odm_dynresop "EFFECTIVE_HMC_VERSION")")"
    roha_session_log "$(print "MAX_SPP_DIFF          = $(roha_session_read_odm_dynresop "MAX_SPP_DIFF")")"
  fi
    roha_session_log "$(print "============================\n")"
} # End of "roha_session_print_odm_dynresop()"

#=============================================================================
#
# Name:        roha_session_read_odm_rohaparam
#
# Description: Display the value of the field read from HACMProhaparam ODM.
#
# Inputs:      field       The field to read from ODM.
#
# Outputs:     None
#
# Returns:     None
#
#=============================================================================
function roha_session_read_odm_rohaparam
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    print -- $($CLODMGET_CMD -nf $1 HACMProhaparam)
} # End of "roha_session_read_odm_rohaparam()"

#=============================================================================
#
# Name:        roha_session_write_odm_rohaparam
#
# Description: Write HACMProhaparam ODM.
#
# Inputs:      field       The field to modify.
#              value       The value to write into the field.
#
# Outputs:     None
#
# Returns:     None
#
#=============================================================================
function roha_session_write_odm_rohaparam
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    if [[ -n $($CLODMGET_CMD -nf $1 HACMProhaparam) ]] ; then
        print -- "HACMProhaparam:\\n$1=\"$2\"" | odmchange -o HACMProhaparam
    else
        print -- "HACMProhaparam:\\n$1=\"$2\"" | odmadd
    fi
} # End of "roha_session_write_odm_rohaparam()"

#=============================================================================
#
# Name:        roha_session_print_odm_rohaparam
#
# Description: Displays all HACMProhaparam fields from ODM.
#
# Inputs:      None
#
# Outputs:     None
#
# Returns:     None
#
#=============================================================================
function roha_session_print_odm_rohaparam
{
    roha_session_log "$(print "==== HACMProhaparam ODM ====")"
    roha_session_log "$(print "ALWAYS_START_RG    = $(roha_session_read_odm_rohaparam "always_start_rg")")"
    roha_session_log "$(print "FORCE_SYNC_RELEASE = $(roha_session_read_odm_rohaparam "force_sync_release")")"
  if (( CONN_TYPE != 2 )); then
    roha_session_log "$(print "ADJUST_SPP_SIZE    = $(roha_session_read_odm_rohaparam "adjust_spp_size")")"
    roha_session_log "$(print "AGREE_TO_COD_COSTS = $(roha_session_read_odm_rohaparam "agree_to_cod_costs")")"
    roha_session_log "$(print "ONOFF_DAYS         = $(roha_session_read_odm_rohaparam "onoff_days")")"
    roha_session_log "$(print "RESOURCE_ALLOCATION_ORDER = $(roha_session_read_odm_rohaparam "resource_allocation_order")")"
  fi
    roha_session_log "$(print "============================\n")"
} # End of "roha_session_print_odm_rohaparam()"


#=========================================================================================
#
# Name:        roha_session_wait_on_async_release
#
# Description: Waits on async release to complete. Maximum waiting time is config_too_long
#              timeout and returns before RG(s) moving to error state
#
# Inputs:      None
#
# Outputs:     None
#
# Returns:     RC_SUCCESS
#              or else an error code
#
#==========================================================================================
function roha_session_wait_on_async_release
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS
    
    #=============================================================================
    # Get cluster max event time and multiply by 2 to get config_too_long timeout.
    #=============================================================================
    typeset -i max_time=$(clmgr -cS -a MAX_EVENT_TIME qu cluster)
    (( max_time *= 2 ))
    typeset -i timeOut=max_time
    
    roha_session_log "Waiting on async release to complete\n"
    while (( $timeOut > 0 )); do
        #====================================================================
        # Check if Async release is in progress
        #====================================================================
        typeset rel_apps=$(ODMDIR=/etc/es/objrepos clodmget -q "key=REL_APPS" -nf value HACMPdynresop 2>/dev/null)
    
        if (( $? == 0 )) && [[ -n $rel_apps ]] ; then
            sleep 2
            (( timeOut -= 2 ))
            continue
        else
            roha_session_log "Coming out of wait loop as there is no async release in progress\n"
            break    
        fi
    
    done
    
    if (( $timeOut <= 0 )); then
        roha_session_log "$(dspmsg -s $ROHA_SET $ROHA_MSGS 535 \
        "%1\$s: ERROR: Waited for %2\$d sec on asynchronous release of resources, but still in progress. Could not proceed, try again later on.\n" \
        "$PROGNAME" "$max_time")"
        rc=$RC_PARAM_ERROR
        return $rc
    fi
            
    return $rc    
} # End of "roha_session_wait_on_async_release()"
