#!/bin/ksh93
#  ALTRAN_PROLOG_BEGIN_TAG                                                    
#  This is an automatically generated prolog.                                  
#                                                                              
#  Copyright (C) Altran ACT S.A.S. 2017,2019,2020,2021,2022.  All rights reserved.  
#                                                                              
#  ALTRAN_PROLOG_END_TAG                                                      
#                                                                              
###############################################################################
# @(#)  5881272 43haes/usr/sbin/cluster/events/utils/clapplyroha.sh, 61aha_r726, 2205A_aha726, May 16 2022 12:15 PM
###############################################################################

#########################################################################
#
#   COMPONENT_NAME: hacmp.events
#
#   CLASS:
#
#       roha_session
#
#   DESCRIPTION:
#
#       This file contains all atomic functions needed to perform
#        'apply' step, which is the fourth step of the ROHA runtime
#        (query step, compute step, identify step, apply step).
#
#       The apply step can be done for acquisition or for release.
#       The apply step is done once for memory and once for CPUs (dedicated
#         processing mode) or for PUs and VPs (shared processing mode).
#
#
#   FUNCTIONS:
#
#       roha_apply_dlpar_resources
#           acquire_dlpar_resources
#           release_dlpar_resources
#
#       roha_apply_onoff_resources
#           activate_onoff_resources
#           deactivate_onoff_resources
#           activate_onoff_mem_resources
#           activate_onoff_cpu_resources
#           reactivate_onoff_mem_resources
#           reactivate_onoff_cpu_resources
#           deactivate_onoff_mem_resources
#           deactivate_onoff_cpu_resources
#
#       roha_apply_yank_codpool_resources
#       roha_apply_codpool_resources
#           allocate_codpool_resources
#           deallocate_codpool_resources
#
#       roha_apply_rollback_all_resources
#
#########################################################################

#================================================
# The following, commented line enforces coding
# standards when this file is edited via vim.
#================================================
# vim:tabstop=4:shiftwidth=4:expandtab:smarttab
#================================================
#Added for Availability Metrics logs
. /usr/es/lib/ksh93/availability/cl_amlib

typeset roha_rollback=(
    typeset dlpar=(
        typeset -F4 mem=0.00
        typeset -i  cpu=0
        typeset -F4 pu=0.00
        typeset -i  vp=0
    )
    typeset onoff=(
        typeset -F4 mem=0.00
        typeset -F4 cpu=0.00
    )
    typeset codpool=(
        typeset -F4 mem=0.00
        typeset -F4 cpu=0.00
    )
)

#=============================================================================
#
# Name:        initialize_apply_globals
#
# Description: Initialize global vars specific to this file that may otherwise
#              be used unreinitialized during reassessment
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_rollback
#
# Returns:     RC_SUCCESS
#
#=============================================================================
function initialize_apply_globals
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    roha_rollback.dlpar.mem=0.00
    roha_rollback.dlpar.cpu=0
    roha_rollback.dlpar.pu=0.00
    roha_rollback.dlpar.vp=0

    roha_rollback.onoff.mem=0.00
    roha_rollback.onoff.cpu=0.00

    roha_rollback.codpool.mem=0.00
    roha_rollback.codpool.cpu=0.00

    return $RC_SUCCESS
}

#=============================================================================
#
# Name:        roha_apply_dlpar_resources
#
# Description: Encapsulation of DLPAR memory and processor acquisition/release
#
# Inputs:      $1 to indicate if we are in last loop of reassessment.
#              If set to 1, we are in last loop of reassessment.
#              If set to 0, we are in normal reassessment.
#
# Outputs:     None
#
# Globals:     roha_session
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function roha_apply_dlpar_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS

    typeset -i LAST_LOOP_OF_FORCE_REASSESSMENT=$1

    #=================================================================
    : LAST_LOOP_OF_FORCE_REASSESSMENT=$LAST_LOOP_OF_FORCE_REASSESSMENT
    #=================================================================

    if [[ ${roha_session.operation} == "acquire" ]] ; then

        acquire_dlpar_resources $LAST_LOOP_OF_FORCE_REASSESSMENT
        rc=$?

    else # [[ ${roha_session.operation} == "release" ]]

        release_dlpar_resources
        rc=$?

    fi

    return $rc
} # End of "roha_apply_dlpar_resources()"

#=============================================================================
#
# Name:        acquire_dlpar_resources
#
# Description: Allocate memory and processor resources from the free pool to
#              the LPAR.
#
# Inputs:      $1 to indicate if we are in last loop of reassessment.
#              If set to 1, we are in last loop of reassessment.
#              If set to 0, we are in normal reassessment.
#
# Outputs:     None
#
# Globals:     roha_query
#              roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function acquire_dlpar_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS
    typeset    opt=""
    typeset    am_log=""

    typeset -i LAST_LOOP_OF_FORCE_REASSESSMENT=$1

    #=================================================================
    : LAST_LOOP_OF_FORCE_REASSESSMENT=$LAST_LOOP_OF_FORCE_REASSESSMENT
    #=================================================================

    # For ROHA on Cloud
    if [[ ${roha_query.lpar.cpu.proc_mode} == "pvm" ]] ; then

        typeset -F4 mem=${roha_compute.total_mem}
        typeset -F4 pu=${roha_compute.total_pu}

        if (( $mem > 0.00 || $pu > 0.00 )) ; then
            if (( $(roha_session_read_odm_rohaparam "always_start_rg") == 0 )) ||
                        (( $(roha_session_read_odm_rohaparam "always_start_rg") == 1 &&
                                $LAST_LOOP_OF_FORCE_REASSESSMENT == 0 )) ; then
                #=========================================================================
                : always_start_rg=0   or   always_start_rg=1 but not last loop of reassessment
                #=========================================================================

                roha_session_update_cluster_manager resource_acquiring DLPAR $PROGNAME

                (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $mem" && am_log=${am_log:+$am_log }"$mem GB memory"
                (( $pu  > 0.00 )) && opt=${opt:+$opt }"-p $pu" && am_log=${am_log:+$am_log, }"$pu processing_units"
                
                amlog_trace $AM_ROHA_ACQUIRE_BEGIN "Cloud DLPAR operation for $am_log"
                roha_apply_cloud_operations acquire "$opt"
                rc=$?
                if (( $rc > 0 )) ; then
                    roha_session_update_cluster_manager resource_error DLPAR $PROGNAME
                else
                    amlog_trace $AM_ROHA_ACQUIRE_END "Cloud DLPAR operation for $am_log"
                    roha_rollback.dlpar.mem=$mem
                    roha_rollback.dlpar.pu=$pu
                    roha_rollback.dlpar.vp=$vp
                    roha_session_write_odm_dynresop DLPAR_MEM        $(( $(roha_session_read_odm_dynresop "DLPAR_MEM")        + mem ))
                    roha_session_write_odm_dynresop DLPAR_PROC_UNITS $(( $(roha_session_read_odm_dynresop "DLPAR_PROC_UNITS") + pu  ))
                    roha_session_update_cluster_manager resource_up DLPAR $PROGNAME
                fi

            else
                #=========================================================================
                : always_start_rg=1 and last loop of reassessment
                #=========================================================================

                typeset -i op_rc=$RC_UNKNOWN

                #=========================================================================
                : mem step
                #=========================================================================

                if (( $mem > 0.00 )); then
                   opt="-m $(( mem )) " && am_log="$(( mem ))GB memory"
                   amlog_trace $AM_ROHA_ACQUIRE_BEGIN "DLPAR operation for am_log"
                   out=$($CLHMCCMD_CMD -o acquire $opt 3>&1)
                   op_rc=$?
                   roha_session_log "$out"
                   if (( $op_rc > 0 )) ; then
                       rc=$op_rc
                   else
                       amlog_trace $AM_ROHA_ACQUIRE_END "DLPAR operation for $am_log"
                       roha_rollback.dlpar.mem=$mem
                       roha_session_write_odm_dynresop DLPAR_MEM        $(( $(roha_session_read_odm_dynresop "DLPAR_MEM")        + mem ))
                   fi
                fi 
                #=========================================================================
                : As we are in always_start_rg=1 and last loop of reassessment,
                :  whatever the result we continue.
                #=========================================================================
                opt=""
                am_log=""
                #=========================================================================
                : procs step
                #=========================================================================
                if (( $pu  > 0.00 )); then
                    opt="-p $pu" && am_log=${am_log:+$am_log }"$pu processing_units" 
                    [[ -n $opt ]] && opt=${opt:+$opt }
                    amlog_trace $AM_ROHA_ACQUIRE_BEGIN "DLPAR operation for $am_log"
                    out=$($CLHMCCMD_CMD -o acquire $opt 3>&1)
                    op_rc=$?
                    roha_session_log "$out"
                    if (( $op_rc > 0 )) ; then
                        rc=$op_rc
                        roha_session_update_cluster_manager resource_error DLPAR $PROGNAME
                    else
                        amlog_trace $AM_ROHA_ACQUIRE_END "DLPAR operation for $am_log"
                        roha_rollback.dlpar.pu=$pu
                        roha_session_write_odm_dynresop DLPAR_PROC_UNITS $(( $(roha_session_read_odm_dynresop "DLPAR_PROC_UNITS") + pu ))
                    fi
                fi
                if (( $rc > 0 )); then
                    #=========================================================================
                    : rc is forced to SUCCESS anyway, as we are in always_start_rg=1
                    :  and last loop of reassessment
                    #=========================================================================
                    rc=$RC_SUCCESS

                    #=========================================================================
                    : both steps mem and procs succeeded or are considered as
                    :  succeeded even if failed
                    #=========================================================================
                    roha_session_update_cluster_manager resource_up DLPAR $PROGNAME
                else
                    #=========================================================================
                    : both steps mem and procs succeeded
                    #=========================================================================
                    roha_session_update_cluster_manager resource_up DLPAR $PROGNAME
                fi

            fi
        fi

    elif [[ ${roha_query.lpar.cpu.proc_mode} == "shared" ]] ; then

        typeset -F4 mem=${roha_identify.dlpar.mem}
        typeset -F4 pu=${roha_identify.dlpar.pu}
        typeset -i  vp=${roha_identify.dlpar.vp}
        typeset -i  spp_size=${roha_identify.dlpar.spp_size}
        max_spp_diff=$(roha_session_read_odm_dynresop "MAX_SPP_DIFF")
        if (( $spp_size > ${roha_query.lpar.cpu.proc_units.spp_max} )) ; then
            if [[ -n ${roha_session.preferred_nova_list} ]]; then
                out=$(clnovacmd -o change -a max_pool_proc_units=$spp_size 3>&1)
            else
                out=$($CLHMCCMD_CMD -o change -a max_pool_proc_units=$spp_size 3>&1)
            fi
            rc=$?
            if (( $rc > 0 )) ; then
                roha_session_update_cluster_manager resource_error SPP_SIZE $PROGNAME
            else
                roha_session_log "$(printf "Shared Processor Pool maximum size changed to: %d\n" $spp_size)"
                max_spp_diff=$(max 0 $(( max_spp_diff + spp_size - ${roha_query.lpar.cpu.proc_units.spp_max} )))
                roha_session_write_odm_dynresop MAX_SPP_DIFF $max_spp_diff
            fi
        fi

        if (( $mem > 0.00 || $pu > 0.00 || $vp > 0.00 )) ; then
            if (( $(roha_session_read_odm_rohaparam "always_start_rg") == 0 )) ||
                        (( $(roha_session_read_odm_rohaparam "always_start_rg") == 1 &&
                                $LAST_LOOP_OF_FORCE_REASSESSMENT == 0 )) ; then
                #=========================================================================
                : always_start_rg=0   or   always_start_rg=1 but not last loop of reassessment
                #=========================================================================

                roha_session_update_cluster_manager resource_acquiring DLPAR $PROGNAME

                (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $(( mem * 1024 ))" && am_log=${am_log:+$am_log }"$(( mem * 1024 ))MB memory"
                (( $pu  > 0.00 )) && opt=${opt:+$opt }"-p $pu" && am_log=${am_log:+$am_log, }"$pu processing_units"
                (( $vp  > 0.00 )) && opt=${opt:+$opt }"-q $vp" && am_log=${am_log:+$am_log, }"$vp virtual_processors"
                
                amlog_trace $AM_ROHA_ACQUIRE_BEGIN "DLPAR operation for $am_log" 
                if [[ -n ${roha_session.preferred_nova_list} ]]; then
                    out=$(clnovacmd -o acquire -r dlpar $opt 3>&1)
                else
                    out=$($CLHMCCMD_CMD -o acquire -r dlpar $opt 3>&1)
                fi
                rc=$?
                roha_session_log "$out"
                if (( $rc > 0 )) ; then
                    roha_session_update_cluster_manager resource_error DLPAR $PROGNAME
                else
                    amlog_trace $AM_ROHA_ACQUIRE_END "DLPAR operation for $am_log"
                    roha_rollback.dlpar.mem=$mem
                    roha_rollback.dlpar.pu=$pu
                    roha_rollback.dlpar.vp=$vp
                    roha_session_write_odm_dynresop DLPAR_MEM        $(( $(roha_session_read_odm_dynresop "DLPAR_MEM")        + mem ))
                    roha_session_write_odm_dynresop DLPAR_PROCS      $(( $(roha_session_read_odm_dynresop "DLPAR_PROCS")      + vp  ))
                    roha_session_write_odm_dynresop DLPAR_PROC_UNITS $(( $(roha_session_read_odm_dynresop "DLPAR_PROC_UNITS") + pu  ))
                    roha_session_update_cluster_manager resource_up DLPAR $PROGNAME
                fi

            else
                #=========================================================================
                : always_start_rg=1 and last loop of reassessment
                #=========================================================================

                typeset -i op_rc=$RC_UNKNOWN

                #=========================================================================
                : mem step
                #=========================================================================

                if (( $mem > 0.00 )); then
                   opt="-m $(( mem * 1024 )) -f" && am_log="$(( mem * 1024 ))MB memory"
                   amlog_trace $AM_ROHA_ACQUIRE_BEGIN "DLPAR operation for am_log"
                   if [[ -n ${roha_session.preferred_nova_list} ]]; then
                      out=$(clnovacmd -o acquire -r dlpar $opt 3>&1)
                   else
                      out=$($CLHMCCMD_CMD -o acquire -r dlpar $opt 3>&1)
                   fi
                   op_rc=$?
                   roha_session_log "$out"
                   if (( $op_rc > 0 )) ; then
                       rc=$op_rc
                   else
                       amlog_trace $AM_ROHA_ACQUIRE_END "DLPAR operation for $am_log"
                       roha_rollback.dlpar.mem=$mem
                       roha_session_write_odm_dynresop DLPAR_MEM        $(( $(roha_session_read_odm_dynresop "DLPAR_MEM")        + mem ))
                   fi
                fi 
                #=========================================================================
                : As we are in always_start_rg=1 and last loop of reassessment,
                :  whatever the result we continue.
                #=========================================================================
                opt=""
                am_log=""
                #=========================================================================
                : procs step
                #=========================================================================
                if ((  $pu  > 0.00 || $vp  > 0.00 )); then
                    (( $pu  > 0.00 )) && opt="-p $pu" && am_log=${am_log:+$am_log }"$pu processing_units" 
                    (( $vp  > 0.00 )) && opt=${opt:+$opt }"-q $vp" && am_log=${am_log:+$am_log, }"$vp virtual_processors"
                    [[ -n $opt ]] && opt=${opt:+$opt }"-f"
                    amlog_trace $AM_ROHA_ACQUIRE_BEGIN "DLPAR operation for $am_log"
                    if [[ -n ${roha_session.preferred_nova_list} ]]; then
                        out=$(clnovacmd -o acquire -r dlpar $opt 3>&1)
                    else
                        out=$($CLHMCCMD_CMD -o acquire -r dlpar $opt 3>&1)
                    fi
                    op_rc=$?
                    roha_session_log "$out"
                    if (( $op_rc > 0 )) ; then
                        rc=$op_rc
                        roha_session_update_cluster_manager resource_error DLPAR $PROGNAME
                    else
                        amlog_trace $AM_ROHA_ACQUIRE_END "DLPAR operation for $am_log"
                        roha_rollback.dlpar.pu=$pu
                        roha_rollback.dlpar.vp=$vp
                        roha_session_write_odm_dynresop DLPAR_PROCS      $(( $(roha_session_read_odm_dynresop "DLPAR_PROCS")      + vp ))
                        roha_session_write_odm_dynresop DLPAR_PROC_UNITS $(( $(roha_session_read_odm_dynresop "DLPAR_PROC_UNITS") + pu ))
                    fi
                fi
                if (( $rc > 0 )); then
                    #=========================================================================
                    : rc is forced to SUCCESS anyway, as we are in always_start_rg=1
                    :  and last loop of reassessment
                    #=========================================================================
                    rc=$RC_SUCCESS

                    #=========================================================================
                    : both steps mem and procs succeeded or are considered as
                    :  succeeded even if failed
                    #=========================================================================
                    roha_session_update_cluster_manager resource_up DLPAR $PROGNAME
                else
                    #=========================================================================
                    : both steps mem and procs succeeded
                    #=========================================================================
                    roha_session_update_cluster_manager resource_up DLPAR $PROGNAME
                fi

            fi
        fi

    else # [[ ${roha_query.lpar.cpu.proc_mode} == "ded" ]]

        typeset -F4 mem=${roha_identify.dlpar.mem}
        typeset -i  cpu=${roha_identify.dlpar.cpu}

        if (( $mem > 0.00 || $cpu > 0.00 )) ; then

            if (( $(roha_session_read_odm_rohaparam "always_start_rg") == 0 )); then
                #=========================================================================
                : always_start_rg=0
                #=========================================================================
                roha_session_update_cluster_manager resource_acquiring DLPAR $PROGNAME

                (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $(( mem * 1024 ))" && am_log=${am_log:+$am_log }"$(( mem * 1024 ))MB memory"
                (( $cpu > 0.00 )) && opt=${opt:+$opt }"-q $cpu" && am_log=${am_log:+$am_log, }"$cpu CPUs"
                (( $(roha_session_read_odm_rohaparam "always_start_rg") == 1 )) && opt=${opt:+$opt }"-f"
                   
                amlog_trace $AM_ROHA_ACQUIRE_BEGIN "DLPAR operation for $am_log"
                if [[ -n ${roha_session.preferred_nova_list} ]]; then
                    out=$(clnovacmd -o acquire -r dlpar $opt 3>&1)
                else
                    out=$($CLHMCCMD_CMD -o acquire -r dlpar $opt 3>&1)
                fi
                rc=$?
                roha_session_log "$out"
                if (( $rc > 0 )) ; then
                    roha_session_update_cluster_manager resource_error DLPAR $PROGNAME
                else
                    amlog_trace $AM_ROHA_ACQUIRE_END "DLPAR operation for $am_log"
                    roha_rollback.dlpar.mem=$mem
                    roha_rollback.dlpar.cpu=$cpu
                    roha_session_write_odm_dynresop DLPAR_MEM        $(( $(roha_session_read_odm_dynresop "DLPAR_MEM")   + mem ))
                    roha_session_write_odm_dynresop DLPAR_PROCS      $(( $(roha_session_read_odm_dynresop "DLPAR_PROCS") + cpu ))
                    roha_session_update_cluster_manager resource_up DLPAR $PROGNAME
                fi
            else
                #=========================================================================
                : always_start_rg=1
                #=========================================================================
                typeset -i op_rc=$RC_UNKNOWN

                #=========================================================================
                : mem step
                #=========================================================================
                if (( $mem > 0.00 )); then
                    opt="-m $(( mem * 1024 )) -f" && am_log=${am_log:+$am_log }"$(( mem * 1024 ))MB memory"

                    amlog_trace $AM_ROHA_ACQUIRE_BEGIN "DLPAR operation for $am_log"
                    if [[ -n ${roha_session.preferred_nova_list} ]]; then
                       out=$(clnovacmd -o acquire -r dlpar $opt 3>&1)
                    else
                       out=$($CLHMCCMD_CMD -o acquire -r dlpar $opt 3>&1)
                    fi
                    op_rc=$?
                    roha_session_log "$out"
                    if (( $op_rc > 0 )) ; then
                       rc=$op_rc
                    else
                       amlog_trace $AM_ROHA_ACQUIRE_END "DLPAR operation for $am_log"
                       roha_rollback.dlpar.mem=$mem
                       roha_session_write_odm_dynresop DLPAR_MEM        $(( $(roha_session_read_odm_dynresop "DLPAR_MEM")   + mem ))
                    fi
                fi

                #=========================================================================
                : As we are in always_start_rg=1, whatever the result we continue.
                #=========================================================================
                opt=""
                am_log=""
                #=========================================================================
                : procs step
                #=========================================================================
                if (( $cpu > 0.00 )); then
                    opt="-q $cpu -f" && am_log=${am_log:+$am_log }"$cpu CPUs"

                    amlog_trace $AM_ROHA_ACQUIRE_BEGIN "DLPAR operation for $am_log"
                    if [[ -n ${roha_session.preferred_nova_list} ]]; then
                       out=$(clnovacmd -o acquire -r dlpar $opt 3>&1)
                    else
                       out=$($CLHMCCMD_CMD -o acquire -r dlpar $opt 3>&1)
                    fi
                    op_rc=$?
                    roha_session_log "$out"
                    if (( $op_rc > 0 )) ; then
                       rc=$op_rc
                    else
                        amlog_trace $AM_ROHA_ACQUIRE_END "DLPAR operation for $am_log"
                        roha_rollback.dlpar.cpu=$cpu
                        roha_session_write_odm_dynresop DLPAR_PROCS      $(( $(roha_session_read_odm_dynresop "DLPAR_PROCS") + cpu ))
                        roha_session_update_cluster_manager resource_up DLPAR $PROGNAME
                    fi

                fi 
                if (( $rc > 0 )); then
                   #=========================================================================
                   : rc is forced to SUCCESS anyway, as we are in always_start_rg=1
                   #=========================================================================
                   rc=$RC_SUCCESS

                   #=========================================================================
                   : at least one step mem or procs failed
                   #=========================================================================
                   roha_session_update_cluster_manager resource_error DLPAR $PROGNAME
                else
                   #=========================================================================
                   : both steps mem and procs succeeded
                   #=========================================================================
                   roha_session_update_cluster_manager resource_up DLPAR $PROGNAME
                fi
            fi    
        fi
    fi
    return $rc
} # End of "acquire_dlpar_resources()"

#=============================================================================
#
# Name:        release_dlpar_resources
#
# Description: Release memory and processor resources back to the free pool.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_query
#              roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function release_dlpar_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS
    typeset opt=""
    typeset am_log=""
    if (( ${roha_session.synchronous} == 0 )) ; then
        #=======================================================
        : In case of asynchronous release, this workaround has
        : been added to avoid ssh hangs.
        : Wait at most 3 minutes for the particular process
        : "clcallev rg_move xxx xxx RELEASE" to terminate.
        : Thus, parent process is ensured to be attached to the
        : shell and ssh will not hang.
        : This piece of code is run in the background. Thus, it
        : should not block event scripts and should not generate
        : config_too_long event.
        #=======================================================
        typeset -i wait_loop=0
        typeset -i wait_time=180
        typeset wpid=$(ps -edf | awk '$8=="clcallev" && $9=="rg_move" && $12=="RELEASE" {print $2}')
        roha_session_log "waiting up to $wait_time seconds for parent process which pid is $wpid to terminate before initiate ssh release commands"
        if [[ -n $wpid ]] ; then
            while (( wait_loop < $wait_time )) ; do
                ps -edfo pid | grep -qw $wpid
                if (( $? == 0 )) ; then
                    sleep 1;
                    (( wait_loop++ ))
                    (( wait_loop % 30 == 0 )) && roha_session_log "waited $wait_loop seconds ..."
                else
                    roha_session_log "operation completed after $wait_loop seconds"
                    break
                fi
            done
        fi
    fi

    # For ROHA on Cloud
    if [[ ${roha_query.lpar.cpu.proc_mode} == "pvm" ]] ; then

        typeset -F4 mem=${roha_compute.total_mem}
        typeset -F4 pu=${roha_compute.total_pu}
        typeset -F4 new_pu

        if (( $mem > 0.00 || $pu > 0.00 )) ; then

            roha_session_update_cluster_manager resource_releasing DLPAR $PROGNAME

            (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $mem" && am_log=${am_log:+$am_log }"$mem GB memory"
            (( $pu  > 0.00 )) && opt=${opt:+$opt }"-p $pu" && am_log=${am_log:+$am_log, }"$pu processing_units"

            amlog_trace $AM_ROHA_RELEASE_BEGIN "Cloud DLPAR operation for $am_log"
            roha_apply_cloud_operations release "$opt"
            rc=$?
            if (( $rc > 0 )) ; then
                roha_session_update_cluster_manager resource_error DLPAR $PROGNAME
            else
                amlog_trace $AM_ROHA_RELEASE_END "Cloud DLPAR operation for $am_log"
                new_pu=$(max 0 $(( $(roha_session_read_odm_dynresop "DLPAR_PROC_UNITS") - pu  )))
                roha_session_write_odm_dynresop DLPAR_MEM        $(max 0 $(( $(roha_session_read_odm_dynresop "DLPAR_MEM") - mem )))
                roha_session_write_odm_dynresop DLPAR_PROC_UNITS $new_pu
                roha_session_update_cluster_manager resource_down DLPAR $PROGNAME
            fi
        fi

    elif [[ ${roha_query.lpar.cpu.proc_mode} == "shared" ]] ; then

        typeset -F4 mem=${roha_identify.dlpar.mem}
        typeset -F4 pu=${roha_identify.dlpar.pu}
        typeset -i  vp=${roha_identify.dlpar.vp}
        typeset -i max_spp_size=${roha_query.lpar.cpu.proc_units.spp_max}
        typeset -F4 free_spp_size=${roha_query.lpar.cpu.proc_units.spp_free}
        typeset -F4 reserved_spp_size=${roha_query.lpar.cpu.proc_units.spp_reserved}
        typeset -i old_max_spp_diff=0
        typeset -i new_max_spp_diff=0
        typeset -i new_spp_size=0
        typeset -F4 required_pu
        typeset -F4 new_pu
        typeset -F4 other_lpars_using_pool

        if (( $mem > 0.00 || $pu > 0.00 || $vp > 0.00 )) ; then

            roha_session_update_cluster_manager resource_releasing DLPAR $PROGNAME

            (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $(( mem * 1024 ))" && am_log=${am_log:+$am_log }"$(( mem * 1024 ))MB memory"
            (( $pu  > 0.00 )) && opt=${opt:+$opt }"-p $pu" && am_log=${am_log:+$am_log, }"$pu processing_units"
            (( $vp  > 0.00 )) && opt=${opt:+$opt }"-q $vp" && am_log=${am_log:+$am_log, }"$vp virtual_processors"
            opt=${opt:+$opt }"-f"

            amlog_trace $AM_ROHA_RELEASE_BEGIN "DLPAR operation for $am_log" 
            if [[ -n ${roha_session.preferred_nova_list} ]]; then
                out=$(clnovacmd -o release -r dlpar $opt 3>&1)
            else
                out=$($CLHMCCMD_CMD -o release -r dlpar $opt 3>&1)
            fi
            rc=$?
            roha_session_log "$out"
            if (( $rc > 0 )) ; then
                roha_session_update_cluster_manager resource_error DLPAR $PROGNAME
            else
                amlog_trace $AM_ROHA_RELEASE_END "DLPAR operation for $am_log"
                new_pu=$(max 0 $(( $(roha_session_read_odm_dynresop "DLPAR_PROC_UNITS") - pu  )))
                roha_session_write_odm_dynresop DLPAR_MEM        $(max 0 $(( $(roha_session_read_odm_dynresop "DLPAR_MEM")        - mem )))
                roha_session_write_odm_dynresop DLPAR_PROCS      $(max 0 $(( $(roha_session_read_odm_dynresop "DLPAR_PROCS")      - vp  )))
                roha_session_write_odm_dynresop DLPAR_PROC_UNITS $new_pu
                if [[ ${roha_query.lpar.cpu.proc_units.spp_name} != "DefaultPool" ]]; then
                    old_max_spp_diff=$(roha_session_read_odm_dynresop "MAX_SPP_DIFF")
                    new_max_spp_diff=$(max 0 $(( old_max_spp_diff - pu - free_spp_size )))
                    new_spp_size=$(max 0 $(( max_spp_size - old_max_spp_diff + new_max_spp_diff )))
                    other_lpars_using_pool=$(( max_spp_size - free_spp_size - new_pu - reserved_spp_size - pu ))
                    required_pu=$(( reserved_spp_size + new_pu + other_lpars_using_pool ))
                    if (( required_pu > new_spp_size )); then
                        new_spp_size=$(( new_spp_size + 1 ))
                        new_max_spp_diff=$(( new_max_spp_diff + 1 ))
                    fi
                    if [[ -n ${roha_session.preferred_nova_list} ]]; then
                        out=$(clnovacmd -o change -a max_pool_proc_units=$new_spp_size 3>&1)
                    else
                        out=$($CLHMCCMD_CMD -o change -a max_pool_proc_units=$new_spp_size 3>&1)
                    fi
                    rc=$?
                    if (( $rc > 0 )) ; then
                        roha_session_update_cluster_manager resource_error SPP_SIZE $PROGNAME
                    else
                        roha_session_log "$(printf "Shared Processor Pool maximum size changed to: %d\n" $new_spp_size)"
                        roha_session_write_odm_dynresop MAX_SPP_DIFF $new_max_spp_diff
                    fi
                fi  
                roha_session_update_cluster_manager resource_down DLPAR $PROGNAME
            fi

        fi

    else # [[ ${roha_query.lpar.cpu.proc_mode} == "ded" ]]

        typeset -F4 mem=${roha_identify.dlpar.mem}
        typeset -i  cpu=${roha_identify.dlpar.cpu}

        if (( $mem > 0.00 || $cpu > 0.00 )) ; then

            roha_session_update_cluster_manager resource_releasing DLPAR $PROGNAME

            (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $(( mem * 1024 ))" && am_log=${am_log:+$am_log }"$(( mem * 1024 ))MB memory"
            (( $cpu > 0.00 )) && opt=${opt:+$opt }"-q $cpu" && am_log=${am_log:+$am_log, }"$cpu CPUs" 
            opt=${opt:+$opt }"-f"

            amlog_trace $AM_ROHA_RELEASE_BEGIN "DLPAR operation for $am_log" 
            if [[ -n ${roha_session.preferred_nova_list} ]]; then
                out=$(clnovacmd -o release -r dlpar $opt 3>&1)
            else
                out=$($CLHMCCMD_CMD -o release -r dlpar $opt 3>&1)
            fi
            rc=$?
            roha_session_log "$out"
            if (( $rc > 0 )) ; then
                roha_session_update_cluster_manager resource_error DLPAR $PROGNAME
            else
                amlog_trace $AM_ROHA_RELEASE_END "DLPAR operation for $am_log"
                roha_session_write_odm_dynresop DLPAR_MEM   $(max 0 $(( $(roha_session_read_odm_dynresop "DLPAR_MEM")   - mem )))
                roha_session_write_odm_dynresop DLPAR_PROCS $(max 0 $(( $(roha_session_read_odm_dynresop "DLPAR_PROCS") - cpu )))
                roha_session_update_cluster_manager resource_down DLPAR $PROGNAME
            fi

        fi

    fi

    return $rc
} # End of "release_dlpar_resources()"

#=============================================================================
#
# Name:        roha_apply_onoff_resources
#
# Description: Encapsulation of On/Off CoD memory and processor acquisition/release
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_session
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function roha_apply_onoff_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS

    if [[ ${roha_session.operation} == "acquire" ]] ; then

        activate_onoff_resources
        rc=$?

    else # [[ ${roha_session.operation} == "release" ]]

        deactivate_onoff_resources
        rc=$?

    fi

    return $rc
} # End of "roha_apply_onoff_resources()"

#=============================================================================
#
# Name:        activate_onoff_resources
#
# Description: Activate the necessary On/Off CoD resources.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     None
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function activate_onoff_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS
    typeset -F4 mem=${roha_identify.onoff.mem}
    typeset -i  cpu=${roha_identify.onoff.cpu}

    if (( $mem > 0.00 )) ; then
        if (( ${roha_query.onoff.mem.activated} > 0.00 )) ; then
            (( roha_identify.onoff.mem = ${roha_query.onoff.mem.activated} + $mem ))
            roha_identify_onoff_days "roha_identify.onoff.mem" "roha_identify.onoff.mem_days" ${roha_query.onoff.mem.days_avail} ${roha_query.onoff.mem.days_left}
        fi
        if (( ${roha_query.onoff.mem.unreturned} > 0.00 )) ; then
            (( roha_identify.onoff.mem = ${roha_query.onoff.mem.unreturned} + $mem ))
            roha_identify_onoff_days "roha_identify.onoff.mem" "roha_identify.onoff.mem_days" ${roha_query.onoff.mem.days_avail} ${roha_query.onoff.mem.days_left}
        fi
        activate_onoff_mem_resources $mem
        rc=$?
        roha_identify.onoff.mem=$mem
        (( $rc > 0 )) && return $rc
        roha_rollback.onoff.mem=${roha_identify.onoff.mem}
        #===============================================================================================
        : Update and store the true value, acquired as part of ROHA in ODM. 
        : OnOff COD and unreturned memory should be excluded as these are included during identify step.
        : True value that will be updated in ODM, should not be more than real activated OnOff COD memory
        #===============================================================================================
        roha_session_write_odm_dynresop ONOFF_MEM $(min $(( ${roha_query.onoff.mem.activated} + $mem )) \
                                                         $(( $(roha_session_read_odm_dynresop "ONOFF_MEM") + ${roha_identify.onoff.mem} \
                                                         - ${roha_query.onoff.mem.unreturned} )))
    fi

    if (( $cpu > 0 )) ; then
        if (( ${roha_query.onoff.cpu.activated} > 0.00 )) ; then
            (( roha_identify.onoff.cpu = ${roha_query.onoff.cpu.activated} + $cpu ))
            roha_identify_onoff_days "roha_identify.onoff.cpu" "roha_identify.onoff.cpu_days" ${roha_query.onoff.cpu.days_avail} ${roha_query.onoff.cpu.days_left}
        fi
        if (( ${roha_query.onoff.cpu.unreturned} > 0.00 )) ; then
            (( roha_identify.onoff.cpu = ${roha_query.onoff.cpu.unreturned} + $cpu ))
            roha_identify_onoff_days "roha_identify.onoff.cpu" "roha_identify.onoff.cpu_days" ${roha_query.onoff.cpu.days_avail} ${roha_query.onoff.cpu.days_left}
        fi
        activate_onoff_cpu_resources $cpu
        rc=$?
        roha_identify.onoff.cpu=$cpu
        (( $rc > 0 )) && return $rc
        roha_rollback.onoff.cpu=${roha_identify.onoff.cpu}
        #===============================================================================================
        : Update and store the true value, acquired as part of ROHA in ODM. 
        : OnOff COD and unreturned CPUs should be excluded as these are included during identify step.
        : True value that will be updated in ODM, should not be more than real activated OnOff COD CPUs
        #===============================================================================================
        roha_session_write_odm_dynresop ONOFF_CPU $(min $(( ${roha_query.onoff.cpu.activated} + $cpu )) \
                                                         $(( $(roha_session_read_odm_dynresop "ONOFF_CPU") + $cpu \
                                                         - ${roha_query.onoff.cpu.unreturned} )))
    fi

    return $rc
} # End of "activate_onoff_resources()"

#=============================================================================
#
# Name:        deactivate_onoff_resources
#
# Description: De-activate the necessary On/Off CoD resources.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_identify
#              roha_query
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function deactivate_onoff_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS
    typeset -F4 mem=${roha_identify.onoff.mem}
    typeset -i  cpu=${roha_identify.onoff.cpu}

    if (( $mem > 0.00 )) ; then
        if (( ${roha_query.onoff.mem.activated} > $mem )) ; then
            (( roha_identify.onoff.mem = ${roha_query.onoff.mem.activated} - $mem ))
            (( roha_identify.onoff.mem_days = ${roha_query.onoff.mem.days_left} / ${roha_query.onoff.mem.activated} ))
            reactivate_onoff_mem_resources $mem
        else
            deactivate_onoff_mem_resources
        fi
        rc=$?
        roha_identify.onoff.mem=$mem
        (( $rc > 0 )) && return $rc
    fi

    if (( $cpu > 0.00 )) ; then
        if (( ${roha_query.onoff.cpu.activated} > $cpu )) ; then
            (( roha_identify.onoff.cpu = ${roha_query.onoff.cpu.activated} - $cpu ))
            (( roha_identify.onoff.cpu_days = ${roha_query.onoff.cpu.days_left} / ${roha_query.onoff.cpu.activated} ))
            reactivate_onoff_cpu_resources $cpu
        else
            deactivate_onoff_cpu_resources
        fi
        rc=$?
        roha_identify.onoff.cpu=$cpu
        (( $rc > 0 )) && return $rc
    fi

    return $rc
} # End of "deactivate_onoff_resources()"

#=============================================================================
#
# Name:        activate_onoff_mem_resources
#
# Description: Activate the necessary On/Off CoD resources.
#
# Inputs:      [IN]  true_mem - value really released for this node
#
# Outputs:     None
#
# Globals:     roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function activate_onoff_mem_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS
    typeset -F4 mem=${roha_identify.onoff.mem}
    typeset -i  mem_days=${roha_identify.onoff.mem_days}
    typeset -i  true_mem=$1
    typeset opt=""

    if (( $mem > 0.00 )) ; then

        roha_session_update_cluster_manager resource_acquiring ONOFF $PROGNAME

        (( $(roha_session_read_odm_rohaparam "always_start_rg") == 1 )) && opt=${opt:+$opt }"-f"

        out=$($CLHMCCMD_CMD -o acquire -r onoff -m $(( mem * 1024 )) -d $mem_days $opt 3>&1)
        rc=$?
        roha_session_log "$out"
        if (( $rc > 0 )) ; then
            roha_session_update_cluster_manager resource_error ONOFF $PROGNAME
        else
            roha_session_update_cluster_manager resource_up ONOFF $PROGNAME
        fi

    fi

    return $rc
} # End of "activate_onoff_mem_resources()"

#=============================================================================
#
# Name:        activate_onoff_cpu_resources
#
# Description: Activate the necessary On/Off CoD resources.
#
# Inputs:      [IN]  true_cpu - value really released for this node
#
# Outputs:     None
#
# Globals:     roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function activate_onoff_cpu_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS
    typeset -i cpu=${roha_identify.onoff.cpu}
    typeset -i cpu_days=${roha_identify.onoff.cpu_days}
    typeset -i true_cpu=$1
    typeset opt=""

    if (( $cpu > 0.00 )) ; then

        roha_session_update_cluster_manager resource_acquiring ONOFF $PROGNAME

        (( $(roha_session_read_odm_rohaparam "always_start_rg") == 1 )) && opt=${opt:+$opt }"-f"

        out=$($CLHMCCMD_CMD -o acquire -r onoff -q $cpu -d $cpu_days $opt 3>&1)
        rc=$?
        roha_session_log "$out"
        if (( $rc > 0 )) ; then
            roha_session_update_cluster_manager resource_error ONOFF $PROGNAME
        else
            roha_session_update_cluster_manager resource_up ONOFF $PROGNAME
        fi
    fi

    return $rc
} # End of "activate_onoff_cpu_resources()"

#=============================================================================
#
# Name:        reactivate_onoff_mem_resources
#
# Description: Re-activate the necessary On/Off CoD resources.
#
# Inputs:      [IN]  true_mem - value really released for this node
#
# Outputs:     None
#
# Globals:     roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function reactivate_onoff_mem_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS
    typeset -F4 mem=${roha_identify.onoff.mem}
    typeset -i  mem_days=${roha_identify.onoff.mem_days}
    typeset -i  true_mem=$1

    if (( $mem > 0.00 )) ; then

        roha_session_update_cluster_manager resource_releasing ONOFF $PROGNAME

        out=$($CLHMCCMD_CMD -o acquire -r onoff -m $(( mem * 1024 )) -d $mem_days -f 3>&1)
        rc=$?
        roha_session_log "$out"
        if (( $rc > 0 )) ; then
            roha_session_update_cluster_manager resource_error ONOFF $PROGNAME
        else
            #======================================================================================================
            : Remove released OnOff COD memory resources and unreturned OnOff COD memory update in ODM accordingly.
            #======================================================================================================
            typeset -F4 pending_onoff_mem=0.00
            pending_onoff_mem=$(max 0 $(( $(roha_session_read_odm_dynresop "ONOFF_MEM") - $true_mem - ${roha_query.onoff.mem.unreturned} )))
            roha_session_write_odm_dynresop ONOFF_MEM $pending_onoff_mem

            roha_session_update_cluster_manager resource_down ONOFF $PROGNAME
        fi
    fi

    return $rc
} # End of "reactivate_onoff_mem_resources()"

#=============================================================================
#
# Name:        reactivate_onoff_cpu_resources
#
# Description: Re-activate the necessary On/Off CoD resources.
#
# Inputs:      [IN]  true_cpu - value really released for this node
#
# Outputs:     None
#
# Globals:     roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function reactivate_onoff_cpu_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS
    typeset -i cpu=${roha_identify.onoff.cpu}
    typeset -i cpu_days=${roha_identify.onoff.cpu_days}
    typeset -i true_cpu=$1

    if (( $cpu > 0.00 )) ; then

        roha_session_update_cluster_manager resource_releasing ONOFF $PROGNAME

        out=$($CLHMCCMD_CMD -o acquire -r onoff -q $cpu -d $cpu_days -f 3>&1)
        rc=$?
        roha_session_log "$out"
        if (( $rc > 0 )) ; then
            roha_session_update_cluster_manager resource_error ONOFF $PROGNAME
        else
            #=================================================================================================
            : Remove released OnOff COD CPU resources and unreturned OnOff COD CPUs update in ODM accordingly.
            #=================================================================================================
            typeset -F4 pending_onoff_cpu=0.00
            pending_onoff_cpu=$(max 0 $(( $(roha_session_read_odm_dynresop "ONOFF_CPU") - $true_cpu - ${roha_query.onoff.cpu.unreturned} )))
            roha_session_write_odm_dynresop ONOFF_CPU $pending_onoff_cpu

            roha_session_update_cluster_manager resource_down ONOFF $PROGNAME
        fi

    fi

    return $rc
} # End of "reactivate_onoff_cpu_resources()"

#=============================================================================
#
# Name:        deactivate_onoff_mem_resources
#
# Description: De-activate the necessary On/Off CoD resources.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function deactivate_onoff_mem_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS
    typeset -F4 mem=${roha_identify.onoff.mem}

    if (( $mem > 0.00 )) ; then

        roha_session_update_cluster_manager resource_releasing ONOFF $PROGNAME

        out=$($CLHMCCMD_CMD -o release -r onoff -m $(( mem * 1024 )) -f 3>&1)
        rc=$?
        roha_session_log "$out"
        if (( $rc > 0 )) ; then
            roha_session_update_cluster_manager resource_error ONOFF $PROGNAME
        else
            #======================================================================================================
            : Remove released OnOff COD memory resources and unreturned OnOff COD memory update in ODM accordingly.
            #======================================================================================================
            typeset -F4 pending_onoff_mem=0.00
            pending_onoff_mem=$(max 0 $(( $(roha_session_read_odm_dynresop "ONOFF_MEM") - $mem - ${roha_query.onoff.mem.unreturned} )))
            roha_session_write_odm_dynresop ONOFF_MEM $pending_onoff_mem

            roha_session_update_cluster_manager resource_down ONOFF $PROGNAME
        fi

    fi

    return $rc
} # End of "deactivate_onoff_mem_resources()"

#=============================================================================
#
# Name:        deactivate_onoff_cpu_resources
#
# Description: De-activate the necessary On/Off CoD resources.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function deactivate_onoff_cpu_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS
    typeset -i cpu=${roha_identify.onoff.cpu}

    if (( $cpu > 0.00 )) ; then

        roha_session_update_cluster_manager resource_releasing ONOFF $PROGNAME

        out=$($CLHMCCMD_CMD -o release -r onoff -q $cpu -f 3>&1)
        rc=$?
        roha_session_log "$out"
        if (( $rc > 0 )) ; then
            roha_session_update_cluster_manager resource_error ONOFF $PROGNAME
        else
            #=================================================================================================
            : Remove released OnOff COD CPU resources and unreturned OnOff COD CPUs update in ODM accordingly.
            #=================================================================================================
            typeset -F4 pending_onoff_cpu=0.00
            pending_onoff_cpu=$(max 0 $(( $(roha_session_read_odm_dynresop "ONOFF_CPU") - $cpu - ${roha_query.onoff.cpu.unreturned} )))
            roha_session_write_odm_dynresop ONOFF_CPU $pending_onoff_cpu
 
            roha_session_update_cluster_manager resource_down ONOFF $PROGNAME
        fi

    fi

    return $rc
} # End of "deactivate_onoff_cpu_resources()"

#=============================================================================
#
# Name:        roha_apply_yank_codpool_resources
#
# Description: Yank resources from the other node.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_query
#              roha_identify
#
# Returns:     RC_SUCCESS
#              or else, failure return code.
#
#=============================================================================
function roha_apply_yank_codpool_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS
    typeset -i  op_rc=$RC_SUCCESS
    typeset -F4 mem=${roha_identify.codpool.mem_to_yank}
    typeset -F4 cpu=${roha_identify.codpool.cpu_to_yank}
    typeset opt=""
    typeset hmc_list=""

    if (( $mem > 0.00 || $cpu > 0.00 )) ; then

        roha_session_update_cluster_manager resource_releasing YANK_CODPOOL $PROGNAME

        (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $(( mem * 1024 ))"
        (( $cpu > 0.00 )) && opt=${opt:+$opt }"-q $cpu"
        opt=${opt:+$opt }"-f"

        #=============================================================================
        : If HMC are configured at Node or Site level and If CECs are not present in 
        : all HMCs need to pass the correct HMC list for other Node
        #=============================================================================
        other_node=$(cl_dynresop -l ${roha_query.other_node.lpar.name} -N 2>/dev/null)
        op_rc=$?
        if (( $op_rc > 0 )) ; then
            other_node=${roha_query.other_node.lpar.name}
            hmc_list=$(cl_get_hmc_list -n $other_node | tr ' ' ':')
        else
            hmc_list=$(cl_get_hmc_list -n $other_node | tr ' ' ':')
        fi

        out=$($CLHMCCMD_CMD -H $hmc_list -o release -r epcod $opt -L ${roha_query.other_node.lpar.name} 3>&1)
        op_rc=$?
        roha_session_log "$out"
        if (( $op_rc > 0 )) ; then
            roha_session_update_cluster_manager resource_error YANK_CODPOOL $PROGNAME
        else
            roha_session_update_cluster_manager resource_down YANK_CODPOOL $PROGNAME
        fi

        opt=""
    fi


    #=============================================================================
    : Yank force
    : For each MANAGED SYSTEM of roha_query_codpool_members
    :   cumulate yank_compliant with yank_uncompliant and perform EPCoD release force
    #=============================================================================
    typeset SYS_NAME=""
    for SYS_NAME in ${roha_query_codpool_members_names}; do

        if [[ $SYS_NAME == "${roha_query.cec.name}" ]]; then
            continue
        fi

        #=============================================================================
        : Loop for $SYS_NAME Managed System
        #=============================================================================
        (( mem = ${roha_identify.codpool.mem_to_yank_compliant[$SYS_NAME]:-0.00} + ${roha_identify.codpool.mem_to_yank_uncompliant[$SYS_NAME]:-0.00} ))
        (( cpu = ${roha_identify.codpool.cpu_to_yank_compliant[$SYS_NAME]:-0.00} + ${roha_identify.codpool.cpu_to_yank_uncompliant[$SYS_NAME]:-0.00} ))

        if (( $mem > 0.00 || $cpu > 0.00 )) ; then
            roha_session_update_cluster_manager resource_releasing YANK_CODPOOL $PROGNAME

            (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $(( mem * 1024 ))"
            (( $cpu > 0.00 )) && opt=${opt:+$opt }"-q $cpu"
            opt=${opt:+$opt }"-f"

            out=$($CLHMCCMD_CMD -o release -r epcod $opt -M ${SYS_NAME} 3>&1)
            op_rc=$?
            roha_session_log "$out"
            if (( $op_rc > 0 )) ; then
                roha_session_update_cluster_manager resource_error YANK_CODPOOL $PROGNAME
            else
                roha_session_update_cluster_manager resource_down YANK_CODPOOL $PROGNAME
            fi

            opt=""
        fi
    done


    #=============================================================================
    # yank return code is always successfull
    # - because we do not want to rollback or undo undo whas remotely acquired.
    # - because if it fails, the acquisition of epcod which follows will fail
    #  and will trigger the reassessment.
    #=============================================================================
    return $rc
} # End of "roha_apply_yank_codpool_resources()"

#=============================================================================
#
# Name:        roha_apply_codpool_resources
#
# Description: Encapsulation of Enterprise Pool CoD memory and processor
#              acquisition/release
#
# Inputs:      $1 to indicate if we are in last loop of reassessment.
#              If set to 1, we are in last loop of reassessment.
#              If set to 0, we are in normal reassessment.
#
# Outputs:     None
#
# Globals:     roha_session
#
# Returns:     RC_SUCCESS
#              or else, failure return code.
#
#=============================================================================
function roha_apply_codpool_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS

    typeset -i LAST_LOOP_OF_FORCE_REASSESSMENT=$1

    #=================================================================
    : LAST_LOOP_OF_FORCE_REASSESSMENT=$LAST_LOOP_OF_FORCE_REASSESSMENT
    #=================================================================

    if [[ ${roha_session.operation} == "acquire" ]] ; then

        allocate_codpool_resources $LAST_LOOP_OF_FORCE_REASSESSMENT
        rc=$?

    else # [[ ${roha_session.operation} == "release" ]]

        deallocate_codpool_resources
        rc=$?

    fi

    return $rc
} # End of "roha_apply_codpool_resources()"

#=============================================================================
#
# Name:        allocate_codpool_resources
#
# Description: Allocate the necessary Mobile CoD resources from the Enterprise
#              Pool to the system.
#
# Inputs:      $1 to indicate if we are in last loop of reassessment.
#              If set to 1, we are in last loop of reassessment.
#              If set to 0, we are in normal reassessment.
#
# Outputs:     None
#
# Globals:     roha_query
#              roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function allocate_codpool_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS
    typeset -F4 mem=$(( roha_identify.codpool.mem + roha_identify.codpool.mem_to_yank ))
    typeset -F4 cpu=$(( roha_identify.codpool.cpu + roha_identify.codpool.cpu_to_yank ))
    typeset     opt=""

    typeset -i LAST_LOOP_OF_FORCE_REASSESSMENT=$1

    #=================================================================
    : LAST_LOOP_OF_FORCE_REASSESSMENT=$LAST_LOOP_OF_FORCE_REASSESSMENT
    #=================================================================

    if (( $mem > 0.00 || $cpu > 0.00 )) ; then
        if (( $(roha_session_read_odm_rohaparam "always_start_rg") == 0 )) ||
               (( $(roha_session_read_odm_rohaparam "always_start_rg") == 1 &&
                       $LAST_LOOP_OF_FORCE_REASSESSMENT == 0 )) ; then
            #=========================================================================
            : always_start_rg=0   or   always_start_rg=1 but not last loop of reassessment
            #=========================================================================
            roha_session_update_cluster_manager resource_acquiring CODPOOL $PROGNAME

            (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $(( mem * 1024 ))"
            (( $cpu > 0.00 )) && opt=${opt:+$opt }"-q $cpu "

            out=$($CLHMCCMD_CMD -o acquire -r epcod $opt 3>&1)
            rc=$?
            roha_session_log "$out"
            if (( $rc > 0 )) ; then
                roha_session_update_cluster_manager resource_error CODPOOL $PROGNAME
            else
                roha_rollback.codpool.mem=$mem
                roha_rollback.codpool.cpu=$cpu
                #===================================================================
                : Update and store the true value, acquired as part of ROHA in ODM. 
                : Unreturned resource value should be excluded. Value should not be 
                :  more than system mobile COD resources
                #===================================================================
                roha_session_write_odm_dynresop CODPOOL_MEM $(min $(( ${roha_query.codpool.mem.mobile} + $mem )) $(( $(roha_session_read_odm_dynresop "CODPOOL_MEM") + $mem - ${roha_query.codpool.mem.unreturned} )))
                roha_session_write_odm_dynresop CODPOOL_CPU $(min $(( ${roha_query.codpool.cpu.mobile} + $cpu )) $(( $(roha_session_read_odm_dynresop "CODPOOL_CPU") + $cpu - ${roha_query.codpool.cpu.unreturned} )))
                roha_session_update_cluster_manager resource_up CODPOOL $PROGNAME
            fi
        else
            #=========================================================================
            : always_start_rg=1 and last loop of reassessment
            #=========================================================================
            typeset -i op_rc=$RC_UNKNOWN

            #=========================================================================
            : mem step
            #=========================================================================
            (( $mem > 0.00 )) && opt="-m $(( mem * 1024 )) -f"

            out=$($CLHMCCMD_CMD -o acquire -r epcod $opt 3>&1)
            op_rc=$?
            roha_session_log "$out"
            if (( $op_rc > 0 )) ; then
                rc=$op_rc
            else
                roha_rollback.codpool.mem=$mem
                #===================================================================
                : Update and store the true value, acquired as part of ROHA in ODM. 
                : Unreturned resource value should be excluded. Value should not be 
                :  more than system mobile COD resources
                #===================================================================
                roha_session_write_odm_dynresop CODPOOL_MEM $(min $(( ${roha_query.codpool.mem.mobile} + $mem )) $(( $(roha_session_read_odm_dynresop "CODPOOL_MEM") + $mem - ${roha_query.codpool.mem.unreturned} )))
            fi

            #=========================================================================
            : As we are in always_start_rg=1 and last loop of reassessment,
            :  whatever the result we continue.
            #=========================================================================
            opt=""
            #=========================================================================
            : procs step
            #=========================================================================
            (( $cpu > 0.00 )) && opt="-q $cpu -f"

            out=$($CLHMCCMD_CMD -o acquire -r epcod $opt 3>&1)
            op_rc=$?
            roha_session_log "$out"
            if (( $op_rc > 0 )) ; then
                rc=$op_rc
            else
                roha_rollback.codpool.cpu=$cpu
                #===================================================================
                : Update and store the true value, acquired as part of ROHA in ODM. 
                : Unreturned resource value should be excluded. Value should not be 
                :  more than system mobile COD resources
                #===================================================================
                roha_session_write_odm_dynresop CODPOOL_CPU $(min $(( ${roha_query.codpool.cpu.mobile} + $cpu )) $(( $(roha_session_read_odm_dynresop "CODPOOL_CPU") + $cpu - ${roha_query.codpool.cpu.unreturned} )))
            fi


            if (( $rc > 0 )); then
                #=========================================================================
                : rc is forced to SUCCESS anyway, as we are in always_start_rg=1 and
                :  last loop of reassessment
                #=========================================================================
                rc=$RC_SUCCESS

                #=========================================================================
                : both steps mem and procs succeeded or are considered as
                :  succeeded even if failed
                #=========================================================================
                roha_session_update_cluster_manager resource_up CODPOOL $PROGNAME
            else
                #=========================================================================
                : both steps mem and procs succeeded
                #=========================================================================
                roha_session_update_cluster_manager resource_up CODPOOL $PROGNAME
            fi

        fi
    fi

    return $rc
} # End of "allocate_codpool_resources()"

#=============================================================================
#
# Name:        deallocate_codpool_resources
#
# Description: Return Mobile CoD resources back to the Enterprise Pool CoD.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_query
#              roha_identify
#
# Returns:     RC_SUCCESS
#              or else, clhmccmd error return code.
#
#=============================================================================
function deallocate_codpool_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i  rc=$RC_SUCCESS
    typeset -F4 mem=${roha_identify.codpool.mem}
    typeset -F4 cpu=${roha_identify.codpool.cpu}
    typeset -F4 pending_codpool_mem=0.00
    typeset -F4 pending_codpool_cpu=0.00
    typeset -F4 odm_codpool_mem=0.00
    typeset -F4 odm_codpool_cpu=0.00
    typeset opt=""


    if (( ${roha_session.synchronous} == 1 )) ; then
        typeset -F4 freepool_mem_available=0.00
        typeset -F4 freepool_cpu_available=0.00
        typeset out_str=""
        typeset -F4 round_up=0.00
        typeset -F4 round_down=0.00

        typeset attrs="curr_avail_sys_mem:curr_avail_sys_proc_units"
        #============================================================================================
        : Release EPCoD Resources if they are available only to avoid creating unreturned
        #============================================================================================

        if [[ -n ${roha_session.preferred_nova_list} ]]; then
            out_str=$(clnovacmd -o query -a $attrs 3>&1)
        else
            out_str=$($CLHMCCMD_CMD -o query -a $attrs 3>&1)
        fi
        rc=$?
        if (( $rc > 0 )) ; then
            roha_session_update_cluster_manager resource_error CODPOOL $PROGNAME
            return $rc
        fi
        print "$out_str" | IFS=: read freepool_mem_available freepool_cpu_available

        mem=$(min $freepool_mem_available $mem)
        cpu=$(min $freepool_cpu_available $cpu)

        round_up=$(print ${cpu} | awk '{printf "%0.4f\n",int($0+0.9999)}')
        round_down=$(print ${cpu} | awk '{printf "%0.4f\n",int($0)}')
        if (( $cpu >= $round_up )) ; then
            cpu=$round_up
        else
            cpu=$round_down
        fi

        round_up=$(print ${mem} | awk '{printf "%0.4f\n",int($0+0.9999)}')
        round_down=$(print ${mem} | awk '{printf "%0.4f\n",int($0)}')
        if (( $mem >= $round_up )) ; then
            mem=$round_up
        else
            mem=$round_down
        fi
    fi

    if (( $mem > 0.00 || $cpu > 0.00 )) ; then

        roha_session_update_cluster_manager resource_releasing CODPOOL $PROGNAME

        (( $mem > 0.00 )) && opt=${opt:+$opt }"-m $(( mem * 1024 ))"
        (( $cpu > 0.00 )) && opt=${opt:+$opt }"-q $cpu"
        opt=${opt:+$opt }"-f"

        out=$($CLHMCCMD_CMD -o release -r epcod $opt 3>&1)
        rc=$?
        roha_session_log "$out"
        if (( $rc > 0 )) ; then
            roha_session_update_cluster_manager resource_error CODPOOL $PROGNAME
        else
            #=========================================================================================
            : Remove released EPCOD resources and unreturned resources then update in ODM accordingly.
            #=========================================================================================

            odm_codpool_mem=$(roha_session_read_odm_dynresop "CODPOOL_MEM")
            pending_codpool_mem=$(max 0 $(( $odm_codpool_mem - $mem - ${roha_query.codpool.mem.unreturned} )))

            roha_session_write_odm_dynresop CODPOOL_MEM $pending_codpool_mem

            odm_codpool_cpu=$(roha_session_read_odm_dynresop "CODPOOL_CPU")
            pending_codpool_cpu=$(max 0 $(( $odm_codpool_cpu - $cpu - ${roha_query.codpool.cpu.unreturned} )))

            roha_session_write_odm_dynresop CODPOOL_CPU $pending_codpool_cpu
            
            roha_session_update_cluster_manager resource_down CODPOOL $PROGNAME
        fi

    fi

    return $rc
} # End of "deallocate_codpool_resources()"

#=============================================================================
#
# Name:        roha_apply_rollback_all_resources
#
# Description: Reset resources to initial state only if 'always_start_rg'
#              tunable is not set because we want applications to start.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_identify
#
# Returns:     RC_SUCCESS (always succeeds)
#
#=============================================================================
function roha_apply_rollback_all_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS

    [[ $(roha_session_read_odm_rohaparam "always_start_rg") != 0 ]] \
        && return $rc

    roha_session_log "INFO: Start rollback of all resources."

    #=======================================================
    : Rollback all that can be rolled back. Do not return on
    : failure
    #=======================================================

    #=======================================================
    : Rollback DLPAR.
    # Do not force roha_session.synchronous to true as
    # acquisition is always synchronous
    #=======================================================
    if (( ${roha_rollback.dlpar.mem} > 0.00 || ${roha_rollback.dlpar.cpu} > 0.00 \
        || ${roha_rollback.dlpar.pu} > 0.00 || ${roha_rollback.dlpar.vp}  > 0.00 )) ; then
        roha_identify.dlpar.mem=${roha_rollback.dlpar.mem}
        roha_identify.dlpar.cpu=${roha_rollback.dlpar.cpu}
        roha_identify.dlpar.pu=${roha_rollback.dlpar.pu}
        roha_identify.dlpar.vp=${roha_rollback.dlpar.vp}
        release_dlpar_resources
        #=======================================================
        : If RC is already RC_FAILURE, do no allow to go back to
        : RC_SUCCESS and keep RC_FAILURE
        #=======================================================
        (( $rc != $RC_FAILURE )) && rc=$?
    fi

  if (( CONN_TYPE != 2 )); then
    #=======================================================
    : Rollback On/off CoD.
    #=======================================================
    if (( ${roha_rollback.onoff.mem} > 0.00 || ${roha_rollback.onoff.cpu} > 0.00 )) ; then
        roha_query_onoff nosave
        roha_identify.onoff.mem=${roha_rollback.onoff.mem}
        roha_identify.onoff.cpu=${roha_rollback.onoff.cpu}
        deactivate_onoff_resources
        #=======================================================
        : If RC is already RC_FAILURE, do no allow to go back to
        : RC_SUCCESS and keep RC_FAILURE
        #=======================================================
        (( $rc != $RC_FAILURE )) && rc=$?
    fi

    #=======================================================
    : Rollback Enterprise Pool CoD.
    #=======================================================
    if (( ${roha_rollback.codpool.mem} > 0.00 || ${roha_rollback.codpool.cpu} > 0.00 )) ; then
        roha_identify.codpool.mem=${roha_rollback.codpool.mem}
        roha_identify.codpool.cpu=${roha_rollback.codpool.cpu}
        deallocate_codpool_resources
        #=======================================================
        : If RC is already RC_FAILURE, do no allow to go back to
        : RC_SUCCESS and keep RC_FAILURE
        #=======================================================
        (( $rc != $RC_FAILURE )) && rc=$?
    fi
  fi

    roha_session_log "INFO: End rollback of all resources."

    return $rc
} # End of "roha_apply_rollback_all_resources()"


#=============================================================================
#
# Name:        roha_apply_cloud_operations
#
# Description: Performs acquire/release operation. Makes REST API calls to
#              cloud VM and sets the provided resources.
#
# Inputs:      None
#
# Outputs:     None
#
# Globals:     roha_compute
#
# Returns:     RC_SUCCESS, RC_QUERY_ERROR
#
#=============================================================================
function roha_apply_cloud_operations
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i rc=$RC_SUCCESS
    typeset action=$1
    shift;

    typeset -i timer=0
    typeset -i timeout=0
    typeset -i interval=60     #seconds
    typeset -i def_timeout=5   #default timeout in minutes

    timeout=$($CLODMGET_CMD -q "name=$LOCALNODENAME and object=POWERVS_TIMEOUT" -nf value HACMPnode 2>/dev/null)
    if [[ -z $timeout ]] || (( $timeout <= 0 )); then
        timeout=$def_timeout
    fi
    (( timeout *= 60 )) # Convert to seconds

    while (( $timer < $timeout )); do

        out_str=$($CLHMCCMD_CMD -o query -a status:health)
        (( $? > 0 )) && return $RC_QUERY_ERROR
        print "$out_str" | grep -v "#" | IFS=: read vmstatus vmhealth

        if [[ $vmstatus == "ACTIVE" && $vmhealth == "OK" ]]; then
            # now make clcloudroha call; CLHMCCMD_CMD=clcloudroha
            out=$($CLHMCCMD_CMD -o $action $* 3>&1)
            rc=$?
            roha_session_log "$out"
            break
        else
            sleep $interval
            (( timer += interval ))
            roha_session_log "Waiting on Cloud VM health status check, $timer seconds elapsed."
        fi

    done

    if (( $timer == $timeout )); then
        roha_session_log "Cloud operations timeout exhausted. Could not $action the required resources.\n"
    fi

    return $rc
} # End of "roha_apply_cloud_operations()"