#!/bin/ksh93
#  ALTRAN_PROLOG_BEGIN_TAG
#  This is an automatically generated prolog.
#
#  Copyright (C) Altran ACT S.A.S. 2017,2018,2019,2021.  All rights reserved.
#
#  ALTRAN_PROLOG_END_TAG
#
# @(#)  d7d31f1 43haes/usr/sbin/cluster/events/utils/clasyncroha.sh, 726, 2147A_aha726, Sep 09 2021 09:43 AM

#########################################################################
#
#   COMPONENT_NAME: hacmp.events
#
#
#########################################################################

#================================================
# The following, commented line enforces coding
# standards when this file is edited via vim.
#================================================
# vim:tabstop=4:shiftwidth=4:expandtab:smarttab
#================================================

#=============================================================================
#
# NAME:        clasyncroha
#
# DESCRIPTION: This script handles Asynchronous release of resources for 
#       Resource Optimized High Availability (ROHA) functionality for the
#       account of PowerHA SystemMirror.
#
# ARGUMENTS: None
#
# OUTPUT:    None
#
# RETURN VALUE DESCRIPTION:
#       RC_SUCCESS
#
#=============================================================================

#=============================================================================
#
# Name:        roha_process_resources
#
# Description: Convenient encapsulation of the resources (cpu and memory)
#   acquisition and release process.
#
# Inputs:      None
#
# Outputs:     None
#
# Returns:     RC_SUCCESS
#              or else an error code
#
#=============================================================================
function roha_process_resources
{
    [[ $VERBOSE_LOGGING == high ]] && set -x

    typeset -i FORCE_REASSESSMENT=0
    typeset -F4 expected_cur_mem=0.00
    typeset -i  expected_cur_procs=0    
    typeset -F4 expected_cur_proc_units=0.00
    typeset hmc_list=
    typeset nova_list=

    nova_list=$(cl_get_nova_list 2>/dev/null)
    if (( $? == 0 )) && [[ -n $nova_list ]]; then
        roha_session.preferred_nova_list=$nova_list
    fi
    
    hmc_list=$(cl_get_hmc_list 2>/dev/null)
    if (( $? == 0 )) && [[ -n $hmc_list ]]; then
        roha_session.preferred_hmc_list=$hmc_list
    fi

    #=======================================================
    # Query step
    #=======================================================
    roha_query_partition
    rc=$?
    
    roha_query_partition_report
    (( $rc > 0 )) && return $RC_QUERY_ERROR

    roha_query_provisioning "roha_query.optimal" "${roha_session.optimal_apps}"
    rc=$?
    
    roha_query_optimal_report
    (( $rc > 0 )) && return $RC_QUERY_ERROR

    roha_query_provisioning "roha_query.running" "${roha_session.running_apps}"
    rc=$?
    
    roha_query_running_report
    (( $rc > 0 )) && return $RC_QUERY_ERROR

    #=================================================================
    : Reset to 0 all variables used in clcomputeroha
    #=================================================================
    initialize_compute_globals

    #=======================================================
    # Compute step
    #=======================================================
    cl_compute_roha
    rc=$?
    (( $rc > 0 )) && return $rc

    #=======================================================
    # If no needed resources, return on success
    #=======================================================
    roha_compute_check_needed_resources
    (( $? > 0 )) && return $RC_SUCCESS

    #=======================================================
    # Write future LPAR resource current values in ODM
    #=======================================================    
    if (( ${roha_compute.mem} > 0.00 )) ; then
        (( expected_cur_mem = ${roha_query.lpar.mem.curr} - ${roha_compute.mem} ))    
        roha_session_write_odm_dynresop ACTUAL_LPAR_CUR_MEM     "$expected_cur_mem"
    fi
    
    if [[ ${roha_query.lpar.cpu.proc_mode} == "shared" ]] ; then
        if (( ${roha_compute.pu} > 0.00 )) ; then
            (( expected_cur_proc_units = ${roha_query.lpar.cpu.proc_units.curr} - ${roha_compute.pu} ))
            roha_session_write_odm_dynresop ACTUAL_LPAR_CUR_PROC_UNITS     "$expected_cur_proc_units"
        fi
        if (( ${roha_compute.vp} > 0 )) ; then
            (( expected_cur_procs = ${roha_query.lpar.cpu.procs.curr} - ${roha_compute.vp} ))
            roha_session_write_odm_dynresop ACTUAL_LPAR_CUR_PROCS     "$expected_cur_procs"
        fi
    else # [[ ${roha_query.lpar.cpu.proc_mode} == "ded" ]]
        if (( ${roha_compute.cpu} > 0 )) ; then
            (( expected_cur_procs = ${roha_query.lpar.cpu.proc_units.curr} - ${roha_compute.cpu} ))    
            roha_session_write_odm_dynresop ACTUAL_LPAR_CUR_PROCS     "$expected_cur_procs"
        fi
    fi 

    roha_query_managed_system
    rc=$?

    roha_query_managed_system_report
    (( $rc > 0 )) && return $RC_QUERY_ERROR

    if [[ -n ${roha_session.preferred_hmc_list} ]]; then
        roha_query_onoff
        rc=$?
        
        roha_query_onoff_report
        (( $rc > 0 )) && return $RC_QUERY_ERROR
    fi

    #=================================================================
    : Reset to 0 all variables used in clidentifyroha
    #=================================================================
    initialize_identify_globals

    #=================================================================
    : Reset to 0 all variables used in clapplyroha
    #=================================================================
    initialize_apply_globals

    #=======================================================
    # Identify step
    #=======================================================
    cl_identify_roha
    rc=$?
    (( $rc > 0 )) && return $rc

    #================================================================
    # Apply step, reassessment will not be done for release operation 
    #================================================================
    cl_apply_roha $FORCE_REASSESSMENT

    rc=$?
    (( $rc > 0 )) && return $rc
    
    return $RC_SUCCESS
} # End of "roha_process_resources()"

#=============================================================================
#
# Name:        roha_process_operation
#
# Description: Convenient encapsulation of the resources (cpu and memory)
#              release process.
#
# Inputs:      $1 to indicate if we are in last loop of reassessment.
#              If set to 0, we are in normal reassessment.
#
# Outputs:     None
#
# Returns:     RC_SUCCESS
#              or else an error code
#
#=============================================================================
function roha_process_operation
{
    roha_process_resources
    rc=$?
    if (( rc == 0 )); then
        roha_session_log "INFO: received rc=$rc."
        roha_session_log "Success on $CURRENT_REASSESSMENT attempt(s)."
    else
        roha_session_log "INFO: received error code $rc."
    fi


    #========================================================
    # Release failures will be transformed into a SUCCESS.
    #========================================================
    if (( rc > 0 )); then
        roha_session_log "A failure in release should not prevent takeover from occuring."
        #==============================================================
        : Release : transform the failure RC into a success RC so that
        :  takeover can normally continue.
        #==============================================================
        rc=$RC_SUCCESS
    fi
    return $rc
} # End of "roha_process_operation()"


###############################################################################
## MAIN
###############################################################################

#====================
# Export environment
#====================
PROGNAME=${0##*/}
. /usr/es/sbin/cluster/events/utils/clcommonroha
[[ $VERBOSE_LOGGING == high ]] && set -x


#=======================================================
# Set the ROHA operation 
#=======================================================
roha_session.operation="release"
#=========================================================================
# Force the synchronous release, as we are already in asynchronous release
#=========================================================================
roha_session.synchronous=1

# Read the ROHA environment variables during
# async operation from /etc/environment
#=======================================================
read_tunables
#=======================================================

#=====================================================================================================
# When first alias is enabled for serviceIP, there is a chance of hanging ssh command since,
# when ssh command is invoked and at the same time service IP is released,
# during this time ssh is confused and it will hang permanently.
# To avoid this, wait for serviceIP to be released if first alias is enabled. We will wait for 30mins. 
#=====================================================================================================
typeset -i sip_timeout=1800
typeset rg_list=""
typeset serviceIPs=""
typeset ip_address=""
apps=$(ODMDIR=/etc/es/objrepos clodmget -q "key=WAIT_APPS" -nf value HACMPdynresop 2>/dev/null)
for app in $apps
do
    #=========================================================
    # Get Resource Group list which need to be released
    #=========================================================
    rg=$(clodmget -n -f group -q value=$app HACMPresource)
    rg_list=${rg_list:+${rg_list} }$rg
done

rg_list=$(echo $rg_list| sort -u)

for rg in $rg_list
do
    #==========================================================
    # Get the service ips of all the Resource Groups
    #==========================================================
    service_ip=$(clodmget -q "name=SERVICE_LABEL AND group=$rg" -nf value HACMPresource)
    serviceIPs=${serviceIPs:+${serviceIPs} }$service_ip
done

for sip in $serviceIPs
do
    #===========================================================
    # Identify the service ip which first aliase is configured 
    #===========================================================
    max_aliases=$(clodmget -q "ip_label=$sip" -nf max_aliases HACMPadapter)
    if (( $max_aliases != 1795 )); then
        #============================================
        # Convert the service ip label into real IP 
        #============================================
        ip_addr=$(clodmget -q "ip_label=$sip" -nf identifier HACMPadapter)
        ip_address=${ip_address:+${ip_address} }$ip_addr
    fi
done

while (( $sip_timeout > 0 )); do
    released=0
    for ip_addr in $ip_address
    do
        #==========================================
        # Verify the IP / service ip is released 
        #==========================================
        if [[ -n $(ifconfig -a | grep -w $ip_addr) ]]; then
            released=1
        fi
    done

    if (( $released == 0 )); then
        echo "All Service ips are released, breaking from the loop."
        break
    else
        #===============================================================================
        #  All service ips are not released yet, Waiting for all service ips to release 
        #===============================================================================
        sleep 5 
        (( sip_timeout -= 5 ))
    fi
done

#=======================================
# Wait for maximum timeout of 30Minutes
#=======================================
typeset -i timeOut=1800
while (( $timeOut > 0 )); do
    roha_session.optimal_apps=$(ODMDIR=/etc/es/objrepos clodmget -q "key=WAIT_APPS" -nf value HACMPdynresop 2>/dev/null)
    #====================================================================
    # Check if clmanageroha script is running and Async release is needed
    #====================================================================
    ps -eo 'args' | grep -vw grep |grep -qw "clmanageroha"
    if (( $? == 0 )) && [[ -n ${roha_session.optimal_apps} ]]; then
        sleep 2
        (( timeOut -= 2 ))
        continue
    fi

    if [[ -n ${roha_session.optimal_apps} ]] ; then
        roha_session_log "Processing the Aysnchronous release of resources"
        #=======================================================
        # Determine current running applications
        #=======================================================
        roha_session_write_odm_dynresop REL_APPS     "${roha_session.optimal_apps}"
        roha_session_write_odm_dynresop WAIT_APPS    ""
        roha_session_set_running_apps
        #========================================================
        # ROHA process of Asynchronous release of resources
        #========================================================
        roha_process_operation
        rc=$?
        roha_session_write_odm_dynresop REL_APPS     ""
        
        roha_session_write_odm_dynresop ACTUAL_LPAR_CUR_MEM     ""
        roha_session_write_odm_dynresop ACTUAL_LPAR_CUR_PROCS     ""
        if [[ ${roha_query.lpar.cpu.proc_mode} == "shared" ]] ; then
            roha_session_write_odm_dynresop ACTUAL_LPAR_CUR_PROC_UNITS     ""
        fi        

    else
        #=================================================================
        # Application list is empty, exit from script
        #=================================================================
        rc=RC_SUCCESS
        break
    fi
done

if (( $timeOut == 0 )); then
    rc=RC_FAILURE
fi
#=======================================================
# Close the session
#=======================================================
roha_session_close

exit $rc