#!/bin/ksh93
#  ALTRAN_PROLOG_BEGIN_TAG                                                    
#  This is an automatically generated prolog.                                  
#                                                                              
#  Copyright (C) Altran ACT S.A.S. 2017,2018,2019,2020,2021.  All rights reserved.  
#                                                                              
#  ALTRAN_PROLOG_END_TAG                                                      
#                                                                              
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r721 src/43haes/lib/ksh93/hacmp/KLIB_HACMP_get_node_attributes.sh 1.33 
#  
# Licensed Materials - Property of IBM 
#  
# Restricted Materials of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2006,2016 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)  7d4c34b 43haes/lib/ksh93/hacmp/KLIB_HACMP_get_node_attributes.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM

#============================================================================
#
# Name:        KLIB_HACMP_get_node_attributes
#
# Description: This is the main, FPATH function that is invoked by clmgr
#              to collect node information. Each node's information is
#              collected in parallel via backgrounded processes, in an
#              attempt to improve overall performance.
#
# Inputs:      See the "devDoc()" function at the bottom of this file.
#
# Outputs:     The properties hash is populated. The only other outputs are
#              any error messages that might be needed.
#
# Returns:     Zero if no errors are detected. Otherwise, an appropriate
#              non-zero value is returned. Refer to the "RETURN" section
#              of the "devDoc()" function at the bottom of this file for
#              the standard return code values/meanings for clmgr.
#
#============================================================================
function KLIB_HACMP_get_node_attributes {
    . $HALIBROOT/log_entry "$0()" "$CL"
    : version="@(#)  7d4c34b 43haes/lib/ksh93/hacmp/KLIB_HACMP_get_node_attributes.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM"
    : INPUTS: $*

    typeset node=${1//\"/}

    # Get the "properties" associative array reference, and initialize it
    if [[ -z $2 ]] || [[ -n $CLMGR_LOGGING && $2 == "|" ]]; then
        node=
        typeset -n properties=$1
    else
        typeset -n properties=$2
    fi

    [[ $CLMGR_LOGGING == 'med' ]] && set +x  # Only trace param values

    #===================================
    : Declare and initialize variables
    #===================================
    for k in ${!properties[*]}; do unset properties[$k]; done
    typeset -A queries
    typeset -i INDEX=1 rc=$RC_UNKNOWN matches=0 i=0
    typeset NODES= value= NODEID= UNUSED= HOSTNAME= HOST_LABEL=
    typeset -u uc_node=$node attr=
    typeset DATAFILE="$TMPDIR/clmgr.KHgna.attrs.$$"
    typeset GNA_CHILDREN=""
    typeset GNA_RESULTS_FILE="$TMPDIR/clmgr.KHgna.rcs.$$"

    [[ $CLMGR_TIMEOUT != +([[:digit:]]) ]] && CLMGR_TIMEOUT=600
    (( $CLMGR_TIMEOUT < 60 )) && CLMGR_TIMEOUT=60

    #==========================================================
    : Check for any specified attribute=value pairs. If any
    : are found, use those to query/search/filter the output.
    #==========================================================
    while (( $# > 0 )); do
        value="$1"
        if [[ $value == *=* ]]; then
            attr=${value%%=*}  # "attr" makes this uppercase
            queries[$attr]="${value#*=}"
        fi
        shift
    done

    [[ $uc_node == "LOCAL" ]] && node=$LOCAL_NODE

    #================================================================
    : Check for a defined cluster. No need to continue without one.
    #================================================================
    CL=$LINENO isClusterDefined
    if (( $? != 0 )); then
        rc=$RC_MISSING_DEPENDENCY

    #=============================================
    : If any nodes were specified, validate them
    #=============================================
    elif [[ -n $node ]]; then

        #=====================================================
        : For convenience, support wildcards in the provided
        : name list. This is equivalent to using wildcards
        : filtering via something like NAME=some_search.
        #=====================================================
        if [[ $node == *[\*\?]* ]]; then
            typeset object= newList=
            for object in ${node//,/ }; do
                if [[ $object == *[\*\?]* ]]; then
                    unset available
                    typeset available
                    CL=$LINENO KLIB_HACMP_list_nodes available ${queries[TYPE]} NAME=$object
                    for (( i=0; i<${#available[*]}; i++ )); do
                        if [[ ",$newList," != *,${available[$i]},* ]]; then
                            [[ -n $newList ]] && newList="$newList,"
                            newList="$newList${available[$i]}"
                        fi
                    done
                elif [[ ",$newList," != *,$object,* ]]; then
                    [[ -n $newList ]] && newList="$newList,"
                    newList="$newList$object"
                fi
            done
            [[ -n $newList ]] && node=$newList
        fi

        for node in ${node//,/ }; do
            node=$(CL=$LINENO KLIB_HACMP_is_known_node "$node" 2>>$CLMGR_TMPLOG)
            if (( $? != RC_SUCCESS )); then
                if [[ $CLMGR_PROGNAME == "clmgr" ]]; then
                    dspmsg -s $CLMGR_SET $CLMGR_MSGS 102 '\nERROR: "%1$s" does not appear to exist!\n\n' "$node" 1>&2
                    dspmsg -s $CLMGR_SET $CLMGR_MSGS 151 "Available Nodes:\n\n" 1>&2

                    typeset available
                    CL=$LINENO KLIB_HACMP_list_nodes available
                    for (( i=0; i<${#available[*]}; i++ )); do
                        if [[ ${available[$i]} != *([[:space:]]) ]]; then
                            print -u2 "\t${available[$i]}"
                        fi
                    done
                    print -u2 ""
                fi
                rc=$RC_NOT_FOUND
            else
                NODES="$NODES $node"
            fi
        done

    #=======================================================================
    : If a verbose query is being done, or if any searches were specified,
    : then retrieve the full list of known, available nodes.
    #=======================================================================
    elif (( CLMGR_VERBOSE || ${#queries[*]} > 0 )); then
        print "$0()[$LINENO]($SECONDS): clnodename -c" >>$CLMGR_TMPLOG
        NODES=$(clnodename -c)
        NODES=${NODES//:/ }
        print "$0()[$LINENO]($SECONDS): clnodename RC: $?" >>$CLMGR_TMPLOG

    else
        if [[ $CLMGR_PROGNAME == "clmgr" ]]; then
            NODES=$LOCAL_NODE  # Default to the local node
        else
            rc=$RC_MISSING_INPUT
        fi
    fi
    (( $rc != RC_UNKNOWN )) && unset NODES

    #========================================================================
    : Collecting OS levels is slow. So those retrievals are launched first
    : as backgrounded processes, so they can run while the rest of the host
    : data is being collected. 35% performance improvement.
    #========================================================================
    typeset OSLEVEL_PIDS=""
    typeset OSLEVEL_DATA="$TMPDIR/clmgr.KHgna.oslevel.$$"
    typeset -A NODE_IDS
    for node in $NODES; do
        #=====================================================================
        : Retrieve the ID number of node "$node". Note, however, that if the
        : cluster has never been synchronized with this node in it, then the
        : ID will likely be zero. In that case, a faux number will have to
        : be temporarily assigned to avoid having overwritten data.
        #=====================================================================
        print "$0()[$LINENO]($SECONDS): clodmget -q \"name=$node and object=COMMUNICATION_PATH\" -f node_id HACMPnode" >>$CLMGR_TMPLOG  # Always log commands
        NODEID=$(clodmget -q "name=$node and object=COMMUNICATION_PATH" -f node_id HACMPnode)
        print "$0()[$LINENO]($SECONDS): clodmget RC: $?; NODEID == \"$NODEID\"" >>$CLMGR_TMPLOG  # Always log command result
        if [[ -z $NODEID ]] || (( NODEID == 0 )); then
            NODEID=$(print -- "$node" | sum)
            NODEID=${NODEID%%+([[:space:]])*}
            if [[ $NODEID != +([[:digit:]]) ]]; then
                NODEID=$RANDOM
            fi

            #==================================================
            : Make sure we do not end up with a leading zero.
            : That will break the search function, which uses
            : this node ID as a hash key.
            #==================================================
            [[ $NODEID == 0* ]] && NODEID="1$NODEID"
        fi
        NODE_IDS[$node]=$NODEID

        if [[ ${node%%\.*} == ${LOCAL_NODE%%\.*} ]]; then
            (
                print "properties[AIX_LEVEL$NODEID]=\"$(oslevel -s)\"" >>$OSLEVEL_DATA 2>>$CLMGR_TMPLOG
            )&
        else
            (
                print "properties[AIX_LEVEL$NODEID]=\"$($CLRSH $node /usr/bin/oslevel -s 2>>$CLMGR_TMPLOG)\"" >>$OSLEVEL_DATA 2>>$CLMGR_TMPLOG
            )&
        fi
        OSLEVEL_PIDS="$OSLEVEL_PIDS $!"
    done

    #================================================================
    : Populate the return hash with the retrieved attributes/values
    : To improve performance, the data retrieval for each node is
    : done in parallel, via backgrounded processes.
    #================================================================
    rm -f $DATAFILE
    for node in $NODES; do
      (
        : Store the STDERR for $node in $CLMGR_ERRLOG.$node
        exec 2>>$CLMGR_ERRLOG.$node

        NODEID=${NODE_IDS[$node]}

        #===========================================================
        : Determine the fully-qualified ID number for node "$node"
        #===========================================================
        typeset CID=$(clodmget -f id HACMPcluster)
        properties[GID$NODEID]="$CID$NODEID"

        properties[NAME$NODEID]=$node

        print "$0()[$LINENO]($SECONDS): clodmget -q \"name=$node and object=COMMUNICATION_PATH\" -n -f value HACMPnode" >>$CLMGR_TMPLOG  # Always log commands
        properties[COMMPATH$NODEID]=$(clodmget -q "name=$node and object=COMMUNICATION_PATH" -n -f value HACMPnode)
        print "$0()[$LINENO]($SECONDS): clodmget RC: $? (\"${properties[COMMPATH$NODEID]}\")" >>$CLMGR_TMPLOG

        #==============================================
        : Determine the current state of node "$node"
        #==============================================
        typeset state=$(CL=$LINENO KLIB_HACMP_get_node_state $node 2>>$CLMGR_TMPLOG)
        properties[RAW_STATE$NODEID]=$state
        if [[ $state == *ST_INIT* ]]; then
            state="OFFLINE"
        elif [[ $state == *NOT_CONFIGURED* ]]; then
            state="NOT_CONFIGURED"
        elif [[ $state == *@(FAILED|ERROR)* ]]; then
            state="ERROR"
        elif [[ $state == "WARNING" ]]; then
            state="WARNING"
            : WARNING is an interpolated state, so go get the true, raw state
            properties[RAW_STATE$NODEID]=$($CLRSH $node "LC_ALL=C /usr/bin/lssrc -ls clstrmgrES" | grep "^Current state:" | cut -d" " -f3)
        elif [[ $state == *@(STABLE|JOINING|UNSTABLE|VOTING|RUNNING|BARRIER|DONE)* ]]
        then
            state="NORMAL"
        elif [[ $state == *UNMANAGED* ]]; then
            state="UNMANAGED"
        else
            state="UNKNOWN"
        fi
        properties[STATE$NODEID]=$state

        #===========================================
        : Determine the hostname for node "$node".
        : This attribute is used by the GUI during
        : the cluster discovery operation     
        #===========================================
        properties[HOST_LABEL$NODEID]=""
        print "$0()[$LINENO]($SECONDS): $CLRSH -n $node /usr/bin/hostname"  >>$CLMGR_TMPLOG  # Always log commands
        HOST_LABEL=$($CLRSH -n $node /usr/bin/hostname 2>/dev/null)
        print "$0()[$LINENO]($SECONDS): hostname RC: $?; HOST_LABEL == \"$HOST_LABEL\"" >>$CLMGR_TMPLOG  # Always log command results
        properties[HOST_LABEL$NODEID]=$HOST_LABEL

        #==========================================================
        : Determine the fully-qualified hostname for node "$node"
        #==========================================================
        properties[HOSTNAME$NODEID]=""
        if [[ -n ${properties[COMMPATH$NODEID]} ]]; then
            print "$0()[$LINENO]($SECONDS): LC_ALL=C host ${properties[COMMPATH$NODEID]}" >>$CLMGR_TMPLOG  # Always log commands
            HOSTNAME=$(LC_ALL=C host ${properties[COMMPATH$NODEID]} 2>/dev/null)
            print "$0()[$LINENO]($SECONDS): host RC: $?; HOSTNAME == \"$HOSTNAME\"" >>$CLMGR_TMPLOG  # Always log command result

            typeset FQDN=${HOSTNAME%%+([[:space:]])*}
            if [[ $FQDN != *\.* ]]; then
                : The first component of the host output does not
                : look fully-qualified. This might mean there is an incorrect
                : /etc/hosts configuration. Checking aliases...
                typeset ALIASES=${HOSTNAME##*:}
                for FQDN in ${ALIASES//,/ }; do
                    [[ $FQDN == *\.* ]] && break
                done

                if [[ $FQDN != *\.* || \
                      $FQDN == +([0-9])\.+([0-9])\.+([0-9])\.+([0-9]) ]]
                then
                    : None of the aliases were fully-qualified either. There
                    : is definitely something wrong with this configuration.
                    : Defaulting to the output of the hostname utility.
                    HOSTNAME=${HOSTNAME%%+([[:space:]])*}
                else
                    HOSTNAME=$FQDN
                fi
            else
                HOSTNAME=$FQDN
            fi

            properties[HOSTNAME$NODEID]=${HOSTNAME%%[[:space:]]*}
        fi

        #====================================================================
        : Fetch the CAA state of node "$node".
        :     1. First try querying the local, then all remaining nodes
        :     2. If lscluster fails on all nodes, check clcomd on all nodes
        :        a. If clcomd is working everywhere, that means the nodes
        :           are up. So the CAA state can be safely set to DOWN.
        :        b. If one or more clcomd invocations fail, then the CAA
        :           state must be set to UNKNOWN.
        #====================================================================
        typeset CAA_NODE=""
        integer NODE_COUNT=0 CAA_DOWN_COUNT=0
        #===============================================================
        : Get the CAA Node from the PowerHA nodename by calling cl_nn2hn
        : and get the CAA node state using lscluster
        #===============================================================
        caa_node_name=$(VERBOSE_LOGGING="" cl_nn2hn $node)
        
        for CAA_NODE in $LOCAL_NODE $(clnodename | grep -vw $LOCAL_NODE); do
            print "$0()[$LINENO]($SECONDS):  $CLRSH $CAA_NODE \"LC_ALL=C /usr/sbin/lscluster -m ${caa_node_name}\"" >>$CLMGR_TMPLOG
            properties[CAA_STATE$NODEID]=$($CLRSH $CAA_NODE "LC_ALL=C /usr/sbin/lscluster -m ${caa_node_name}" 2>>$CLMGR_TMPLOG | grep "State of node:")
            print "$0()[$LINENO]($SECONDS): lscluster RC: $?" >>$CLMGR_TMPLOG
            if [[ -n ${properties[CAA_STATE$NODEID]} ]]; then
                properties[CAA_STATE$NODEID]=${properties[CAA_STATE$NODEID]#*: }
                properties[CAA_STATE$NODEID]=${properties[CAA_STATE$NODEID]%% *}
                break
            fi
        done
        if [[ -z ${properties[CAA_STATE$NODEID]} ]]; then
            integer CLCOMD_STATUS=0
            for CAA_NODE in $(clnodename); do
                (( NODE_COUNT++ ))
                print "$0()[$LINENO]($SECONDS): $CLRSH $CAA_NODE /bin/true" >>$CLMGR_TMPLOG
                $CLRSH $CAA_NODE /bin/true 2>>$CLMGR_TMPLOG
                CLCOMD_STATUS=$?
                print "$0()[$LINENO]($SECONDS): /bin/true RC: $CLCOMD_STATUS" >>$CLMGR_TMPLOG
                if (( CLCOMD_STATUS != RC_SUCCESS )); then
                    break
                fi
                (( CAA_DOWN_COUNT++ ))
            done
        fi
        if [[ ${properties[CAA_STATE$NODEID]} == *UP* ]]; then
            typeset UP_MSG=$(dspmsg -s 1 cluster 980 "UP")
            properties[CAA_STATE$NODEID]=${properties[CAA_STATE$NODEID]/UP/$UP_MSG}
        elif [[ ${properties[CAA_STATE$NODEID]} == *DOWN* ]] || \
             (( CAA_DOWN_COUNT > 0 && CAA_DOWN_COUNT == NODE_COUNT ))
        then
            : NODE_COUNT == $NODE_COUNT, CAA_DOWN_COUNT == $CAA_DOWN_COUNT
            properties[CAA_STATE$NODEID]=$(dspmsg -s 1 cluster 981 "DOWN")
        else
            : NODE_COUNT == $NODE_COUNT, CAA_DOWN_COUNT == $CAA_DOWN_COUNT
            properties[CAA_STATE$NODEID]=$(dspmsg -s 1 cluster 982 "UNKNOWN")
        fi

        #================================================
        : Get the version information for node "$node",
        : plus its current synchronization status.
        #================================================
        typeset VRMF= unsynced=
        if [[ $node == $LOCAL_NODE ]]; then
            print "$0()[$LINENO]($SECONDS): lslpp -Lcq cluster.es.server.rte" >>$CLMGR_TMPLOG  # Always log commands
            VRMF=$(lslpp -Lcq cluster.es.server.rte)
            print "$0()[$LINENO]($SECONDS): lslpp RC: $?; VRMF == \"$VRMF\"" >>$CLMGR_TMPLOG  # Always log command result

            print "$0()[$LINENO]($SECONDS): clmixver" >>$CLMGR_TMPLOG  # Always log commands
            properties[VERSION_NUMBER$NODEID]=$(clmixver 2>>$CLMGR_TMPLOG)
            print "$0()[$LINENO]($SECONDS): clmixver RC: $?; VERSION_NUMBER == \"${properties[VERSION_NUMBER$NODEID]}\"" >>$CLMGR_TMPLOG  # Always log command result
            if [[ -z ${properties[VERSION_NUMBER$NODEID]} ]]; then
                properties[VERSION_NUMBER$NODEID]=$(grep CLUSTERVERSION= /usr/es/sbin/cluster/etc/rc.cluster)
                properties[VERSION_NUMBER$NODEID]=${properties[VERSION_NUMBER$NODEID]#*=}
            fi

            print "$0()[$LINENO]($SECONDS): clodmget -f handle HACMPcluster" >>$CLMGR_TMPLOG  # Always log commands
            unsynced=$(clodmget -f handle HACMPcluster)
            print "$0()[$LINENO]($SECONDS): clodmget RC: $?; unsynced == \"$unsynced\"" >>$CLMGR_TMPLOG  # Always log command result
        else
            print "$0()[$LINENO]($SECONDS): $CLRSH $node \"/usr/bin/lslpp -Lcq cluster.es.server.rte\"" >>$CLMGR_TMPLOG  # Always log commands
            VRMF=$($CLRSH $node "/usr/bin/lslpp -Lcq cluster.es.server.rte" 2>>$CLMGR_TMPLOG)
            print "$0()[$LINENO]($SECONDS): lslpp RC: $?; VRMF == \"$VRMF\"" >>$CLMGR_TMPLOG  # Always log command result

            print "$0()[$LINENO]($SECONDS): $CLRSH $node \"$HAUTILS/clmixver\"" >>$CLMGR_TMPLOG  # Always log commands
            properties[VERSION_NUMBER$NODEID]=$($CLRSH $node "$HAUTILS/clmixver" 2>>$CLMGR_TMPLOG)
            print "$0()[$LINENO]($SECONDS): clmixver RC: $?; VERSION_NUMBER == \"${properties[VERSION_NUMBER$NODEID]}\"" >>$CLMGR_TMPLOG  # Always log command result
            if [[ -z ${properties[VERSION_NUMBER$NODEID]} ]]; then
                properties[VERSION_NUMBER$NODEID]=$($CLRSH $node "/usr/bin/grep CLUSTERVERSION= /usr/es/sbin/cluster/etc/rc.cluster" 2>>$CLMGR_TMPLOG)
                properties[VERSION_NUMBER$NODEID]=${properties[VERSION_NUMBER$NODEID]#*=}
            fi

            print "$0()[$LINENO]($SECONDS): $CLRSH $node \"$HAUTILS/clodmget -f handle HACMPcluster\"" >>$CLMGR_TMPLOG  # Always log commands
            unsynced=$($CLRSH $node "$HAUTILS/clodmget -f handle HACMPcluster" 2>>$CLMGR_TMPLOG)
            print "$0()[$LINENO]($SECONDS): clodmget RC: $?; unsynced == \"$unsynced\"" >>$CLMGR_TMPLOG  # Always log command result
        fi
        print -- "$VRMF" | IFS=: read UNUSED UNUSED properties[VERSION$NODEID] UNUSED
        properties[UNSYNCED_CHANGES$NODEID]=false
        unsynced=${unsynced##*= }
        [[ $unsynced == 0 ]] && properties[UNSYNCED_CHANGES$NODEID]=true

        #======================================================================
        : Determine what edition of SystemMirror is installed on node "$node"
        #======================================================================
        properties[EDITION$NODEID]="STANDARD"
        CL=$LINENO isEnterprise $node
        (( $? == 1 )) && properties[EDITION$NODEID]="ENTERPRISE"

        #===============================================================
        : Gather the current cluster service settings for node "$node"
        #===============================================================
        typeset -l BROADCAST_ON_START="" START_ON_BOOT="false"
        typeset    CLINFO_ON_START="" CGodm=""
        integer    CGlslpp=-1
        if [[ -z $LOCAL_NODE || $LOCAL_NODE == $node ]]; then
            BROADCAST_ON_START=$(clodmget -q "nodename=$node and type=start and object=broadcast" -n -f value HACMPdaemons)

            lsitab hacmp6000 >/dev/null
            (( $? == 0 )) && START_ON_BOOT="true"

            CLINFO_ON_START=$(clodmget -q "nodename=$node and daemon=clinfo and object=time" -n -f value HACMPdaemons)
            # Override the default clinfo flag based on consistency
            # groups being installed and configured
            CGodm=$(odmget HACMPpprcconsistgrp)
            lslpp -lc cluster.es.cgpprc.rte >/dev/null 2>&1
            CGlslpp="$?"

        else
            BROADCAST_ON_START=$($CLRSH $node "$HAUTILS/clodmget -q \"nodename=$node and type=start and object=broadcast\" -n -f value HACMPdaemons" 2>>$CLMGR_TMPLOG)

            $CLRSH $node /usr/sbin/lsitab hacmp6000 >/dev/null 2>>$CLMGR_TMPLOG
            (( $? == 0 )) && START_ON_BOOT="true"

            CLINFO_ON_START=$($CLRSH $node "$HAUTILS/clodmget -q \"nodename=$node and daemon=clinfo and object=time\" -n -f value HACMPdaemons" 2>>$CLMGR_TMPLOG)
            # Override the default clinfo flag based on consistency
            # groups being installed and configured
            CGodm=$($CLRSH $node "/usr/bin/odmget HACMPpprcconsistgrp" 2>>$CLMGR_TMPLOG)
            $CLRSH $node "/usr/bin/lslpp -lc cluster.es.cgpprc.rte >/dev/null 2>&1" 2>>$CLMGR_TMPLOG
            CGlslpp=$?

        fi
        [[ $BROADCAST_ON_START != "true" ]] && BROADCAST_ON_START="false"
        if [[ $CLINFO_ON_START == "true_CG_support" && $CGlslpp == 1 ]] || \
             [[ $CLINFO_ON_START == "true_CG_support" && -z $CGodm ]]
        then
            # User had clinfo started with CG support, but CG is no longer
            # installed or configured. Leave it set to true, but without
            # trap support.
            CLINFO_ON_START="true"
        fi
        if [[ $CLINFO_ON_START == *([[:space:]]) ]]; then
            CLINFO_ON_START="false"
        fi
        properties[START_ON_BOOT$NODEID]=$START_ON_BOOT
        properties[BROADCAST_ON_START$NODEID]=$BROADCAST_ON_START
        properties[CLINFO_ON_START$NODEID]=$CLINFO_ON_START

        #====================================================
        : Retrieve the HMCS for node "$node"
        #====================================================
        typeset HMCLABELS=
        typeset hmclabels=
        print "$0()[$LINENO]($SECONDS): cllshmcparam -n $node -o " >>$CLMGR_TMPLOG  # Always log commands
        hmclabels=$(cllshmcparam -n $node -o)
        rc=$?
        print "$0()[$LINENO]($SECONDS): cllshmcparam -n $node -o rc=$rc" >>$CLMGR_TMPLOG  # Always log commands result
        if (( $rc == $RC_SUCCESS )); then
            for hmclabel in $hmclabels; do
                if [[ -n $hmclabel ]]; then
                    HMCLABELS="${HMCLABELS:+$HMCLABELS }$hmclabel"
                fi
            done
            properties[HMCS$NODEID]=$HMCLABELS
        fi

        #====================================================
        : Retrieve the NovaLinks for node "$node"
        #====================================================
        typeset NOVALABELS=
        typeset novalabels=
        print "$0()[$LINENO]($SECONDS): cl_get_nova_list $node 2>/dev/null " >>$CLMGR_TMPLOG  # Always log commands
        novalabels=$(cl_get_nova_list $node 2>/dev/null)
        rc=$?
        print "$0()[$LINENO]($SECONDS): cl_get_nova_list $node 2>/dev/null rc=$rc" >>$CLMGR_TMPLOG  # Always log commands result
        if (( $rc == $RC_SUCCESS )); then
            for novalabel in $novalabels; do
                if [[ -n $novalabels ]]; then
                    NOVALABELS="${NOVALABELS:+$NOVALABELS }$novalabel"
                fi
            done
            properties[NOVAS$NODEID]=$NOVALABELS
        fi

        properties[PERSISTENT_IPS$NODEID]="$(clodmget -q "nodename = $node AND function = persistent" -n -f ip_label HACMPadapter)"
        properties[PERSISTENT_IPS$NODEID]="${properties[PERSISTENT_IPS$NODEID]//$NL/ }"

        #====================================================
        : Retrieve the adapter information for node "$node"
        #====================================================
        if [[ $CLMGR_GUI == "SMUI" ]]
        then
            print "$0()[$LINENO]($SECONDS): clodmget -q \"nodename = $node\" -n -f ip_label,identifier,network,netmask,function HACMPadapter" >>$CLMGR_TMPLOG  # Always log commands
            clodmget -q "nodename = $node" -n -f ip_label,identifier,network,netmask,function HACMPadapter | sort -t: -k 5,5 |\
            while IFS=: read IPLABEL IPADDRESS IPNETWORK IPNETMASK IPTYPE rest
            do
                if [[ -n $IPLABEL ]]
                then
                    properties[IPLABEL$NODEID.$INDEX]=$IPLABEL
                    properties[IPADDRESS$NODEID.$INDEX]=$IPADDRESS
                    properties[NETWORK$NODEID.$INDEX]=$IPNETWORK
                    properties[NETMASK$NODEID.$INDEX]=$IPNETMASK
                    properties[TYPE$NODEID.$INDEX]=$IPTYPE
                    (( INDEX++ ))
                    [[ $INDEX == *0 ]] && (( INDEX++ ))
                else
                    break
                fi
            done
        fi

        #========================================================
        : Attempt to determine if the local node can initiate
        : communications to clcomd, and if so, if the remote
        : node can reverse that process.
        #
        # NOTE: This code only validates the local node against
        #       the remote node. It will not cover the case
        #       where two remote nodes might not be talking to
        #       each other. It does not attempt to test every
        #       possible combination of every node. That would
        #       get lengthy in a 32 node cluster... This simple
        #       test is just for *this* node.
        #========================================================
        typeset COMMUNICATION="DOWN"
        $CLRSH $node /bin/true 2>>$CLMGR_TMPLOG
        if (( $? == RC_SUCCESS )); then
            : Outbound worked. Try inbound.
            $CLRSH $node "$HAUTILS/$CLRSH $LOCAL_NODE /bin/true" 2>>$CLMGR_TMPLOG
            if (( $? == RC_SUCCESS )); then
                COMMUNICATION="UP"
            fi
        fi
        properties[COMMUNICATION$NODEID]=$COMMUNICATION

        properties[LOCALHOST$NODEID]="false"
        [[ $node == $LOCAL_NODE ]] && properties[LOCALHOST$NODEID]="true"

        #====================================================
        : Check that Live Update is enabled for node $node
        #====================================================
        typeset -l ENABLE_LIVE_UPDATE="false"
        typeset -i cmd_rc=$RC_UNKNOWN
        if [[ -z $LOCAL_NODE || $LOCAL_NODE == $node ]]; then
            print "$0()[$LINENO]($SECONDS): clenablelku -c" >>$CLMGR_TMPLOG  # Always log commands
            clenablelku -c >>$CLMGR_TMPLOG 2>&1
            cmd_rc=$?
            print "$0()[$LINENO]($SECONDS): clenablelku RC: $cmd_rc" >>$CLMGR_TMPLOG  # Always log commands
            (( $cmd_rc == 0 )) && ENABLE_LIVE_UPDATE="true"
        else
            print "$0()[$LINENO]($SECONDS): $CLRSH $node $HAUTILS/clenablelku -c" >>$CLMGR_TMPLOG  # Always log commands
            $CLRSH $node $HAUTILS/clenablelku -c >>$CLMGR_TMPLOG 2>&1
            cmd_rc=$?
            print "$0()[$LINENO]($SECONDS): $CLRSH $node $HAUTILS/clenablelku RC: $cmd_rc" >>$CLMGR_TMPLOG  # Always log commands
            (( $cmd_rc == 0 )) && ENABLE_LIVE_UPDATE="true"
        fi
        properties[ENABLE_LIVE_UPDATE$NODEID]=$ENABLE_LIVE_UPDATE

        #===================================================================
        : Check node is Cloud enabled LPAR or not
        #===================================================================
        typeset -l CLOUD_BASED="On_Premise"
        typeset -i cmd_rc=$RC_UNKNOWN

        nodeVM_flag=$($CLRSH $node /usr/es/sbin/cluster/utilities/cl_iscloud_vm)
        cmd_rc=$?
        print "$0()[$LINENO]($SECONDS): $CLRSH $node /usr/es/sbin/cluster/utilities/cl_iscloud_vm RC: $cmd_rc" >>$CLMGR_TMPLOG  # Always log commands 
        if (( $cmd_rc == 0)); then
            if [[ "$nodeVM_flag" == "no" ]];then
               CLOUD_BASED="On_Premise"
            else
               CLOUD_BASED="Cloud"
            fi
        fi
        properties[CLOUD_BASED$NODEID]=$CLOUD_BASED

 
        #===================================================================
        : get the critical daemon restart grace period for the node  $node
        #===================================================================
        
        print "$0()[$LINENO]($SECONDS): clodmget -q \"name=$node and object=CRIT_DAEMON_RESTART_GRACE_PERIOD\" -n -f value HACMPnode" >>$CLMGR_TMPLOG  # Always log commands
        properties[CRIT_DAEMON_RESTART_GRACE_PERIOD$NODEID]=$(clodmget -q "name=$node and object=CRIT_DAEMON_RESTART_GRACE_PERIOD" -n -f value HACMPnode)
        print "$0()[$LINENO]($SECONDS): clodmget RC: $? (\"${properties[CRIT_DAEMON_RESTART_GRACE_PERIOD$NODEID]}\")" >>$CLMGR_TMPLOG

        #======================================================
        : Store the retrieved data in a data file, since this
        : operation is being done as a backgrounded process
        #======================================================
        for attr in ${!properties[*]}; do
            print "$attr^${properties[$attr]}" >>$DATAFILE
        done

        print -n $RC_SUCCESS >>$GNA_RESULTS_FILE
       )&
       GNA_CHILDREN="$GNA_CHILDREN $!"
    done

    if [[ -n $GNA_CHILDREN ]]; then

        : ##############################
        : Set an alarm to prevent a hang
        (
            sleep $CLMGR_TIMEOUT &
            alarmPID=$!
            print -n $alarmPID >$TMPDIR/clmgr.KHgna.alarm.pid.$$
            wait $alarmPID

            cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 187 'Warning: the current operation has exceeded its allotted execution time (%1$s seconds).\n' "$CLMGR_TIMEOUT" 1>&2
            [[ -n $CLMGR_COMMAND ]] && print -u2 "\n\t${CLMGR_COMMAND/ -T $CLMGR_TRANSACTION_ID}\n"
            print -n 1 >>$GNA_RESULTS_FILE

            cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 470 "Attempting to terminate any remaining, active processes...\n\n" "$CLMGR_TIMEOUT" 1>&2
            for pid in $GNA_CHILDREN; do kill $pid 2>/dev/null; done
            sleep 5
            for pid in $GNA_CHILDREN; do kill -9 $pid 2>/dev/null; done
        )&
        typeset -i reaperPID=$!
        : ######################################################
        : Wait for all the backgrounded processes to complete...
        wait $GNA_CHILDREN
        kill -9 $reaperPID $([[ -f $TMPDIR/clmgr.KHgna.alarm.pid.$$ ]] && cat $TMPDIR/clmgr.KHgna.alarm.pid.$$) 2>/dev/null
        [[ -f $TMPDIR/clmgr.KHgna.alarm.pid.$$ ]] && rm -f $TMPDIR/clmgr.KHgna.alarm.pid.$$

        #=====================================================================
        : Make sure the OS retrievals are complete, then collect any results
        #=====================================================================
        if [[ -n $OSLEVEL_PIDS ]]; then
            wait $OSLEVEL_PIDS
            if [[ -f $OSLEVEL_DATA && -s $OSLEVEL_DATA ]]; then
                typeset OSLEVELS=$(cat $OSLEVEL_DATA)
                [[ $OSLEVELS == *=* ]] && eval "$OSLEVELS"
            fi
        fi
        rm -f $OSLEVEL_DATA

        #============================================================
        : Append the STDERR from each node to the log in one piece.
        : Doing this avoids log data interleaving.
        #============================================================
        for node in $NODES; do
            cat $CLMGR_ERRLOG.$node >> $CLMGR_ERRLOG
            rm -f $CLMGR_ERRLOG.$node
        done

        if [[ -f $DATAFILE ]]; then
            while IFS="^" read attr value; do
                properties[$attr]=$value
            done <$DATAFILE
            rm -f $DATAFILE
        fi

        if (( $rc == RC_UNKNOWN )); then
            typeset NON_ZERO_RCS=$(cat $GNA_RESULTS_FILE)
            rm -f $GNA_RESULTS_FILE
            NON_ZERO_RCS=${NON_ZERO_RCS//0/}
            NON_ZERO_RCS=${NON_ZERO_RCS//+([[:space:]])/}
            if [[ $NON_ZERO_RCS != *([[:space:]]) ]]; then
                rc=$RC_ERROR
            else
                rc=$RC_SUCCESS
            fi
        fi

        #==========================================================
        : If any searches/filters were specified, handle them now
        #==========================================================
        if (( $rc == RC_SUCCESS && ${#queries[*]} > 0 )); then
            CL=$LINENO search_properties queries properties
            rc=$?
        fi
    fi

    (( $rc == RC_SUCCESS )) && CL=$LINENO prune_indexes properties

    log_return_msg "$rc" "$0()" "$LINENO"
    return $?
} # End of "KLIB_HACMP_get_node_attributes()"


#============================================================================
#
# Name:        devDoc
#
# Description: This is a never-to-be-called, wrapper function that all the
#              clmgr FPATH functions implement in order to hide embedded
#              syntax from trace logging. This information is implemented
#              in POD format, and can be viewed in a number of ways using
#              POD tools. Some viewing suggestions for this function's POD-
#              formatted information are:
#
#                  perldoc <FILENAME>
#                  pod2text -c <FILENAME>
#                  pod2text -c --code <FILENAME>
#                  pod2html <FILENAME>
#
#              However, the more important use for this information is that
#              it is parsed by clmgr to display the syntax for this file's
#              operation. The information in the "SYNOPSIS" section is used
#              for this purpose. This feature was originally implemented
#              using the man page information. However, in a code review it
#              was pointed out that this approach had to be changed because
#              customers do not have to install the man pages! Therefore, a
#              built-in dependency on man page information would break the
#              automatic help feature of clmgr. So the SYNPOSIS section must
#              be used instead.
#
#              IMPORTANT: As a result of this, it is imperative that the
#                         information in this SYNOPSIS be kept in sync
#                         with the man page information, which is owned
#                         by the IDD team.
#
# Inputs:      None.
#
# Outputs:     None.
#
# Returns:     n/a (not intended to be invoked)
#
#============================================================================
function devDoc {
    : <<'=cut' >/dev/null 2>&1

=head1 NAME

 KLIB_HACMP_get_node_attributes

=head1 SYNOPSIS

 clmgr query node {<node>|LOCAL}[,<node#2>,...]
 clmgr -v query node

 NOTE: the alias for "node" is "no".

=head1 DESCRIPTION

Retrieves the configuration data for one or more PowerHA nodes.

=head1 ARGUMENTS

 1. node [REQUIRED] [string]
    The label of one or more PowerHA nodes (comma
    or space separated) that are to be queried.

 2. properties [REQUIRED] [hash ref]
    An associative array within which data about the
    queried objects can be returned to the caller.

=head1 RETURN

 0: no errors were detected; the operation appears to have been successful
 1: a general error has occurred
 2: a specified resource does not exist, or could not be found
 3: some required input was missing
 4: some detected input was incorrect in some way
 5: a required dependency does not exist
 6: a specified search failed to match any data

=cut
} # End of "devDoc()"


#==============================================================================
# The following, comment block attempts to enforce coding standards when this
# file is edited via emacs or vim. This block _must_ appear at the very end
# of the file, or the editor will not find it, and it will be ignored.
#==============================================================================
# Local Variables:
# indent-tabs-mode: nil
# tab-width: 4
# End:
#==============================================================================
# vim: tabstop=4 shiftwidth=4 expandtab
#==============================================================================