#!/bin/ksh93
#  ALTRAN_PROLOG_BEGIN_TAG
#  This is an automatically generated prolog.
#
#  Copyright (C) Altran ACT S.A.S. 2017,2021.  All rights reserved.
#
#  ALTRAN_PROLOG_END_TAG
#
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r721 src/43haes/lib/ksh93/ezupdate/Cluster_t.sh 1.4 
#  
# Licensed Materials - Property of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2016 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 

# @(#)  7d4c34b 43haes/lib/ksh93/ezupdate/Cluster_t.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM

###############################
# Function list:
#  create
#  check_rte
#  display
#  file_propagate
#  refresh
###############################

typeset scriptname=${0##*/}

. ${EZU_LIB_DIR}/common
. ${EZU_LIB_DIR}/log

if [[ -z $NODE_T_DEFINED ]]; then
    . ${EZU_LIB_DIR}/Node_t
fi

###################################################
# This type to encapsulate cluster management
###################################################
typeset -T Cluster_t=(
    typeset -h 'name of the cluster'    name=""
    typeset -h 'cluster state'          state=""
    typeset -h 'cluster version'        version=""
    typeset -h 'cluster version number' version_number=""
    typeset -h 'resource groups list'   rg_list=""
    typeset -h 'unsync changes'         unsynced_changes=""
    typeset -h 'mixed cluster'          mixed=""
    typeset -h 'local node name'        local_node_name=""

    Node_t -A nodes

    #####################################################################
    #
    # NAME: display
    #
    # FUNCTION:
    #     Display the current Cluster_t object
    #
    # EXECUTION ENVIRONMENT:
    #
    # NOTES:
    #
    # RECOVERY OPERATION:
    #
    # DATA STRUCTURES:
    #     parameters:
    #     global:
    #           Cluster_t
    #
    # RETURNS: none
    #
    # OUTPUT:
    #####################################################################
    function display {
        [[ "$VERBOSE_LOGGING" == "high" ]] && set -x
        [[ "$DEBUG_MODE" == "yes" ]] && set -x

        typeset current_name=""
        print -- " cluster"
        print -- "    name="${_.name}
        print -- "    state="${_.state}
        print -- "    version="${_.version}
        print -- "    version number="${_.version_number}
        print -- "    resources groups="${_.rg_list}
        print -- "    unsync changes="${_.unsynced_changes}
        print -- "    mixed cluster="${_.mixed}

        for current_name in ${managed_nodes}
        do
            # This eval is necessary to set a compound object
            # to another compound object and this is necessary if ever
            # we want to call a method on this compound object
            eval "Node_t currentNode=${_.nodes[$current_name]}"
            currentNode.display
        done
    } # End of "display()"

    #####################################################################
    #
    # NAME: refresh
    #
    # FUNCTION:
    #     Refresh Cluster_t data
    #
    # EXECUTION ENVIRONMENT:
    #
    # NOTES:
    #     It runs the refresh function for each node of the cluster.
    #
    # RECOVERY OPERATION:
    #
    # DATA STRUCTURES:
    #     parameters:
    #     global:
    #           Cluster_t
    #
    # RETURNS: (int)
    #     RC.FAILURE - error with the clmgr command or while refreshing the nodes
    #     RC.OK      - Success
    #
    # OUTPUT:
    #####################################################################
    function refresh {
        [[ "$VERBOSE_LOGGING" == "high" ]] && set -x
        [[ "$DEBUG_MODE" == "yes" ]] && set -x

        typeset cmd=""
        typeset -i rc=${RC.OK}
        typeset cmd_output=""
        typeset tmp_version=""

        cmd="$CLMGR_CMD -cSa STATE,VERSION,VERSION_NUMBER,UNSYNCED_CHANGES query cluster"
        log_trace 0 "$0()[$LINENO]($SECONDS): execute cmd: $cmd"
        cmd_output=$($cmd 2>&1)
        rc=$?
        log_trace 0 "$0()[$LINENO]($SECONDS): command returns code: $rc"
        log_trace 0 "$0()[$LINENO]($SECONDS): command output='$cmd_output'"
        (( $rc != 0 )) && return ${RC.FAILURE}
        echo $cmd_output | IFS=: read _.state _.version _.version_number _.unsynced_changes

        # Refresh all nodes defined in the cluster
        for current_name in "${!_.nodes[@]}"
        do
            # This eval is necessary to set a compound object to
            # another compound object and this is necessary if ever
            # we want to call a method on this compound object

            eval "Node_t currentNode=${_.nodes[$current_name]}"
            currentNode.refresh
            (( $? != ${RC.OK} && rc == ${RC.OK} )) && rc=${RC.FAILURE}

            eval "_.nodes[$current_name]=$currentNode"
            [[ -z ${tmp_version} ]] && tmp_version=${currentNode.version}

            if [[ ${_.version} != ${currentNode.version} ]]
            then
                _.mixed="YES"
                echo ${_.version} | IFS="." read cl_mr cl_r cl_tl cl_sp
                echo ${currentNode.version} | IFS="." read node_mr node_r node_tl node_sp
                if (( ${node_mr} < ${cl_mr} ))
                then
                    _.version=${currentNode.version}
                elif (( ${node_r} < ${cl_r} ))
                then
                    _.version=${currentNode.version}
                elif (( ${node_tl} < ${cl_tl} ))
                then
                    _.version=${currentNode.version}
                elif (( ${node_sp} < ${cl_sp} ))
                then
                    _.version=${currentNode.version}
                fi
            fi
        done

        return $rc
    } # End of "refresh()"


    #####################################################################
    #
    # NAME: create
    #
    # FUNCTION:
    #     Create the Cluster_t object when it is declared.
    #a
    # EXECUTION ENVIRONMENT:
    #
    # NOTES:
    #     The discipline function create is special, it is invoked like a
    #     constructor when an instance is created.
    #
    # RECOVERY OPERATION:
    #
    # DATA STRUCTURES:
    #     parameters:
    #     global:
    #           Cluster_t
    #
    # RETURNS: Upon error this function exit the script with return code set.
    #     RC.FAILURE - a clmgr or other command failed
    #     RC.OK      - Success
    #
    # OUTPUT:
    #####################################################################
    function create {
        [[ "$VERBOSE_LOGGING" == "high" ]] && set -x
        [[ "$DEBUG_MODE" == "yes" ]] && set -x
        trap 'print -- "$CANNOT_INT_MSG"' INT

        typeset local_hostname=""
        typeset node_list=""
        typeset cmd=""
        typeset -i rc=${RC.OK}
        typeset cmd_output=""

        _.mixed="NO"

        # Get local hostname
        cmd="/usr/bin/hostname"
        log_trace 0 "$0()[$LINENO]($SECONDS): execute cmd: $cmd"
        cmd_output=$($cmd)
        rc=$?
        log_trace 0 "$0()[$LINENO]($SECONDS): command returns code: $rc"
        log_trace 0 "$0()[$LINENO]($SECONDS): command output='$cmd_output'"
        (( $rc != 0 )) && exit ${RC.FAILURE}
        local_hostname=$cmd_output

        # Get local node name
        _.local_node_name=$(VERBOSE_LOGGING="" $GET_LOCAL_NODENAME)

        # Check run time environment
        _.check_rte
        rc=$?
        (( $rc != ${RC.OK} )) && exit $rc

        # Get cluster information using clmgr
        cmd="$CLMGR_CMD -cSa CLUSTER_NAME,STATE,VERSION,VERSION_NUMBER,UNSYNCED_CHANGES query cluster"
        log_trace 0 "$0()[$LINENO]($SECONDS): execute cmd: $cmd"
        cmd_output=$($cmd 2>&1)
        rc=$?
        log_trace 0 "$0()[$LINENO]($SECONDS): command returns code: $rc"
        log_trace 0 "$0()[$LINENO]($SECONDS): command output='$cmd_output'"
        if (( $rc != 0 ))
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 4 'ERROR: command "%1$s" failed.\n' "$cmd"
            exit ${RC.FAILURE}
        fi
        echo $cmd_output | IFS=: read _.name _.state _.version _.version_number _.unsynced_changes

	#Check if the cluster is in unsync state then exit
	if [[ ${_.unsynced_changes} == "true" ]]
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 23 'ERROR: The cluster is in unsynced state. Please verify and sync the cluster and retry\n'
            exit ${RC.FAILURE}
        fi	
	
	DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 24 'INFO: The cluster: %1$s is in state: %2$s' "${_.name}" "${_.state}"

        # Get node list
        cmd="$CLMGR_CMD query nodes"
        log_trace 0 "$0()[$LINENO]($SECONDS): execute cmd: $cmd"
        cmd_output=$($cmd 2>&1)
        rc=$?
        log_trace 0 "$0()[$LINENO]($SECONDS): command returns code: $rc"
        log_trace 0 "$0()[$LINENO]($SECONDS): command output='$cmd_output'"
        if (( $rc != 0 ))
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 4 'ERROR: command "%1$s" failed.\n' "$cmd"
            exit ${RC.FAILURE}
        fi
        node_list=$cmd_output

        # Get RG list
        cmd="$CLMGR_CMD query resource_group"
        log_trace 0 "$0()[$LINENO]($SECONDS): execute cmd: $cmd"
        cmd_output=$($cmd 2>&1)
        rc=$?
        log_trace 0 "$0()[$LINENO]($SECONDS): command returns code: $rc"
        log_trace 0 "$0()[$LINENO]($SECONDS): command output='$cmd_output'"
        if (( $rc != 0 ))
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 4 'ERROR: command "%1$s" failed.\n' "$cmd"
            exit ${RC.FAILURE}
        fi
        _.rg_list=$cmd_output

        # Build the array of Node_t for each node of the cluster
        for node_name in $node_list
        do
            Node_t node
            node.rg_list=${_.rg_list}
            node.name=$node_name

            node.build
            if (( $? != ${RC.OK} ))
            then
                    DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 4 'ERROR: command "%1$s" failed.\n' "$cmd"
                    exit ${RC.FAILURE}
            fi
            [[ -z ${_.version} ]] && _.version=${node.version}

            if [[ ${_.version} != ${node.version} ]]
            then
                _.mixed="YES"
                echo ${_.version} | IFS="." read cl_mr cl_r cl_tl cl_sp
                echo ${node.version} | IFS="." read node_mr node_r node_tl node_sp
                if (( ${node_mr} < ${cl_mr} ))
                then
                    _.version=${node.version}
                elif (( ${node_r} < ${cl_r} ))
                then
                    _.version=${node.version}
                elif (( ${node_tl} < ${cl_tl} ))
                then
                    _.version=${node.version}
                elif (( ${node_sp} < ${cl_sp} ))
                then
                    _.version=${node.version}
                fi
            fi

            eval "_.nodes[${node_name}]=${node}"
        done

        return $rc
    } # End of "create()"

    #####################################################################
    #
    # NAME: clrsh_cmd
    #
    # FUNCTION:
    #   Execute a command on a remote node.
    #
    # EXECUTION ENVIRONMENT:
    #
    # NOTES:
    #   It uses /usr/es/sbin/cluster/utilities/cl_rsh for remote node.
    #
    # RECOVERY OPERATION:
    #
    # DATA STRUCTURES:
    #     parameters:
    #           1: OUTPUT to put the output of the command execution
    #           2: node_name is the name of the node to run the command
    #           3: as many parameter as wanted to specify the command
    #     global:
    #           Cluster_t
    #
    # RETURNS: (int)
    #     the command return code
    #
    # OUTPUT:
    #     OUTPUT contains the output of the command
    #####################################################################
    function clrsh_cmd {
        [[ "$VERBOSE_LOGGING" == "high" ]] && set -x
        [[ "$DEBUG_MODE" == "yes" ]] && set -x

        typeset -n OUTPUT=$1
        shift;
        typeset node_name=$1
        shift;
        typeset cmd="$*"
        typeset -i cmd_rc
        typeset cmd_output=""
        cmd="$cmd 2>&1"

        cmd="$CLRSH -n ${node_name} '$cmd'"
        log_trace 0 "$0()[$LINENO]($SECONDS): execute cmd: $cmd"
        cmd_output=$(eval $cmd)
        cmd_rc=$?
        log_trace 0 "$0()[$LINENO]($SECONDS): command returns code:$cmd_rc"
        log_trace 0 "$0()[$LINENO]($SECONDS): command output='$cmd_output'"
        OUTPUT=$cmd_output
        return $cmd_rc
    } # End of "clrsh_cmd()"

    #####################################################################
    #
    # NAME: file_propagate
    #
    # FUNCTION:
    #     Copy all files in a given directory to all nodes in the cluster.
    #a
    # EXECUTION ENVIRONMENT:
    #
    # NOTES:
    #     The discipline function create is special, it is invoked like a
    #     constructor when an instance is created.
    #
    # RECOVERY OPERATION:
    #
    # DATA STRUCTURES:
    #     parameters:
    #        1  -  The directory of files to propagate
    #        2  _  the list of nodes to propagate
    #     global:
    #           Cluster_t
    #
    # RETURNS: (int)
    #     RC.EMPTY_DIR - Specified repository directory is empty
    #     RC.FAILURE   - an error occured
    #     RC.OK        - Success
    #
    # OUTPUT:
    #####################################################################
    function file_propagate {
        [[ "$VERBOSE_LOGGING" == "high" ]] && set -x
        [[ "$DEBUG_MODE" == "yes" ]] && set -x

        typeset SDIR="$1"
        typeset node_list=$2
        typeset cmd=""
        typeset cmd_output=""           # to get command output
        typeset node_name=""
        typeset FCHR=""
        typeset TDIR=""
        typeset FCHR=""
        typeset FCHR=""

        # remove ending slash if there is one
        SDIR=${SDIR%\/}

        if [[ -z $SDIR ]]
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 1 'Error, arg1 (directory name) is required.\n'
            return ${RC.FAILURE}
        fi

        # turn relative path into full path name
        FCHR=$(echo $SDIR | head -c1)
        if [[ "$FCHR" == "/" ]]
        then
            TDIR=$SDIR
        else
            TDIR=$(pwd)
            TDIR=$TDIR/$SDIR
        fi

        # Check for empty directory
        if [[ -z "$(ls -A $TDIR)" ]]
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 2 'ERROR: Directory %1$s is empty.\n' "$SDIR"
            return ${RC.EMPTY_DIR}
        fi

        # Get local node
        if [[ -z ${_.local_node_name} ]]
        then
           DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 3 'ERROR: getting local node.\n'
           return ${RC.FAILURE}
        fi

        [[ -z $node_list ]] && node_list="${!_.nodes[@]}"
        log_trace 0 "$0()[$LINENO]($SECONDS): node_list=$node_list"

        # Make the target directory on each node if needed
        for node_name in  $node_list
        do
            if [[ "${_.local_node_name}" != "${node}" ]]
            then
                cmd="/usr/bin/mkdir -p $TDIR"
                _.clrsh_cmd cmd_output $node_name "$cmd"
                if (( $? != 0 ))
                then
                    DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 4 'ERROR: command "%1$s" failed.\n' "$cmd"
                    return ${RC.FAILURE}
                fi
            fi
        done

        # for each file in the list, copy it to all specified nodes
        for file in $(ls $TDIR)
        do
            if [[ -f $TDIR/$file ]]
            then
                for node_name in $node_list
                do
                    if [[ "${_.local_node_name}" != "${node_name}" ]]
                    then
                        $CLRCP $TDIR/$file $node_name:$TDIR/$file
                        if (( $? != 0 ))
                        then
                            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 4 'ERROR: command "%1$s" failed.\n' "cl_rcp $TDIR/$file $node_name:$TDIR/$file"
                            return ${RC.FAILURE}
                        else
                            log_trace 0 "$0()[$LINENO]($SECONDS): copied $TDIR/$file to node $node_name\n"
                        fi
                    fi
                done
            else
                log_trace 0 "$0()[$LINENO]($SECONDS): $file is not a regular file so is not copied\n"
            fi
        done

        DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 5 'Source directory %1$s successfully copied to nodes: %2$s.\n' "$SDIR" "$node_list"

        return ${RC.OK}
    } # End of "file_propagate()"


    #####################################################################
    #
    # NAME: check_rte
    #
    # FUNCTION:
    #     checks the run time environment required for EZUpdate
    #         - root user
    #         - PowerHA version 7.1.3 or later
    #         - AIX version 7 or greater
    #         - each node can send and receive clcomd messages
    # NOTES:
    #
    # DATA STRUCTURES:
    #     parameters:
    #     global:
    #           Node_t
    #
    # RETURNS: (int)
    #     RC.FAILURE       - RTE check error
    #     RC.PHA_CMD_ERROR - PowerHA command error
    #     RC.OK            - Success
    #
    # OUTPUT
    ###################################################################
    function check_rte {
        [[ "$VERBOSE_LOGGING" == "high" ]] && set -x
        [[ "$DEBUG_MODE" == "yes" ]] && set -x

        typeset cmd=""
        typeset cmd_output=""           # to get command output
        typeset -i rc=${RC.OK}
        typeset VV="" RR="" MM="" FF=""
        typeset -i MAX_TRIES=5
        typeset node=""
        typeset node_list=""
        typeset -i try=0 noOutbound=1 noInbound=1

        #
        : Check for root authority
        #
        DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 6 'Checking for root authority...\n'
        log_trace 0 "$0: whoami: [[ \"$(whoami)\" != \"root\" ]]"
        if [[ "$(whoami)" != "root" ]]
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 7 'ERROR: you must be root to use this program.\n'
            return ${RC.FAILURE}
        else
            DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 8 '    Running as root.\n\n'
        fi

        #
        : Check for AIX level greater than 6
        #
        DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 9 'Checking for AIX level...\n'
        /usr/bin/oslevel | IFS=. read VV RR MM FF
        log_trace 0 "$0()[$LINENO]($SECONDS): VV=$VV RR=$RR MM=$MM FF=$FF"

        if (( $VV < 7 ))
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 10 'ERROR: EZUpdate requires AIX version 7 or later.\n'
            return ${RC.FAILURE}
        fi
        DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 11 '    The installed AIX version is supported.\n\n'

        #
        : Check PowerHA version - must be 7.1.3 or later
        #
        DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 12 'Checking for PowerHA SystemMirror version.\n'
        if ! /usr/bin/lslpp -lcqOr cluster.es.server.rte 2>/dev/null | /usr/bin/cut -f3 -d":" | IFS=. read version release mod fix
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 13 'ERROR: PowerHA SystemMirror is not installed.\n'
            return ${RC.FAILURE}
        else
            if (( $version == 7 ))
            then
                if (( $release == 1 ))
                then
                    if (( $mod < 3 ))
                    then
                        DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 14 'ERROR: EZUpdate requires PowerHA SystemMirror version 7.1.3 or later.\n'
                        return ${RC.FAILURE}
                    fi
                elif (( $version < 1 ))
                then
                    DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 14 'ERROR: EZUpdate requires PowerHA SystemMirror version 7.1.3 or later.\n'
                    return ${RC.FAILURE}
                fi
            elif (( $version < 7 ))
            then
                DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 14 'ERROR: EZUpdate requires PowerHA SystemMirror version 7.1.3 or later.\n'
                return ${RC.FAILURE}
            fi
        fi
        DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 19 '    The installed PowerHA SystemMirror version is supported.\n\n'

        #
        : Get node list
        : Need a query here as the node list is not built so far
        #
        cmd="$CLMGR_CMD query nodes"
        log_trace 0 "$0()[$LINENO]($SECONDS): execute cmd: $cmd"
        cmd_output=$($cmd 2>&1)
        rc=$?
        log_trace 0 "$0()[$LINENO]($SECONDS): command returns code: $rc"
        log_trace 0 "$0()[$LINENO]($SECONDS): command output='$cmd_output'"
        if (( $rc != 0 ))
        then
            DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 4 'ERROR: command "%1$s" failed.\n' "$cmd"

            #Print the error output to specify the cause of the failure
	    #Printing only the command output which need not be translated so just printing in English
	    print -- "$cmd_output"
	    return ${RC.PHA_CMD_ERROR}
        fi
        node_list=$cmd_output

        #
        : Check clcomd communication
        #
        DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 15 'Checking for clcomd communication on all nodes...\n'
        for node in ${node_list}
        do
            #
            : Check outbound communication for $node
            #
            noOutbound=1
            for (( try=0; try<$MAX_TRIES; try++ ))
            do
                cmd="/usr/bin/hostname"
                _.clrsh_cmd output $node "$cmd"
                if (( $? == 0 ))
                then
                    noOutbound=0
                    break
                elif (( try < 4 ))
                then
                    sleep 1
                fi
            done
            if (( noOutbound ))
            then
                DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 16 'ERROR: Unable to verify outbound clcomd communication to node: %1$s\n' "${node}"
                return ${RC.FAILURE}
            fi

            #
            : Check inbound communication for $node
            #
            noInbound=1
            for (( try=0; try<$MAX_TRIES; try++ ))
            do
                $CLRSH -n $node "$CLRSH -n ${_.local_node_name} /usr/bin/hostname" >/dev/null 2>&1
                if (( $? == 0 ))
                then
                    # Inbound communication is working
                    noInbound=0
                    break
                elif (( try < 4 ))
                then
                    sleep 1
                fi
            done
            if (( noInbound ))
            then
                DSP_MSG ${MSG_TYPE.ERR} $CLUSTER_T_SET 17 'ERROR: Unable to verify inbound clcomd communication to node: %1$s\n' "${node}"
                return ${RC.FAILURE}
            fi
        done

        DSP_MSG ${MSG_TYPE.INF} $CLUSTER_T_SET 18 '    clcomd on each node can both send and receive messages.\n\n'
        return ${RC.OK}
    } # End of "check_rte()"
)


export CLUSTER_T_DEFINED=1