#!/bin/ksh93
#  ALTRAN_PROLOG_BEGIN_TAG
#  This is an automatically generated prolog.
#
#  Copyright (C) Altran ACT S.A.S. 2017,2019,2020,2021.  All rights reserved.
#
#  ALTRAN_PROLOG_END_TAG
#
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r714 src/43haes/lib/ksh93/hacmp/KLIB_HACMP_add_node.sh 1.21 
#  
# Licensed Materials - Property of IBM 
#  
# Restricted Materials of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2006,2013 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)  2650eea 43haes/lib/ksh93/hacmp/KLIB_HACMP_add_node.sh, 726, 2147A_aha726, Apr 14 2021 09:06 PM

#================================================
# The following, commented line enforces coding
# standards when this file is edited via vim.
#================================================
# vim:tabstop=4:shiftwidth=4:expandtab:smarttab
#================================================

# Start of POD-formatted documentation. Viewing suggestions:
#      perldoc <FILENAME>
#      pod2text -c <FILENAME>
#      pod2text -c --code <FILENAME>
#      pod2html <FILENAME>
function devDoc {
    : <<'=cut' >/dev/null 2>&1

=head1 NAME

 KLIB_HACMP_add_node

=head1 SYNOPSIS

 clmgr add node <node> \
             [ COMMPATH=<ip_address_or_network-resolvable_name> ] \
             [ RUN_DISCOVERY={true|false} ] \
             [ PERSISTENT_IP=<IP> NETWORK=<network>
               {NETMASK=<255.255.255.0 | PREFIX=1..128} ] \
             [ START_ON_BOOT={false|true} ] \
             [ BROADCAST_ON_START={true|false} ] \
             [ CLINFO_ON_START={false|true|consistent} ] \
             [ SITE=<sitename> ] \
             [ CRIT_DAEMON_RESTART_GRACE_PERIOD=<0..240> ]

 NOTE: the alias for "node" is "no".

=head1 DESCRIPTION

Attempts to add a new node within the cluster that conforms
to the provided specifications. Discovery is automatically
executed on the node after a successful creation unless
explicitly disabled.

=head1 ARGUMENTS

 1. properties [REQUIRED] [hash ref]
    An associative array within which data about the
    created object can be returned to the caller.

 2. nodename [REQUIRED] [string]
    The label to apply to the new node.

 3. commpath [REQUIRED] [string]
    This is the communication path to the node, and may be either an IP
    address, or a resolvable hostname. This value is only truly required
    when either the node name does not equal the hostname, or when there
    is more than one IP available on the node, and a specific address is
    desired.

 4. discovery [OPTIONAL] [string]
    A Boolean-like indicator of whether or not discovery should be run if
    the node is successfully created.

 5. persistentIP [OPTIONAL] [string]
    A persistent IP to add to the node.

 6. network [OPTIONAL] [string]
    A network to which the specified persistent IP (if any) should
    be added to.

 7. prefix [OPTIONAL] [string]
    The specified network's (if any) prefix length (IPv6).

 8. netmask [OPTIONAL] [string]
    The specified network's (if any) netmask (IPv4).

 9. restart [OPTIONAL] [string]
    A Boolean-like indicator of whether or not to start cluster
    services on this node automatically whenever the system itself
    is restarted.

    Valid values include: {false|true}

 10. broadcast [OPTIONAL] [string]
    A Boolean-like indicator of whether or not to broadcast cluster
    services startup.

    Valid values include: {true|false}

 11. clinfo [OPTIONAL] [string]
    A Boolean-like indicator of whether or not to start the CLINFO
    subsystem when cluster services start.

    Valid values include: {false|true|consistent}

 12. site [OPTIONAL] [string]
    Used to specify which site the node is being added to.
 
 13. crit_daemon_restart_grace_period [OPTIONAL] [integer]
    Sets the RSCT resource attribute IBM.PeerNode.CritDaemonRestartGracePeriod
    for tuning the response to loss of critical daemons in heavily loaded systems.
    Value is in seconds and the recommended value is less than 120 seconds.
    Default value is 0 and range is 0-240.

=head1 RETURN

 0: no errors were detected; the operation appears to have been successful
 1: a general error has occurred
 2: a specified resource does not exist, or could not be found
 3: some required input was missing
 4: some detected input was incorrect in some way
 5: a required dependency does not exist
 6: a specified search failed to match any data
 7: a given operation was not really necessary

=cut
} # End of POD-formatted documentation.


#============================================================================
#
# Name:        KLIB_HACMP_add_node
#
# Description: This is the main, FPATH function that is invoked by clmgr
#              to add a node to the cluster configuration. If the node is
#              being added to an existing, fully configured cluster (i.e.
#              the CAA cluster has been formed already), then a temporary
#              addition/removal of the node to/from the CAA cluster is
#              required in order to run discovery on the new node. Be
#              aware that this is a *temporary* process, so unless some
#              problem occurs, a cluster synchronization will still be
#              necessary to truly add the node to the cluster.
#
# Inputs:      refer to the "devDoc()" function, above.
#
# Outputs:     The properties hash is populated. The only other outputs are
#              any error messages that might be needed.
#
# Returns:     Zero if no errors are detected. Otherwise, an appropriate
#              non-zero value is returned. Refer to the "RETURN" section
#              of the "devDoc()" function, above, for the standard return
#              code values/meanings for clmgr.
#
#============================================================================
function KLIB_HACMP_add_node {
    . $HALIBROOT/log_entry "$0()" "$CL"
    : version=@(#)  2650eea 43haes/lib/ksh93/hacmp/KLIB_HACMP_add_node.sh, 726, 2147A_aha726, Apr 14 2021 09:06 PM
    : INPUTS: $*

    typeset -n properties=$1
    typeset    nodename=${2//\"/}
    typeset    commpath=${3//\"/}
    typeset -l discovery=${4//\"/}
    typeset    persistentIP=${5//\"/}
    typeset    network=${6//\"/}
    typeset    prefix=${7//\"/}
    typeset    netmask=${8//\"/}
    typeset -l restart=${9//\"/}
    typeset -l broadcast=${10//\"/}
    typeset -l clinfo=${11//\"/}
    typeset    site=${12//\"/}
    typeset    crit_daemon_restart_grace_period=${13//\"/}
    
    [[ $CLMGR_LOGGING == 'med' ]] && set +x  # Only trace param values

    #==================================================================
    : Since this function supports passing optional node modification
    : parameters directly to KLIB_HACMP_modify_node as "\$*", it is
    : imperative that we eliminate the locally consumed parameters.
    #==================================================================
    typeset -i IDX=9
    (( $# < IDX )) && IDX=$#
    for (( i=$IDX; i>0; i-- )); do
        shift
    done

    #===================================
    : Declare and initialize variables
    #===================================
    typeset -A disc
    typeset -i rc=$RC_UNKNOWN ALREADY_IN_CAA_CL=$RC_UNKNOWN

    #================================================================
    : Check for a defined cluster. No need to continue without one.
    #================================================================
    CL=$LINENO isClusterDefined
    if (( $? != RC_SUCCESS )); then
        log_return_msg "$RC_MISSING_DEPENDENCY" "$0()" "$LINENO"
        return $?
    fi

    #=================
    : Validate input
    #=================

    #====================================================================
    # Perform the following battery of tests and/or conversions for the
    # nodename and communication path:
    #
    #    1. if no nodename or commpath are provided, error out
    #
    #    2. if we get a commpath, but no nodename, try to resolve the
    #       commpath into a hostname, then use the short form of the
    #       hostname as the nodename.
    #
    #    3. if a nodename is provided, but no commpath, then the
    #       nodename *must* be resolvable. Test it.
    #
    #    4. if both a nodename and a commpath are provided, check
    #       that the commpath is resolvable. If nodename is also
    #       resolvable, and does not resolve the same as commpath,
    #       error out, since a typo has likely occurred.
    #
    #====================================================================
    ## TEST #1
    if [[ -z $nodename && -z $commpath ]]; then
        /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 100 "\nERROR: a name/label must be provided.\n\n" 1>&2
        rc=$RC_MISSING_INPUT
    elif (( ${#nodename} > $MAX_NAME_LENGTH )); then
        cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 1466 '\nERROR: Node name %1$s should not be more than %2$ld characters.\n' "$nodename" "$MAX_NAME_LENGTH" 1>&2
        rc=$RC_INCORRECT_INPUT
    fi

    if (( ! CLMGR_FORCE && $rc == RC_UNKNOWN )); then
        #========================================================
        : If a COMMPATH was provided, but no node name, attempt
        : to resolve the commpath into a hostname for the label
        #========================================================
        ## TEST #2
        if [[ -z $nodename && -n $commpath ]]
        then
            cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 291 '*** Warning: no label was provided for this node. An attempt will be made to use the provided communication path, "%1$s", to determine an appropriate node label automatically.\n' "$commpath"

            /usr/bin/host $commpath >$TMPDIR/clmgr.KHan.adnd.$$ 2>&1
            if (( $? == RC_SUCCESS )); then
                nodename=$(cat $TMPDIR/clmgr.KHan.adnd.$$)
                nodename=${nodename%% *}
                nodename=${nodename%%\.*}

                if [[ $nodename != *([[:space:]]) ]]; then
                    cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 292 '\nUsing a label of "%1$s" for this node (communication path "%2$s").\n' "$nodename" "$commpath"
                    print ""
                    sleep 3
                fi
            else
                cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 293 '\nERROR: "%1$s" could not be resolved via the /usr/bin/host command. Please make sure that "%1$s" is correctly defined in the /etc/hosts file.\n\n' "$commpath" 1>&2
                rc=$RC_INCORRECT_INPUT
            fi
            /usr/bin/rm -f $TMPDIR/clmgr.KHan.adnd.$$

        ## TEST #3
        elif [[ -n $nodename && -z $commpath ]]
        then
            #========================================================
            :   No commmunications path was specified by the user.
            :   This is acceptable if the given node name resolves
            :   to an IP address
            #========================================================
            if ! LC_ALL=C host $nodename 2>/dev/null | read FQDN is IP_addr rest
            then
                cl_dspmsg -s 58 cluster.cat 66 "No communications path was specified and node name \"$nodename\"\nwas not resolved by the \"host\" command.\nEither provide a communications path,\nor specify a node name that resolves to an IP address.\n" $nodename 1>&2
                rc=$RC_INCORRECT_INPUT
            else                      # else name resolves to an IP address
                commpath=${IP_addr%,} # so use that IP address as a commpath
            fi

        ## TEST #4
        else
            #========================================================
            :   A communications path has been proved by the user
            #========================================================
            if ! LC_ALL=C host $commpath 2>/dev/null | read node_FQDN is comm_IP_addr rest
            then
                cl_dspmsg -s 58 cluster.cat 67 "Communications path \"$commpath\" is not valid.\nIt cannot be resolved by the \"host\" command.\n" "$commpath" 1>&2
                rc=$RC_INCORRECT_INPUT
            else
                #========================================================
                :   We have a valid communications path.  Make sure that
                :   it is consistent with the given node name
                #========================================================
                if LC_ALL=C host $nodename 2>/dev/null | read node_FQDN is node_IP_addr rest
                then
                    #========================================================
                    :   Both node name and communications path resolve to IP
                    :   addresses.  They must be the same.
                    #========================================================
                    node_IP_addr=${node_IP_addr%,}
                    comm_IP_addr=${comm_IP_addr%,}
                    if [[ $node_IP_addr != $comm_IP_addr ]]
                    then
                        #================================================
                        :   Node name and communications path mismatch.
                        :   Set up an informative error message.
                        #================================================
                        if [[ $commpath == $comm_IP_addr ]]
                        then
                            /usr/bin/dspmsg -s 58 cluster.cat 68 "Node name \"$nodename\" which resolves to IP address \"$node_IP_addr\" is not consistent\nwith communications path \"$commpath\"\n" $nodename $node_IP_addr "$commpath" 1>&2
                            rc=$RC_INCORRECT_INPUT
                        else
                            /usr/bin/dspmsg -s 58 cluster.cat 69 "Node name \"$nodename\" which resolves to IP address \"$node_IP_addr\" is not consistent\nwith communications path \"$commpath\" which resolves to IP address \"$comm_IP_addr\"\n" "$nodename" "$node_IP_addr" "$commpath" "$comm_IP_addr" 1>&2
                            rc=$RC_INCORRECT_INPUT
                        fi
                    fi
                fi
            fi
        fi
    fi

    if [[ -n $nodename ]]; then
        CL=$LINENO KLIB_HACMP_is_known_node "$nodename" >/dev/null
        if (( $? == $RC_SUCCESS )); then 
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 229 "\nERROR: the specified object already exists: \"%1\$s\"\n\n" "$nodename" 1>&2
            rc=$RC_INCORRECT_INPUT
        fi
    fi

    if [[ -n $nodename && -n "${nodename//[!0-9]*/}" || -n "${nodename//[a-zA-Z0-9_\-]/}" ]]; then
        /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 188 "\nERROR: one or more invalid characters were detected in \"%1\$s\" (\"%2\$s\").\n\nValid characters include letters, numbers, underscores, and dashes only.\n\n" "$nodename" "${nodename//[a-zA-Z0-9_\-]/}" 1>&2
        rc=$RC_INCORRECT_INPUT
    fi

    if [[ -n $discovery && $discovery != @(t|f|y|n)* ]]; then
        /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 110 "\nERROR: invalid value specified for \"%1\$s\":  \"%2\$s\".\n" DISCOVERY "$discovery" 1>&2
        /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 3 "Valid values: %1\$s\n\n" "true, false" 1>&2
        rc=$RC_INCORRECT_INPUT
    fi

    if [[ -n $network ]]; then
        CL=$LINENO KLIB_HACMP_is_known_network "$network" >/dev/null 2>&1
        if (( $? != RC_SUCCESS )); then
            rc=$RC_INCORRECT_INPUT
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 102 "\nERROR: \"%1\$s\" does not appear to exist!\n\n" "$network" 1>&2

            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 150 "Available Networks:\n\n" 1>&2
            typeset available
            CL=$LINENO KLIB_HACMP_list_networks available
            for (( i=0; i<${#available[*]}; i++ )); do
                if [[ ${available[$i]} != *([[:space:]]) ]]; then
                    print -u2 "\t${available[$i]}"
                fi
            done
            print -u2 ""
        fi
    fi

    if [[ -n $netmask && -n $prefix ]]; then
        /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 36 "\nERROR: conflicting options were provided,\n\       \"%1\$s\" versus \"%2\$s\".\n\n" NETMASK PREFIX 1>&2
        rc=$RC_INCORRECT_INPUT
    fi

    if [[ $prefix != *([[:space:]]) ]] && \
         ( [[ $prefix != +([[:digit:]]) ]] || (( prefix < 1 || prefix > 128 )) )
    then
        /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 113 "\nERROR: an invalid IPv6 prefix length was specified: %1\$s\n\n" "$prefix" 1>&2
        /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 3 "Valid values: %1\$s\n\n" "1 .. 128" 1>&2
        rc=$RC_INCORRECT_INPUT
    fi

    if [[ -n $netmask ]]; then
        typeset nodots=${netmask//\./}
        if (( ${#netmask} - ${#nodots} != 3 )); then
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 115 "\nERROR: an invalid IPv4 netmask was specified: %1\$s\n\n" "$netmask" 1>&2
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 3 "Valid values: %1\$s\n\n" "<###.###.###.###>; \"###\" must be in the range 0 .. 255." 1>&2
            rc=$RC_INCORRECT_INPUT
        else
            typeset value=$netmask
            for (( i=0; i<4; i++ )); do
                typeset octet=${value%%.*}
                value=${value#*.}
                if [[ $octet == *([[:space:]]) || \
                      $octet != +([[:digit:]]) ]] || \
                   (( octet < 0 || octet > 255 ))
                then
                     /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 115 "\nERROR: an invalid IPv4 netmask was specified: %1\$s\n\n" "$netmask" 1>&2
                     /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 3 "Valid values: %1\$s\n\n" "<###.###.###.###>; \"###\" must be in the range 0 .. 255." 1>&2
                     rc=$RC_INCORRECT_INPUT
                     break
                fi
            done
        fi
    fi

    if [[ -n $restart ]]; then
        if [[ $restart != @(1|0|n|f|y|t)* ]]; then
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 110 "\nERROR: invalid value specified for \"%1\$s\":  \"%2\$s\".\n" START_ON_BOOT "$restart" 1>&2
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 3 "Valid values: %1\$s\n\n" "false, true" 1>&2
            rc=$RC_INCORRECT_INPUT
        fi
    fi

    if [[ -n $broadcast ]]; then
        if [[ $broadcast != @(1|0|n|f|y|t)* ]]; then
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 110 "\nERROR: invalid value specified for \"%1\$s\":  \"%2\$s\".\n" BROADCAST_ON_START "$broadcast" 1>&2
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 3 "Valid values: %1\$s\n\n" "true, false" 1>&2
            rc=$RC_INCORRECT_INPUT
        fi
    fi

    if [[ -n $clinfo ]]; then
        if [[ $clinfo != @(1|0|n|f|y|t|c)* ]]; then
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 110 "\nERROR: invalid value specified for \"%1\$s\":  \"%2\$s\".\n" CLINFO_ON_START "$clinfo" 1>&2
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 3 "Valid values: %1\$s\n\n" "false, true, consistent" 1>&2
            rc=$RC_INCORRECT_INPUT
        fi
    fi

    #
    :  Before validating critical daemon restart grace period, verifying the installed RSCT version
    :  supports crit_daemon_restart_grace_period attribute or not.
    # 
    if [[ -n $crit_daemon_restart_grace_period ]]; then
        typeset RSCT_SUPPORTED_VERSION="3.2.5.0"
        typeset RSCT_VERSION
        RSCT_VERSION=$(ctversion | cut -s -f 2 -d ' ')
        echo $RSCT_VERSION | IFS="." read V R M F
        if (( $V$R$M$F < 3250 ));then
            dspmsg -s 63 cluster.cat 80 "\nERROR: Installed RSCT verion \"%1\$s\" is not supported for CRIT_DAEMON_RESTART_GRACE_PERIOD attribute.\n\
Minimum RSCT version required to support CRIT_DAEMON_RESTART_GRACE_PERIOD is \"%2\$s\"" $RSCT_VERSION  $RSCT_SUPPORTED_VERSION 1>&2
            rc=$RC_INCORRECT_INPUT
        else
           typeset DESC="CRIT_DAEMON_RESTART_GRACE_PERIOD"
           [[ -n $CLMGR_GUI ]] && DESC=$(dspmsg -s 63 cluster.cat 78 "Critical Daemon Restart Grace Period")
           CL=$LINENO verify_numeric_range "$crit_daemon_restart_grace_period" 0 240 "$DESC"
           (( $? != RC_SUCCESS )) && rc=$RC_INCORRECT_INPUT
        fi
    fi

    #==========================================================
    : Cannot add more than one node at a time to caa
    #==========================================================
    if /usr/lib/cluster/incluster; then
        # CAA cluster is configured - figure out how many nodes
        # are known to CAA and PowerHA
        typeset -i pha_nodes=$(LC_ALL=C clnodename | wc -l)
        typeset -i caa_nodes=$(LC_ALL=C lscluster -m | grep "Node name:"  | wc -l)
        if (( pha_nodes > caa_nodes )); then
            # There are already more nodes in the PowerHA config
            # than there are configured in caa - cannot add another 
	    cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 1162 '\nERROR: There are already nodes configured in PowerHA that have not been\nadded to CAA. You can not add node "%1$s" \n until you verify and synchronize the cluster.\n' "$nodename" 1>&2
            rc=$RC_ERROR
        fi
    fi

    #========================================================
    : Create the node if no input errors have been detected
    #========================================================
    typeset repos_disk=""
    if (( $rc == RC_UNKNOWN )); then

        if [[ -n $commpath ]]; then
            #==================
            : Create the node
            #==================
            print -- "$0()[$LINENO]($SECONDS): clnodename -a \"$nodename\" -p \"$commpath\"" >>$CLMGR_TMPLOG  # Always log commands
            clnodename -a "$nodename" -p "$commpath"
            rc=$?
            print "clnodename RC: $rc" >>$CLMGR_TMPLOG  # Always log command result
            if (( $rc != RC_SUCCESS )); then
                rc=$RC_ERROR
                /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 201 "\nERROR: failed to create \"%1\$s\".\n\n" "$nodename" 1>&2

            elif [[ -z $(odmget -q "name=$nodename" HACMPnode) ]]; then
                rc=$RC_ERROR
                /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 201 "\nERROR: failed to create \"%1\$s\".\n\n" "$nodename" 1>&2

            else
                #==========================================================
                : Unless explicitly disabled, always try to run discovery
                : to retrieve information from the newly added node. This
                : will also update the information for any pre-existing
                : nodes.
                #==========================================================

                # Verify whether we are in cluster type conversion from standard to linked.
                # If yes, fetch new site name and repos disk from temporary site created in KLIB_HACMP_add_site.sh
                if [[ -z $site ]] &&
                   [[ $(clodmget -f multi_site_lc HACMPcluster) == 1 ]] &&
                   (( $(LC_ALL=C lscluster -c 2>/dev/null | grep "Multicast for site" | wc -l) == 1 ))
                then
                    site=$(clodmget -q "merge_priority=0" -n -f name HACMPsite)
                    repos_disk=$(clodmget -q "name=${site}_sircol" -n -f repository HACMPsircol)
                    # Force discovery
                    discovery=true
                fi

                if [[ $discovery != @(f|n)* ]]; then # Check for "false" or "no"
                    cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 294 'Attempting to enable communication with node "%1$s"...\n\n' "$nodename"

                    : Temporarily add the node to the CAA cluster
                    CL=$LINENO add_caa_access $commpath $site $repos_disk
                    rc=$?
                    if (( $rc == RC_SUCCESS || $rc == RC_EXTRANEOUS )); then
                        cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 295 'Communication with node "%1$s" is now enabled.\n\n' "$nodename"
                        print "$0()[$LINENO]($SECONDS): added temporary CAA access for node \"$nodename\" to enable clcomd, needed for discovery." >>$CLMGR_TMPLOG

                        ALREADY_IN_CAA_CL=0
                        if (( $rc == RC_EXTRANEOUS )); then
                            ALREADY_IN_CAA_CL=1
                            rc=$RC_SUCCESS
                        fi

                        if (( $rc == RC_SUCCESS )); then
                            cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 219 'Attempting to discover available resources on "%1$s"...\n' "$nodename"
                            CL=$LINENO KLIB_HACMP_run_discovery disc disc
                            rc=$?

                            if [[ -n $repos_disk ]]; then
                                # We are in cluster type conversion, fetch pvid and update in temporary site config
                                repos_pvid=$(cl_rsh $commpath "/usr/es/sbin/cluster/utilities/clodmget -q \"name=$repos_disk and attribute=pvid\" -n -f value CuAt" 2>>$CLMGR_TMPLOG)
                                repos_pvid=$(echo $repos_pvid | sed s/0000000000000000//g)
                                print "HACMPsircol: repository=$repos_pvid" | odmchange -q"name=${site}_sircol" -o HACMPsircol
                            fi
                        fi

                        #=====================================================
                        # Avoid removing the node from the CAA cluster if it
                        # was already in the cluster before this operation
                        # was initiated.
                        #=====================================================
                        if (( ! ALREADY_IN_CAA_CL )); then
                            : Make sure the node is no longer in the CAA cluster
                            if [[ -n $repos_disk ]]; then
                                CL=$LINENO rm_caa_access $commpath $site
                            else
                                CL=$LINENO rm_caa_access $commpath
                            fi
                        fi
                    else
                        cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 296 '\nERROR: failed to enable communication with node "%1$s".\n\n' "$nodename" 1>&2
                    fi
                fi
            fi

        #================================================================
        : Attempt to add the node to the cluster without specifying its
        : communication path. This approach will *not* run discovery.
        #================================================================
        else
            print -- "$0()[$LINENO]($SECONDS): clnodename -a \"$nodename\"" >>$CLMGR_TMPLOG  # Always log commands
            clnodename -a "$nodename"
            rc=$?
            print "clnodename RC: $rc" >>$CLMGR_TMPLOG  # Always log command result
            if (( $rc != RC_SUCCESS )); then
                /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 109 "\nERROR: the operation appears to have failed.\n\n" 1>&2
                rc=$RC_ERROR
            fi
        fi

        if (( $rc == RC_SUCCESS )); then
            #===========================================================
            : If output from this operation was requested, retrieve it
            #===========================================================
            if (( $rc == RC_SUCCESS )); then
                if (( CLMGR_VERBOSE )) || [[ -n $CLMGR_ATTRS ]]; then
                    CL=$LINENO KLIB_HACMP_get_node_attributes "$nodename" properties
                fi
            fi
        fi
    fi

    #===============================================================
    : If any cluster services settings were specified, make them.
    : Since this is a newly added node, however, and setting these
    : options must be done directly on the new node, it is first
    : necessary to sync the cluster, or the operation will fail
    # because the new node has not actually been created yet.
    #===============================================================
    if (( $rc == RC_SUCCESS )) &&
       [[ -n $restart || -n $broadcast || -n $clinfo ]]
    then
        CL=$LINENO KLIB_HACMP_run_discovery null null
        CL=$LINENO KLIB_HACMP_verify_and_sync true false true
        if (( $? == RC_SUCCESS )); then
            typeset opt_i= opt_R= opt_b= 
            if [[ -n $clinfo ]]; then
                case $clinfo in
                    @(1|y|t)*) opt_i=-i ;;
                        @(c)*) opt_i=-I ;;
                esac
            fi
            if [[ -n $restart ]]; then
                case $restart in
                    @(1|y|t)*) opt_R=-R ;;  # Automatically restart
                            *) opt_R=-N ;;  # No auto-restart
                esac
            fi
            if [[ -n $broadcast ]]; then
                case $broadcast in
                    @(1|y|t)*) opt_b=-b ;;  # Broadcast at startup
                            *) opt_b=   ;;  # Disable startup broadcast
                esac
            fi

            if [[ -z $LOCAL_NODE || $LOCAL_NODE == $nodename ]]; then
                print -- "$0()[$LINENO]($SECONDS): cl_auto_versync_options -s $opt_R $opt_b $opt_i" >>$CLMGR_TMPLOG  # Always log commands
                cl_auto_versync_options -s $opt_R $opt_b $opt_i 
                print "cl_auto_versync_options RC: $?" >>$CLMGR_TMPLOG  # Always log command result
            else
                print -- "$0()[$LINENO]($SECONDS): $CLRSH $nodename $HAUTILS/cl_auto_versync_options -s $opt_R $opt_b $opt_i" >>$CLMGR_TMPLOG  # Always log commands
                $CLRSH $nodename $HAUTILS/cl_auto_versync_options -s $opt_R $opt_b $opt_i 
                print "$(basename $CLRSH) cl_auto_versync_options RC: $?" >>$CLMGR_TMPLOG  # Always log command result
            fi
        else
            CL=$LINENO cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 244 "\nERROR: since the synchronization failed, the requested node startup option(s) cannot be established. Removing the incomplete node.\n\n" 1>&2
            rc=$RC_ERROR
        fi
    fi

    if (( $rc == RC_SUCCESS )) && [[ -n $site ]] && [[ -z $repos_disk ]]; then
        typeset NODELIST=$(clodmget -q "name=$site" -f nodelist -n HACMPsite)
        NODELIST="${NODELIST// /,},$nodename"

        typeset -A siteProps
        CL=$LINENO KLIB_HACMP_modify_site siteProps "$site" "" "$NODELIST"
        rc=$?
        unset siteProps NODELIST
    fi

    #==============================================
    : If a persistent IP was specified, create it
    #==============================================
    if (( $rc == RC_SUCCESS )); then
        if [[ -n $persistentIP ]]; then
            if [[ -n $network ]]; then
                typeset NETTYPE=$(/usr/bin/odmget -q "name = $network" HACMPnetwork | /usr/bin/grep "nimname = " | /usr/bin/cut -d'"' -f2)

                # Usage: claddnode [-n <name>] -a <adapter>:<net-type>:<net-name>:<attr>:<func>:[<ipaddr|device-file>]:[<hwaddr>]
                if [[ -n $prefix ]]; then
                    print -- "$0()[$LINENO]($SECONDS): claddnode -n \"$nodename\" -a $persistentIP:$NETTYPE:$network::persistent:::$prefix" >>$CLMGR_TMPLOG  # Always log commands
                    claddnode -n "$nodename" -a $persistentIP:$NETTYPE:$network::persistent:::$prefix
                else
                    print -- "$0()[$LINENO]($SECONDS): claddnode -n \"$nodename\" -a $persistentIP:$NETTYPE:$network::persistent:::$netmask" >>$CLMGR_TMPLOG  # Always log commands
                    claddnode -n "$nodename" -a $persistentIP:$NETTYPE:$network::persistent:::$netmask
                fi
                rc=$?
                print "claddnode RC: $rc" >>$CLMGR_TMPLOG  # Always log command result

            else
                /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 101 "\nERROR: this operation requires the \"%1\$s\" attribute.\n\n" NETWORK 1>&2
                rc=$RC_MISSING_INPUT
            fi

        elif [[ -n $network ]]; then
            /usr/bin/dspmsg -s $CLMGR_SET $CLMGR_MSGS 101 "\nERROR: this operation requires the \"%1\$s\" attribute.\n\n" PERSISTENT_IP 1>&2
            rc=$RC_MISSING_INPUT
        fi
    fi

    if [[ -n $crit_daemon_restart_grace_period ]] && (( $rc == RC_SUCCESS ));then
       typeset node_data
       node_data=$(clodmget -q "object=COMMUNICATION_PATH and name=$nodename" -f node_id,node_handle,version HACMPnode)
       print -- "$node_data" | IFS=: read id handle ver
       print "HACMPnode: name= $nodename\nobject = CRIT_DAEMON_RESTART_GRACE_PERIOD\nvalue=$crit_daemon_restart_grace_period\nnode_id=$id\nnode_handle=$handle\nversion=$ver" | odmadd 2>/dev/null;
       rc=$?
       if (( $rc != RC_SUCCESS ));then
          cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 1453 '\nERROR: An error occurred while configuring critical daemon restart grace period for node "%1$s".\n\n' "$nodename" 1>&2
       else
          cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 1454 "\nSuccessfully configured the node level RSCT critical daemon restart grace period for node %1\$s.\n" $nodename
       fi
    fi 

    if (( $rc == RC_ERROR )); then
        : Clean up any partially created/configured node
        cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 297 '\nERROR: failed to add node "%1$s" to the cluster.\n\n' "$nodename" 1>&2

        #=====================================================
        # Avoid removing the node from the CAA cluster if it
        # was already in the cluster before this operation
        # was initiated.
        #=====================================================
        if (( ALREADY_IN_CAA_CL == 1 )); then   # Must avoid 0 and -1
            : Clean up the CAA cluster
            if [[ -n $repos_disk ]]; then
                CL=$LINENO rm_caa_access $commpath $site >>$CLMGR_TMPLOG 2>&1
            else
                CL=$LINENO rm_caa_access $commpath >>$CLMGR_TMPLOG 2>&1
            fi
        fi

        : Clean up the SystemMirror cluster
        CL=$LINENO KLIB_HACMP_delete_node "$nodename" >>$CLMGR_TMPLOG 2>&1
    fi

    #=======================================================================
    : If a user input error was detected, provide some helpful suggestions
    #=======================================================================
    if (( $rc == RC_MISSING_INPUT || $rc == RC_INCORRECT_INPUT )) && \
       [[ $CLMGR_GUI == *([[:space:]]) ]]
    then
        CL=$LINENO cl_dspmsg -s $CLMGR_SET $CLMGR_MSGS 104 "For more information about available options and syntax, try\n\"$HAUTILS/clmgr %1\$s\". As an\nalternative, if the PowerHA SystemMirror man pages have been installed, invoke\n\"$HAUTILS/clmgr -hv\" (or \"/usr/bin/man clmgr\"),\nsearching for \"%2\$s\" in the displayed text.\n\n" \
        "add node -h" "NODE:" "$CLMGR_PROGNAME" 1>&2
    fi

    log_return_msg "$rc" "$0()" "$LINENO"
} # End of "KLIB_HACMP_add_node()"
