#!/bin/ksh93
#  ALTRAN_PROLOG_BEGIN_TAG
#  This is an automatically generated prolog.
#
#  Copyright (C) Altran ACT S.A.S. 2017,2018,2019,2021.  All rights reserved.
#
#  ALTRAN_PROLOG_END_TAG
#
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r714 src/43haes/lib/ksh93/hacmp/KLIB_HACMP_recover_node.sh 1.4 
#  
# Licensed Materials - Property of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2009,2010 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG
# @(#)  7d4c34b 43haes/lib/ksh93/hacmp/KLIB_HACMP_recover_node.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM

# Start of POD-formatted documentation. Viewing suggestions:
#      perldoc <FILENAME>
#      pod2text -c <FILENAME>
#      pod2text -c --code <FILENAME>
#      pod2html <FILENAME>
function devDoc {
    : <<'=cut' >/dev/null 2>&1

=head1 NAME

 KLIB_HACMP_recover_node

=head1 SYNOPSIS

 clmgr recover node <node>[,<node#2>,<node#n>,...] \
           [ CANCEL_EVENT={false|true} ]

 NOTE: the alias for "node" is "no".

=head1 DESCRIPTION

Attempts to recover from an event failure on the indicated node(s),
defaulting to the local node.

=head1 ARGUMENTS

 1. properties [REQUIRED] [hash ref]
    An associative array within which data about the
    created object can be returned to the caller.

 2. nodelist [OPTIONAL] [string]
    The list of nodes that will be recovered.

 3. cancel event [OPTIONAL] [string]
    A Boolean-like value indicates whether the administrator wants to
    cancel or continue with the remaining event processing.
    Valid values include: {false|true}

=head1 RETURN

 0: no errors were detected; the operation appears to have been successful
 1: a general error has occurred
 2: a specified resource does not exist, or could not be found
 3: some required input was missing
 4: some detected input was incorrect in some way
 5: a required dependency does not exist
 6: a specified search failed to match any data

=cut
} # End of POD-formatted documentation.


function KLIB_HACMP_recover_node {
    . $HALIBROOT/log_entry "$0()" "$CL"
    : version=@(#)  7d4c34b 43haes/lib/ksh93/hacmp/KLIB_HACMP_recover_node.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM
    : INPUTS: $*

    typeset nodelist=${1//\"/}
            nodelist=${nodelist#+([[:space:]])}
            nodelist=${nodelist//,/ }
    typeset -l cancel=${2//\"/}

    [[ $CLMGR_LOGGING == 'med' ]] && set +x  # Only trace param values

    #===================================
    : Declare and initialize variables
    #===================================
    typeset -i rc=$RC_UNKNOWN
    typeset -i i=0
    typeset node=
    typeset runcmd_flag=

    #================================================================
    : Check for a defined cluster. No need to continue without one.
    #================================================================
    CL=$LINENO isClusterDefined
    if (( $? != RC_SUCCESS )); then
        rc=$RC_MISSING_DEPENDENCY

    #=================
    : Validate input
    #=================
    elif [[ $nodelist == *([[:space:]]) ]]; then
        nodelist=$LOCAL_NODE
        if [[ $nodelist == *([[:space:]]) ]]; then
            dspmsg -s $CLVT_SET $CLVT_MSGS 602 '\nERROR: no node could be identified on "%1$s".\n\n' "$(hostname)" 1>&2
            rc=$RC_NOT_FOUND
        fi

    else
        for node in $nodelist; do
            node=$(CL=$LINENO KLIB_HACMP_is_known_node "$node")
            if (( $? != RC_SUCCESS )); then
                dspmsg -s $CLVT_SET $CLVT_MSGS 102 "\nERROR: \"%1\$s\" does not appear to exist!\n\n" "$node" 1>&2
                rc=$RC_NOT_FOUND
            fi
        done
    fi

    #=========================================================
    : Validate the optional recovery mode
    #=========================================================
    if [[ -n $cancel ]]; then
        if [[ $cancel != @(1|0|n|f|y|t)* ]]; then
            dspmsg -s $CLMGR_SET $CLMGR_MSGS 110 "\nERROR: invalid value specified for \"%1\$s\":  \"%2\$s\".\n" CONTINUE_EVENT "$cancel" 1>&2
            dspmsg -s $CLMGR_SET $CLMGR_MSGS 3 "Valid values: %1\$s\n\n" "false, true" 1>&2
            rc=$RC_INCORRECT_INPUT
        elif [[ $cancel == @(1|y|t)* ]]; then
            runcmd_flag="-c"
        fi
    fi

    #=========================================================
    : Recover the node if no input errors have been detected
    #=========================================================
    typeset -i recoveries=0
    if (( $rc == RC_UNKNOWN )); then
        for node in $nodelist; do
            typeset state=$(CL=$LINENO KLIB_HACMP_get_node_state "$node")
            if [[ $state == *FAILED* ]]; then
                dspmsg -s $CLVT_SET $CLVT_MSGS 604 "Attempting to recover from event failures on \"%1\$s\"...\n" "$node"
                print "$0()[$LINENO]($SECONDS): clruncmd \"$node\" $runcmd_flag" >>$CLMGR_TMPLOG  # Always log commands
                clruncmd "$node" $runcmd_flag
                rc=$?
                print "clruncmd RC: $rc" >>$CLMGR_TMPLOG  # Always log command result
                if (( $rc != RC_SUCCESS )); then
                    dspmsg -s $CLVT_SET $CLVT_MSGS 605 "\nERROR: failed to recover from script failure(s) on node \"%1\$s\".\n\n" "$node" 1>&2
                    rc=$RC_ERROR
                else
                    (( recoveries++ ))
                fi
            else
                dspmsg -s $CLVT_SET $CLVT_MSGS 606 "Node \"%1\$s\" is not in a failed state at the moment; no recovery needed.\n" "$node"
                rc=$RC_SUCCESS
            fi
        done

    elif (( $rc == RC_NOT_FOUND )); then
        dspmsg -s $CLVT_SET $CLVT_MSGS 102 "\nERROR: \"%1\$s\" does not appear to exist!\n\n" "$node" 1>&2
        dspmsg -s $CLVT_SET $CLVT_MSGS 151 "Available Nodes:\n\n" 1>&2

        typeset available
        CL=$LINENO KLIB_HACMP_list_nodes available
        for (( i=0; i<${#available[*]}; i++ )); do
            if [[ ${available[$i]} != *([[:space:]]) ]]; then
                print -u2 "\t${available[$i]}"
            fi
        done
        print -u2 ""
    fi

    #=======================================================================
    : If a user input error was detected, provide some helpful suggestions
    #=======================================================================
    if (( $rc == RC_MISSING_INPUT || $rc == RC_INCORRECT_INPUT )) && \
       [[ $CLVT_GUI == *([[:space:]]) ]]
    then
        CL=$LINENO cl_dspmsg -s $CLVT_SET $CLVT_MSGS 104 "For more information about available options and syntax, try\n\"$HAUTILS/clmgr %1\$s\". As an\nalternative, if the PowerHA SystemMirror man pages have been installed, invoke\n\"$HAUTILS/clmgr -hv\" (or \"/usr/bin/man clmgr\"),\nsearching for \"%2\$s\" in the displayed  ext.\n\n" \
        "recover node -h" "NODE:" "$CLVT_PROGNAME" 1>&2

    elif (( $rc == RC_ERROR )); then
        dspmsg -s $CLVT_SET $CLVT_MSGS 601 "\nERROR: the recovery effort has failed.\n\n" 1>&2

    elif (( $rc == RC_SUCCESS )); then
        if (( recoveries > 0 )); then
            dspmsg -s $CLVT_SET $CLVT_MSGS 607 "Any current event script failures have been recovered from.\nWarning: more may occur if the underlying problem has not been corrected.\n"
        else
            dspmsg -s $CLVT_SET $CLVT_MSGS 608 "At the moment, there are no known event script failures\non the following node(s):  %1\$s\n" "${nodelist// /, }"
        fi
    fi

    log_return_msg "$rc" "$0()" "$LINENO"
    return $?
} # End of "KLIB_HACMP_recover_node()"
#================================================
# The following, commented line enforces coding
# standards when this file is edited via vim.
#================================================
# vim:tabstop=4:shiftwidth=4:expandtab:smarttab
#================================================