#  ALTRAN_PROLOG_BEGIN_TAG                                                    
#  This is an automatically generated prolog.                                  
#                                                                              
#  Copyright (C) Altran ACT S.A.S. 2018,2020,2021.  All rights reserved.  
#                                                                              
#  ALTRAN_PROLOG_END_TAG                                                      
#                                                                              
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r721 src/43haes/usr/sbin/cluster/sa/sap/sbin/SAPUtilities.sh 1.26 
#  
# Licensed Materials - Property of IBM 
#  
# Restricted Materials of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2008,2016 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
#
# @(#)  7d4c34b 43haes/usr/sbin/cluster/sa/sap/sbin/SAPUtilities.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM

. /usr/es/lib/ksh93/func_include
version='1.26'


#---------------------------------------------------------------------------
# Global Definitions
#---------------------------------------------------------------------------

. /usr/es/sbin/cluster/sa/sap/sbin/IO
. /usr/es/lib/ksh93/func_include
. /usr/es/sbin/cluster/sa/sap/etc/SAPGlobals

 typeset -i exe_check=1
 typeset -i valid_version=0
 typeset SID
 typeset SAPMNTDIR

#---------------------------------------------------------------------------



#----------------------------------------------------------------------------
# Function: 
#   setSAPGlobalEnv
#
# Purpose:
#   Sets SAP Gloabl Environment variables by finding SAPSYSTEMNAME.
#
# Arguments:
#   < SAPSYSTEMNAME > : If SAPSYSTEMNAME is passed as argument then it will be set directly
#   <-i Instance Name>: Search for SAPSYSTEMNAME from the profile for that instance   
#   < >               : If no Argument is passed then SAPSYSTEMNAME is greped from the first profile.
#
# Returns:
#   0 -> for Sucess
#   1 -> for Failure
#
#---------------------------------------------------------------------------

function setSAPGlobalEnv {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset startupProfiles
    typeset sapsystem
    typeset instance_name
    typeset Profile

    if [[ $1 == "-i" ]] 
    then 
        instance_name=$2
    elif [[ -n $1 ]]
    then
        SAPSYSTEMNAME=$1
    fi

    if [[ -z $SAPSYSTEMNAME ]]; then
        if [[ -f $SAPSERVICESFILE ]]; then
            getAllSAPInstanceStartupProfiles startupProfiles
                if (( ${#startupProfiles[*]} <= 0 )); then
                    log_msg "No startup profiles found in $SAPSERVICESFILE"
                    return 1
                fi
                if [[ ! -z $instance_name ]]; then 
                    for Profile in ${startupProfiles[*]} ; do
                        if echo "$Profile" 2>/dev/null | grep -q $instance_name
                        then
                            SAPSYSTEMNAME=$(echo "$Profile" 2>/dev/null | \
                                    awk -F"/" '{print $4}' 2>/dev/null)
                            break
                        fi
                    done
                else 
                    #Try finding the SAP System name from the first profile found.
                    SAPSYSTEMNAME=$( echo ${startupProfiles[0]} | awk -F"/" '{print $4}')
                fi
        else
            echo "$SAPSERVICESFILE doesn't exist. It's not a SAP Netweaver 2004s SR2 Installation."
            return 1
        fi
    fi
    SID=$SAPSYSTEMNAME
    SAPMNTDIR=/sapmnt/$SAPSYSTEMNAME
    SAPEXEDIR=/usr/sap/$SID/SYS/exe/run
    SAPPROFILEDIR=$SAPMNTDIR/profile
    SAPADMNUSR=$(echo $SAPSYSTEMNAME | tr '[:upper:]' '[:lower:]')adm
    SAPDBADMN=db2$(echo $SAPSYSTEMNAME | tr '[:upper:]' '[:lower:]')
    return 0; 
}


#----------------------------------------------------------------------------
# Function: 
#   getAllSAPInstanceStartupProfiles
#
# Purpose:
#   Gets All Instance profiles by reading /usr/sap/sapservices file. It expects
#    /usr/sap/sapservices file exists.
#
# Arguments:
#   (1) startupProfiles : A nameref for an array where startup profiles are
#   pushed into.
#
# Returns:
#   No. of Profiles
#----------------------------------------------------------------------------

function getAllSAPInstanceStartupProfiles {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset -n profiles=$1
    typeset ret=0
    typeset tempProfiles
    tempProfiles=$(awk -F";" '/pf=/ {print $3}' $SAPSERVICESFILE | \
                   awk '{print $2}' - | awk -F"=" '{print $2}' 2>/dev/null)
    i=0
    for p in $tempProfiles; do
        profiles[$i]=$p
        (( ret=$ret+1 ))
        (( i=$i+1 ))
    done
    return $ret
}

#----------------------------------------------------------------------------
# Function:
#   getInstanceNameFromProfile
#
# Purpose:
#   Gets the instancename for a given path to a profile. It needs to be called
#   after SAP env is setup by calling setSAPGlobalEnv
#
# Arguments:
#   (1) profile path.
#
# Returns:
#   Instance Name
#----------------------------------------------------------------------------

function getInstanceNameFromProfile {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset profilePath=$1
    typeset instanceName
    instanceName=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C  \
                   $SAPEXEDIR/sappfpar pf=$profilePath INSTANCE_NAME" 2>/dev/null)
    echo $instanceName
    return 0
}

#----------------------------------------------------------------------------
# Function:
#   getAllRunningASInstances
#
# Purpose:
#   Gets the All the SAP instances of Application Instance type.
#   It needs to be called after SAP env is setup by calling setSAPGlobalEnv
#
# Arguments:
#   (1) A nameref for an array where instance names are pushed into.
#
# Returns:
#   All Valid Application Instances names are found from /usr/sap/sapservices file
#----------------------------------------------------------------------------

function getAllRunningASInstances {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset -n allInstances=$1
    typeset allProfiles
    typeset INSTANCE ret
    getAllSAPInstanceStartupProfiles allProfiles
    i=0
    for p in "${!allProfiles[@]}"; do
        INSTANCE=$(getInstanceNameFromProfile ${allProfiles[$p]})
        if [[ $INSTANCE == D* || $INSTANCE == J* ]]; then
            #Just to do some inspections of the found instance.
            if [ -f ${allProfiles[$p]} ]; then
                checkIsASInstanceRunningLocally ${allProfiles[$p]}
                ret=$?
                (( ret == 0 )) && allInstances[$i]=$INSTANCE
                (( i=$i+1 ))
            fi
        fi
    done
}


#----------------------------------------------------------------------------
# Function:
#   getAllRunningSCSInstances
#
# Purpose:
#   Gets the All the SAP instances of Central Services Instance type.
#   It needs to be called after SAP env is setup by calling setSAPGlobalEnv
#
# Arguments:
#   (1) A nameref for an array where instance names are pushed into.
#
# Returns:
#   All Valid SCS Instances names are found from /usr/sap/sapservices file
#----------------------------------------------------------------------------

function getAllRunningSCSInstances {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset -n allInstances=$1
    typeset allProfiles
    typeset INSTANCE ret
    getAllSAPInstanceStartupProfiles allProfiles
    i=0
    for p in "${!allProfiles[@]}"; do
        INSTANCE=$(getInstanceNameFromProfile ${allProfiles[$p]})
        if [[ $INSTANCE == ASCS* || $INSTANCE == SCS* ]]; then
            #Just to do some inspections of the found instance.
            if [ -f ${allProfiles[$p]} ]; then
                checkIsSCSInstanceRunningLocally ${allProfiles[$p]}
                ret=$?
                (( ret == 0 )) && allInstances[$i]=$INSTANCE
                (( i=$i+1 ))
            fi
        fi
    done

}

#----------------------------------------------------------------------------
# Function:
#   getAllRunningERSInstances
#
# Purpose:
#   Gets the All the SAP instances of Enqueue Replication Server Instance type.
#   It needs to be called after SAP env is setup by calling setSAPGlobalEnv
#
# Arguments:
#   (1) A nameref for an array where instance names are pushed into.
#
# Returns:
#   All Valid ERS Instances names are found from /usr/sap/sapservices file
#----------------------------------------------------------------------------

function getAllRunningERSInstances {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x

    typeset -n allInstances=$1
    typeset allProfiles
    typeset INSTANCE
    getAllSAPInstanceStartupProfiles allProfiles
    i=0
    for p in "${!allProfiles[@]}"; do
        INSTANCE=$(getInstanceNameFromProfile ${allProfiles[$p]})
        if [[ $INSTANCE == ERS* ]]; then
            #Just to do some inspections of the found instance.
            if [ -f ${allProfiles[$p]} ]; then
                checkIsERSInstanceRunningLocally ${allProfiles[$p]}
                ret=$?
                (( ret == 0 )) && allInstances[$i]=$INSTANCE
                (( i++ ))
            fi
        fi
    done
}

#----------------------------------------------------------------------------
# Function:
#   checkIsSAPVersionOK
#
# Purpose:
#   Checks if we running SAP Netweaver version 2004s (aka SAP NW 7.0) or greater.
#   It needs to be called after SAP env is setup by calling setSAPGlobalEnv
#
# Arguments:
#   None.
#
# Returns:
#   0 -> For yes
#   1 -> For No
#----------------------------------------------------------------------------

function checkIsSAPVersionOK {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x

    typeset version
    
    if [[ -f $SAPEXEDIR/disp+work ]]; then
       exe_check=0
        version=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C  \
                 $SAPEXEDIR/disp+work -version"|awk '/kernel release/ { print $3 }')
        if (( $version < 720 )); then
           valid_version=1
           log_msg "Will exit Version for $SID is less than 720 which is not supported"
           return 1
        fi
        return 0
     else
        return 1
    fi
}

#----------------------------------------------------------------------------
# Function:
#   checkIsASInstanceRunningLocally
#
# Purpose:
#   Checks if passed SAP Application instance is running locally or not. 
#   
#
# Arguments:
#   1. path to profile
#
# Returns:
#   0 -> For yes
#   1 -> For No
#----------------------------------------------------------------------------

function checkIsASInstanceRunningLocally {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x

    # Check first, if we have a running sapstartsrv process for this instance.
    # If not, we can't use sapcontrol to know the status.
    typeset profile=$1
    typeset host instanceNo ret processDisplayStatus pids pid tempvar
    typeset -i count
    typeset -A allRunningProcess
    typeset tempOut
    TMPFILE=/tmp/env.$$

    checkForRunningSapstartsrv $profile
    ret=$?
    if (( $ret != 0 )); then
  
       instanceNo=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C  \
            $SAPEXEDIR/sappfpar pf=$profile SAPSYSTEM" 2>/dev/null)
       [[ -z $instanceNo ]] && return 1
       tempvar=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C \
                 sapcontrol -nr  $instanceNo  -function GetVersionInfo" 2>/dev/null)
        ret=$?
        if (( $ret != 0 )); then
          return 1
        else
          return 0
        fi
    else
        host=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C  \
               $SAPEXEDIR/sappfpar pf=$profile SAPLOCALHOST" 2>/dev/null)
        [[ -z $host ]] && return 1
        instanceNo=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C  \
                    $SAPEXEDIR/sappfpar pf=$profile SAPSYSTEM" 2>/dev/null)
        [[ -z $instanceNo ]] && return 1
        /usr/bin/su - $SAPADMNUSR -c "env LANG=C \
         sapcontrol -host $host -nr $instanceNo -function GetProcessList \
         -format script" > $TMPFILE 2>/dev/null
        pids=$(cat $TMPFILE | grep "[0-9] "|awk -F":" '/pid:/ {print $2}')
        rm -f $TMPFILE
        #To create a associate array
        tempOut=$(echo "(";ps -aeo pid,user | \
                  awk '{printf "[";printf $1;printf "]=";printf $2;printf " "}';echo ")")
        allRunningProcess=$(echo $tempOut)
        #check if any one of the process from sapcontrol is running here and
        #owned by SAPADMNUSR
        pid=0
        for pid in "${!pids[@]}"
        do
            [[ ${allRunningProcess[${pids[$pid]}]} != $SAPADMNUSR ]] && return 0
        done
    fi
    return 1
}

#----------------------------------------------------------------------------
# Function:
#   checkIsSCSInstanceRunningLocally
#
# Purpose:
#   Checks if passed SAP SCS instance is running locally or not.
#
# Arguments:
#   None.
#
# Returns:
#   0 -> For yes
#   1 -> For No
#----------------------------------------------------------------------------

function checkIsSCSInstanceRunningLocally {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x

    # Check first, if we have a running sapstartsrv process for this instance.
    # If not, we can't use sapcontrol to know the status.
    typeset profile=$1
    typeset host instanceNo ret processDisplayStatus pids pid tempvar
    typeset -i count
    typeset -A allRunningProcess
    typeset tempOut
    TMPFILE=/tmp/env.$$

    checkForRunningSapstartsrv $profile
    ret=$?
    if (( $ret != 0 )); then

       instanceNo=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C  \
            $SAPEXEDIR/sappfpar pf=$profile SAPSYSTEM" 2>/dev/null)
       [[ -z $instanceNo ]] && return 1

       tempvar=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C \
                 sapcontrol -nr  $instanceNo  -function GetVersionInfo" 2>/dev/null)
        ret=$?
        if (( $ret != 0 )); then
          return 1
        else
          return 0
        fi
    else
        host=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C \
               $SAPEXEDIR/sappfpar pf=$profile SAPLOCALHOST" 2>/dev/null)
        [[ -z $host ]] && return 1
        instanceNo=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C  \
                     $SAPEXEDIR/sappfpar pf=$profile SAPSYSTEM" 2>/dev/null)
        [[ -z $instanceNo ]] && return 1
        /usr/bin/su - $SAPADMNUSR -c "env LANG=C \
                           sapcontrol -host $host -nr $instanceNo \
                           -function GetProcessList -format script" > $TMPFILE 2>/dev/null
        pids=$(cat $TMPFILE | grep "[0-9] "|awk -F":" '/pid:/ {print $2}')
        rm -f $TMPFILE
        #To create a associate array
        tempOut=$(echo "(";ps -aeo pid,user | \
                  awk '{printf "[";printf $1;printf "]=";printf $2;printf " "}';echo ")")
        allRunningProcess=$(echo $tempOut)
        #check if any one of the process from sapcontrol is running here and
        #owned by SAPADMNUSR
        pid=0
        for pid in "${!pids[@]}"
        do
            [[ ${allRunningProcess[${pids[$pid]}]} != $SAPADMNUSR ]] && return 0
        done
    fi
    return 1

}
 
#----------------------------------------------------------------------------
# Function:
#   checkIsERSInstanceRunningLocally
#
# Purpose:
#   Checks if passed SAP ERS instance is running locally or not.
#
# Arguments:
#   None.
#
# Returns:
#   0 -> For yes
#   1 -> For No
#----------------------------------------------------------------------------

function checkIsERSInstanceRunningLocally {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x

    # Check first, if we have a running sapstartsrv process for this instance.
    # If not, we can't use sapcontrol to know the status.
    typeset profile=$1
    typeset host instanceNo ret processDisplayStatus pids pid tempvar
    typeset -i count
    typeset -A allRunningProcess
    typeset tempOut
    TMPFILE=/tmp/env.$$

    checkForRunningSapstartsrv $profile
    ret=$?
    if (( $ret != 0 )); then

       instanceNo=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C  \
            $SAPEXEDIR/sappfpar pf=$profile SAPSYSTEM" 2>/dev/null)
       [[ -z $instanceNo ]] && return 1

        tempvar=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C \
                 sapcontrol -nr  $instanceNo  -function GetVersionInfo" 2>/dev/null)
        ret=$?
        if (( $ret != 0 )); then
           return 1
        else
           return 0
        fi
    else
	 host=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C \
               $SAPEXEDIR/sappfpar pf=$profile SAPLOCALHOST" 2>/dev/null)
        [[ -z $host ]] && return 1
        instanceNo=$(/usr/bin/su - $SAPADMNUSR -c "env LANG=C  $SAPEXEDIR/sappfpar pf=$profile SAPSYSTEM" 2>/dev/null)
        [[ -z $instanceNo ]] && return 1
        /usr/bin/su - $SAPADMNUSR -c "env LANG=C \
                            sapcontrol -host $host -nr $instanceNo \
                            -function GetProcessList -format script" > $TMPFILE 2>/dev/null
        pids=$(cat $TMPFILE | grep "[0-9] "|awk -F":" '/pid:/ {print $2}')
        rm -f $TMPFILE
        #To create a associate array
        tempOut=$(echo "(";ps -aeo pid,user | \
                  awk '{printf "[";printf $1;printf "]=";printf $2;printf " "}';echo ")")
        allRunningProcess=$(echo $tempOut)
        #check if any one of the process from sapcontrol is running here and
        #owned by SAPADMNUSR
        pid=0
        for pid in "${!pids[@]}"
        do
            [[ ${allRunningProcess[${pids[$pid]}]} != $SAPADMNUSR ]] && return 0
        done
    fi
    return 1

}

#----------------------------------------------------------------------------
# Function:
#   checkForRunningSapstartsrv
#
# Purpose:
#   Checks if we have a sapstartsrv process running with a given profile
#
# Arguments:
#   None.
#
# Returns:
#   0 -> For yes
#   1 -> For No
#----------------------------------------------------------------------------

function checkForRunningSapstartsrv {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset profile=$1
    typeset ret
    profile=${profile//\//\\/}
    ps -aef | grep sapstartsrv | grep -q $profile > /dev/null
    ret=$?
    return $ret
}

#----------------------------------------------------------------------------
# Function:
#   isInTheList
#
# Purpose:
#   function to check if the value is in the list
#
# Arguments:
#   1. Array by ref
#   2. Value to be searched in the list
#
# Returns:
#   0 -> For yes
#   1 -> For No
#----------------------------------------------------------------------------

function isInTheList
{
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset -n array=$1
    value=$2

    for element in ${array[@]}; do
        [[ "$element" == "$value" ]] && return 0
    done
    return 1
}

#----------------------------------------------------------------------------
# Function:
#   getResource
#
# Purpose:
#   Get the value of a particular resource type given the application name
#
# Arguments:
#   (1) Application Name
#   (2) Resource type (i.e. SERVICE_LABEL)
#
# Output:
#   List of resources defined to HACMP of the specified type
#
# Return:
#   0 on success
#   1 on failure
#----------------------------------------------------------------------------

function getResource {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset appid=$1
    typeset rName=$2
    typeset RGS=$(getResourceGroups $appid)

    for rg in $RGS; do
        clmgr query resource_group $rg | grep $rName | \
             awk -F= '{ print $2 }' | sed -e "s/\"//g"
    done
}
#----------------------------------------------------------------------------
# Function:
#   getResourceGroups
#
# Purpose:
#   Get a list of HACMP resource groups for the specified application name
#
# Arguments:
#   (1) Application Name
#
# Output:
#   List of HACMP resource groups
#
# Returns:
#   0 on success
#   1 on failure
#----------------------------------------------------------------------------

function getResourceGroups {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset appid=$1
    typeset name value

    odmget -q "sa_key=$appid" HACMPgroup | while IFS='=' read name value
    do
        name=$(echo $name)
        [[ "$name" == "group" ]] && {
            value=${value//\"/}
            echo $(echo $value)
        }
    done
}

#----------------------------------------------------------------------------
# Function:
#   getMetadata
#
# Purpose:
#   Get HACMPsa_metadata for a particular HACMP application, name name/value
#   pair.
#
# Arguments:
#   (1) Application Name
#   (2) Name of the name/value pair
#
# Output:
#   Value(s) of the name/value pair
#
# Return:
#   0 on success
#   1 on failure
#----------------------------------------------------------------------------

function getMetadata {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset appid=$1
    typeset name=$2

    clquerysaapp -a $appid | grep $name | awk -F= '{ print $2 }'
}

#----------------------------------------------------------------------------
# Function:
#   getNodes
#
# Purpose:
#   Get a list of nodes from resource groups that belong to the application
#
# Arguments:
#   (1) Application Name
#
# Output:
#   List of nodes, list of nodes is ordered based on appearance in
#   participating node list.
#
# Returns:
#   0 on success
#   1 on failure
#----------------------------------------------------------------------------

function getNodes {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset appid=$1
    typeset arrNodes
    typeset RGS=$(getResourceGroups $appid)

    #
    # Get a list of nodes for the application, need to preserve the order
    # in which the nodes are presented in the resource group list(s)
    #
    for rg in $RGS; do
        set -a
        eval $(clmgr query resource_group $rg | grep NODES)
        set +a

        for nodeA in $NODES; do
            found=0
            for nodeB in $arrNodes; do
                [[ "$nodeA" == "$nodeB" ]] &&
                    found=1
            done
            (( $found == 0 )) &&
                arrNodes="$arrNodes $nodeA"
        done
    done

    echo $arrNodes
}

#----------------------------------------------------------------------------
# Function:
#   stopsapstartsrv
#
# Purpose:
#   Stops sapstartsrv process running with a given profile
#
# Arguments:
#   None.
#
# Returns:
#   0 -> for success
#   non zero -> for failure
#----------------------------------------------------------------------------

function stopsapstartsrv {
    [[ $VERBOSE_LOGGING == "high" ]] && set -x
    typeset profile=$1
    typeset ret pid
    profile=${profile//\//\\/}
    pid=$(ps -aef | grep sapstartsrv | grep $profile | awk '{print $2}')
    kill -9 $pid
    return $?
}

#----------------------------------------------------------------------------
# Function:
#   check_sapservices
#
# Purpose:
#   Verfies whether all the cluster nodes contain /usr/sap/sapservices
#   file and if contains checks if it is same on all nodes
#
# Arguments:
#   None.
#
# Returns:
#   None
#----------------------------------------------------------------------------

function check_sapservices {
      typeset nodes=$(clmgr q nodes)
      typeset localnode=$(/usr/es/sbin/cluster/utilities/get_local_nodename 2>/dev/null)
      for node in $nodes
      do
            typeset sapservices_output
            sapservices_output=$(/usr/es/sbin/cluster/utilities/cl_rsh -n $node "ls -l /usr/sap/sapservices")
            if [[ -z $sapservices_output ]];then
                KLIB_SAP_SA_logmsg ERROR 120 27 sapsa.cat "/usr/sap/sapservices is not present in %s " $node
                exit 1
            else
               if [[ $node == $localnode ]];then
                  continue
               else
                  /usr/es/sbin/cluster/utilities/cl_rcp $node:/usr/sap/sapservices /tmp/sapservices.$node
                  if ! /usr/bin/diff /usr/sap/sapservices /tmp/sapservices.$node  >/dev/null 2>&1
                  then
                      KLIB_SAP_SA_logmsg ERROR 120 28 sapsa.cat "/usr/sap/sapservices is not same on nodes %1$s and %2$s \n" $localnode $node
                      exit 1
                  fi
                fi
             fi
       done
}