#!/usr/bin/ksh93 # ALTRAN_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # Copyright (C) Altran ACT S.A.S. 2017,2018,2021. All rights reserved. # # ALTRAN_PROLOG_END_TAG # # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # 61haes_r721 src/43haes/usr/sbin/cluster/apps/clam_nfsv4/monitor.sh 1.10.1.5 # # Licensed Materials - Property of IBM # # COPYRIGHT International Business Machines Corp. 2007,2016 # All Rights Reserved # # US Government Users Restricted Rights - Use, duplication or # disclosure restricted by GSA ADP Schedule Contract with IBM Corp. # # IBM_PROLOG_END_TAG # @(#) 6cb0ffc 43haes/usr/sbin/cluster/apps/clam_nfsv4/monitor.sh, 726, 2147A_aha726, Feb 15 2021 04:06 PM ###################################### ### Functions ###################################### ############################################################################## # Function: log( ) # # Description: Log messages unconditionally with a timestamp. # # Notes: log( ) writes to stderr so that commands that are called # using $( ) indirection can still call log without the output # being consumed. # # Arguments: The message to print. # # Result: Success (0) ############################################################################## function log { print -- "$(date '+%Y/%m/%d %H:%M:%S') $PROGNAME: $@" 1>&2 } ############################################################################## # Function: trace( ) # # Description: Log debug messages conditionally. This is primarily intended # for cases where some information is needed, but setting # VERBOSE_LOGGING is too verbose. # # Arguments: The message to trace. # # Result: Success if tracing is disabled, otherwise the result of the # log operation. ############################################################################## function trace { [[ -z $DEBUG_CLAM_NFSV4 ]] && return 0 log "$@" } ############################################################################## # Function: run( ) # # Description: This is a wrapper for running a command. It does the # following: # 1. logs that the command will be run # 2. runs the command # 3. captures the result code and logs it. # 4. returns the result. # # Arguments: The command line to execute. # # Result: The result code returned by the command. ############################################################################## run( ) { log "Running $@ ..." "$@" result=$? if (( $result == 0 )) ; then log "$@ succeeded" else log "$@ failed with error code $result" fi return $result } ############################################################################## # Function: dbgrun( ) # # Description: Run a command conditionally. This is primarily intended # for debugging. # # Arguments: The command to run. # # Result: Success if tracing is disabled, otherwise the result of the # command. ############################################################################## dbgrun( ) { [[ -z DEBUG_CLAM_NFSV4 ]] && return 0 run "$@" } ############################################################################## # Function: member_of( ) # # Syntax: member_of item [list ...] # # Description: Checks if the first argument is a member of the space i # separated list consisting of the remaining parameters. # # Arguments: item: The item to search for # list: The list to search # # Result: Returns true (0) if the value is found. # Returns false (1) if the value is not found. ############################################################################## function member_of { typeset value1=$1 shift if [[ $@ == @(?(* )${value1}?( *)) ]] then return 0 fi return 1 } ############################################################################## # Function: success( ) # # Syntax: success # # Description: Exits the application monitor indicating success. # # Result: Exits with success (0) ############################################################################## function success { log "Monitor succeeded" exit 0 } ############################################################################## # Function: fatal( ) # # Syntax: fatal [message to print] # # Description: Exits the application monitor indicating failure. # # Result: Exits with an error (1) ############################################################################## function fatal { if (( $# > 0 )) ; then log "Monitor failed: $@" else log "Monitor failed" fi exit 1 } ############################################################################## # Function: check_if_running( ) # # Syntax: check_if_running subsystem # # Description: Checks if the subsystem is running. # # Arguments: subsystem: The subsystem to check. # # Result: Returns success (0) if the subsystem is running. # Returns an error (1) if the subsystem is not running. ############################################################################## function check_if_running { subsystem=$1 LC_ALL=C lssrc -s $subsystem | tail -1 | read subsys group pid status if [[ $subsys == $subsystem ]] && [[ $status != "active" ]] then return 1 else return 0 fi } ############################################################################## # Function: check_nfs_subsystems # # Syntax: check_nfs_subsystems # # Description: Checks if the NFS subsystems are running. # # Notes: This function just checks NFSv4 subsystems. # # Result: Returns a count of the number of errors detected. ############################################################################## function check_nfs_subsystems { log "$0 $@" [[ $VERBOSE_LOGGING == "high" ]] && set -x typeset subsystems="nfsrgyd nfsd" if [[ -n $EXPORT_FILESYSTEM ]] then subsystems="$subsystems rpc.mountd rpc.statd rpc.lockd" fi typeset -i error run lssrc -g nfs for subsystem in $subsystems do if ! clcheck_server $subsystem then log "$0: $subsystem is running." else log "$0: ERROR: $subsystem is not running." (( error++ )) fi done return $error } ############################################################################## # Function: check_nfs_node # # Syntax: check_nfs_node # # Description: This function checks that the NFS node for the resource # group is registered. It also gives a warning if the # stable storage associated with the resource group does # not match that configured for the resource group. # # Result: Returns a count of the number of errors detected. ############################################################################## function check_nfs_node { log "$0 $@" [[ $VERBOSE_LOGGING == "high" ]] && set -x typeset rg=$GROUPNAME typeset -i error=0 typeset nfs_node sspath typeset KERNEL_BITS=$(/usr/sbin/bootinfo -K) (( KERNEL_BITS != 64 )) && return 0 [[ ! -x /usr/sbin/nfs4smctl ]] && return 0 # : If grace periods are not enabled then return # ps -p $nfsd_pid -o args | grep -q -- "-gp on" || return 0 # : If NFSv4 exports are configured, then check to see if this resource : group\'s nfs node is registered. # if [[ -n $EXPORT_FILESYSTEM_V4 ]] then nfs_node=$(cl_nfs4smctl -Q -N $rg -d '|' 2> /dev/null) # : If stable storage is not configured for the resource group, then use : the default location. # if [[ -z $STABLE_STORAGE_PATH ]] then STABLE_STORAGE_PATH=/var/adm/nfsv4.hacmp/$rg fi if [[ -z $nfs_node ]] then log "$0: ERROR: NFS node '$rg' is not registered." (( error++ )) else # : Check that the stable storage is what we expect. # sspath=$(echo "$nfs_node" | cut -d'|' -f3) if [[ $STABLE_STORAGE_PATH != $sspath ]] then if [[ -z $sspath ]] then log "$0: ERROR: Unexpected stable storage path associated with NFS node '$rg': Expected value: '$STABLE_STORAGE_PATH', Actual value: '$sspath'" (( error++ )) else # : We don\'t need to trigger a fallover in this situation, so just print : a warning, don\'t increment error. # log "$0: WARNING: Unexpected stable storage path associated with NFS node '$rg': Expected value: '$STABLE_STORAGE_PATH', Actual value: '$sspath'" fi fi if [[ -n $sspath ]] then if [[ -d $sspath ]] then if [[ ! -f $sspath/sm4.grace ]] then # : If the stable storage does not exist, print a warning. : Don\'t create an error since NFS will sometimes delete this file. # log "$0: WARNING: The stable storage file associated with NFS node '$rg' is missing." fi else log "$0: ERROR: stable storage directory '$sspath' does not exist." (( error++ )) fi fi fi fi return $error } ############################################################################## # Function: check_nfs_exports # # Syntax: check_nfs_exports # # Description: This function checks that the NFS exports that are configured # for the resource group are properly exported with the correct # options. The options that get checked are the version and # hasrv. # # Result: Returns a count of the number of errors detected. ############################################################################## function check_nfs_exports { log "$0 $@" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset rg=$GROUPNAME typeset -i error=0 log "$0: NFS Exports:" log "$0: NFSv3 Exports: $EXPORT_FILESYSTEM" log "$0: NFSv4 Exports: $EXPORT_FILESYSTEM_V4" typeset fs options typeset exports= typeset v3_exports= typeset v4_exports= typeset hasrv_exports= # : Loop through all of the exported filesystems ... # run exportfs | grep / | while read fs options ; do options=${options:1} # strip off the leading - typeset v3= typeset v4= typeset hasrvs= trace "$0[$LINENO] fs: $fs" trace "$0[$LINENO] options: $options" member_of $fs $EXPORT_FILESYSTEM && v3=:2:3 if member_of $fs $EXPORT_FILESYSTEM_V4 ; then v4=:4 hasrvs="$SERVICE_LABEL" fi # : vers contains a list of versions that the export is expected to have. # typeset vers=$(echo $v3$v4 | cut -d: -f2-) # : Skip exports that aren\'t part of this resource group # [[ -z "$vers" ]] && continue exports="$exports $fs" trace "$0[$LINENO] exports: $exports" options=$(echo $options | tr ',' '\n') typeset option value # : Loop through each option for the export we are processing # echo "$options" | tr , '\n' | while IFS='=' read option value ; do trace "$0[$LINENO] option: '$option'" trace "$0[$LINENO] value: '$value'" case $option in vers) # : If it is exported v2 or v3, then add it to the list of v3 exports : that were found. # case $value in *2*) v3_exports="$v3_exports $fs" ;; *3*) v3_exports="$v3_exports $fs" ;; esac # : If it is exported v4, then add it to the list of v4 exports : that were found. # case $value in *4*) v4_exports="$v4_exports $fs" ;; esac trace " vers: '$vers'" [[ "$vers" == *$value* ]] && continue log "$0: ERROR: option mismatch found while checking exports: export: $fs, option: $option, actual: $value expected: $vers" (( error++ )) ;; hasrv) # : Add the export to the list of exports found using hasrv # hasrv_exports="$hasrv_exports $fs" # : Check if the hasrv value is a valid value # member_of $value $hasrvs && continue log "$0: ERROR: option mismatch found while checking exports: export: $fs, option: $option, actual: $value expected: $hasrvs" (( error++ )) ;; esac done done # : Check for configured v3 exports that aren\'t exported. # for fs in $EXPORT_FILESYSTEM ; do trace "$0[$LINENO] fs: '$fs'" trace "$0[$LINENO] exports: '$v3_exports'" # : Did we find this filesystem exported v3 as expected? # member_of $fs $v3_exports && continue # : If it doesn\'t have a vers option, then it will be in exports but not : in v3_exports or v4_exports. Since the default export protocol is : v2/v3, treat this as a v2/v3 export. # member_of $fs $exports && continue log "$0: ERROR: A configured v2/v3 export is not exported: $fs" (( not_exported++ )) done # : Check for configured v4 exports that aren\'t exported. # for fs in $EXPORT_FILESYSTEM_V4 ; do trace "$0[$LINENO] fs: '$fs'" trace "$0[$LINENO] v4_exports: '$v4_exports'" # : Did we find this filesystem exported v4 as expected? # member_of $fs $v4_exports && continue log "$0: ERROR: A configured v4 export is not exported: $fs" (( not_exported++ )) done # : Check for configured v4 exports that aren\'t using the \"hasrv\" option. # for fs in $EXPORT_FILESYSTEM_V4 ; do trace "$0[$LINENO] fs: '$fs'" trace "$0[$LINENO] hasrv_exports: '$hasrv_exports'" # : Skip over missing v4 exports. We just reported errors for these. # member_of $fs $v4_exports || continue # : Did we find this filesystem exported v4 with the hasrv option as expected? : Treat this as just a warning. # member_of $fs $hasrv_exports && continue log "$0: WARNING: An NFSv4 export is not exported with the hasrv option: $fs" done return $error } ############################################################################## # Function: check_nfs_crossmounts # # Syntax: check_nfs_crossmounts # # Description: This function checks that the NFS crossmounts (if configured) # are properly mounted and that the NFS server can respond to # client requests made through the crossmount. # # Result: Returns a count of the number of errors detected. ############################################################################## function check_nfs_crossmounts { log "$0 $@" [[ $VERBOSE_LOGGING == "high" ]] && set -x typeset -i error=0 typeset rg=$GROUPNAME log "$0: NFS Mount Points: $MOUNT_FILESYSTEM" typeset host remote local type remaining typeset crossmounts= # : Loop through all active NFS mounts ... # mount | tr -s ' ' | tr ' ' ':' | while IFS=: read host remote local type remaining ; do # : Skip for non nfs mounts # [[ $type != nfs[234] ]] && continue # : Skip over nfs mounts that don\'t belong to this resource group # member_of $host $SERVICE_LABEL || continue # : Check if the crossmount is a configured one ... # if ! member_of "$local;$remote" $MOUNT_FILESYSTEM ; then log "$0: WARNING: An unconfigured crossmount was found: $local;$remote. This crossmount will not be checked." continue fi # : Add the crossmount to the list of crossmounts that have been found. # crossmounts="$crossmounts $local;$remote" # : Determine the expected NFS protocol version based on the configuration. : Note that we should be mounting at the highest protocol version that is : exported. # typeset exp_type= typeset exp_types= if member_of $remote $EXPORT_FILESYSTEM ; then exp_type=nfs3 exp_types="$exp_types nfs3" fi if member_of $remote $EXPORT_FILESYSTEM_V4 ; then exp_type=nfs4 exp_types="$exp_types nfs4" fi member_of $remote $EXPORT_FILESYSTEM_V4 && exp_type=nfs4 # : Check if the crossmount is mounted with the preferred NFS protocol. # [[ $type == $exp_type ]] && continue log "$0: WARNING: crossmount version mismatch: crossmount: $local;$remote, actual: $type expected: $exp_type" # : Check if the crossmount is mounted with a NFS protocol that is not : exported. # member_of $type $exp_types && continue log "$0: ERROR: A crossmount is mounted with a protocol that the server is not configured to export: $local;$remote, client protocol: $type, server protocol(s): ${exp_types:1}" done typeset crossmount for crossmount in $MOUNT_FILESYSTEM do # : Is this crossmount mounted? # if member_of $crossmount $crossmounts then # : The crossmount is mounted. : Verify that df -m of the crossmount responds. # log "$0: Checking df -m ${crossmount%";"*} ..." if df -m ${crossmount%";"*} then log "$0: df -m ${crossmount%";"*} succeeded" else log "$0: ERROR: df -m ${crossmount%";"*} failed: $?" (( not_crossmounted++ )) fi # : Verify that ls -ld of the crossmount root directory responds. # log "$0: Checking ls -ld ${crossmount%";"*} ..." if ls -ld ${crossmount%";"*} then log "$0: ls -ld ${crossmount%";"*} succeeded" else log "$0: ERROR: ls -ld ${crossmount%";"*} failed: $?" (( not_crossmounted++ )) fi else # : The crossmount is not mounted. # log "$0: ERROR: A configured crossmount is not mounted: $crossmount" (( error++ )) fi done return $error } ############################################################################## # Function: check_error_notification # # Syntax: check_error_notification error_label ... # # Description: This function checks whether the specified error occurred. # # Result: Returns a count of the number of errors detected. ############################################################################## function check_error_notification { log "$0 $@" [[ $VERBOSE_LOGGING == "high" ]] && set -x typeset -i error=0 typeset label typeset file for label in $* do for file in /usr/es/sbin/cluster/etc/errnotify/clam_nfsv4.$label.* do # : If no files exist matching this pattern, then file will be set to the : pattern itself, so check if file is actually a file first. # [[ -f $file ]] || break # : The file exists. See if it is a stale file. The filename is of : the form: 'clam_nfsv4...' : If the nfsd pid does not match the one currently running, then : this is a stale file and should be deleted. # typeset pid1=$nfsd_pid typeset pid2=${file##*.} if [[ $pid1 != $pid2 ]] then # : The file is stale, remove it. # rm -f $file > /dev/null 2>&1 continue fi log "$0: ERROR: AIX Error $label has been detected associated with nfsd process $pid1." cat $file (( error++ )) done done return $error } ############################################################################## # Function: handle_error_notification # # Syntax: handle_error_notification error_label sequence_number # monitor -errnotify error_label sequence_number # # Description: This function is called as a registered error notification # method. It writes to a file to indicate that the error # has occurred. # # Result: Returns a count of the number of errors detected. ############################################################################## function handle_error_notification { [[ $VERBOSE_LOGGING == "high" ]] && set -x trace "$0 $@" typeset label=${1:-} typeset seq=${2:-} if [[ -z $label || -z $seq ]] then log "usage: $PROGNAME -errnotify error_label sequence_number" return 1 fi typeset pid=$nfsd_pid if [[ -z $pid ]] then log "$PROGNAME: nfsd is not running. Ignoring error." return 0 fi typeset dir="/usr/es/sbin/cluster/etc/errnotify" typeset file1="$dir/.clam_nfsv4.$label.$seq.$pid.$$" typeset file2="$dir/clam_nfsv4.$label.$seq.$pid" mkdir -p $dir # : The notification file contains the errpt output for this error. : We save them in the file in case the error log wraps between now : and when the application monitor sees the file. # errpt -l $seq -a > $file1 # : Create the file atomically by renaming it. This way an application : monitor will not see a partially written notification file. # rm -f $file2 if ! mv -f $file1 $file2 ; then log "$PROGNAME: Error $? creating notification file $file2" return 1 fi return 0 } ############################################################################## # Function: Main # # Syntax: monitor # # Description: This is the application monitor script for the clam_nfsv4 # application monitor. It verifies # are properly mounted and that the NFS server can respond to # client requests made through the crossmount. # # Result: Returns a count of the number of errors detected. ############################################################################## ###################################### ### Initialization ###################################### ###################################### ### Global Variables ###################################### export PATH=$(/usr/es/sbin/cluster/utilities/cl_get_path all) export ODMDIR=/usr/es/sbin/cluster/etc/objrepos/active eval export $(/usr/es/sbin/cluster/utilities/cllsparam -x) monitor_interval=$(clodmget -n -f value -q "name=MONITOR_INTERVAL AND monitor=clam_nfsv4" HACMPmonitor) start_time=$(LC_ALL=C date -u +"%s") typeset -i not_exported=0 PROGNAME=${0##*/} if [[ $VERBOSE_LOGGING == "high" ]] then PS4_TIMER=true set -x version='1.10.1.5' fi ps -eo comm,pid | grep -w nfsd | grep -vw grep | read skik nfsd_pid ###################################### ### Function Main ###################################### case ${1:-} in -errnotify) shift handle_error_notification "$@" exit ;; esac if [[ -z $GROUPNAME ]] then log "WARNING: Resource group name not specified! Return success without further checking." exit 0 fi RESOURCES= RESOURCES="$RESOURCES SERVICE_LABEL" RESOURCES="$RESOURCES EXPORT_FILESYSTEM" RESOURCES="$RESOURCES EXPORT_FILESYSTEM_V4" RESOURCES="$RESOURCES MOUNT_FILESYSTEM" RESOURCES="$RESOURCES STABLE_STORAGE_PATH" for resource in $RESOURCES ; do eval export $resource= done RES_LIST="SERVICE_LABEL|EXPORT_FILESYSTEM|EXPORT_FILESYSTEM_V4|MOUNT_FILESYSTEM|STABLE_STORAGE_PATH" eval $(cllsres -g $GROUPNAME | egrep "$RES_LIST") log "Running $0 $@" log "Resource group: $GROUPNAME" log "Service Label(s): $SERVICE_LABEL" log "NFSv4 Node Name: $GROUPNAME" log "NFSv3 Export(s): ${EXPORT_FILESYSTEM:-(NONE)}" log "NFSv4 Export(s): ${EXPORT_FILESYSTEM_V4:-(NONE)}" log "Stable Storage Path: ${STABLE_STORAGE_PATH:-(NONE)}" log "NFS Mount(s): ${MOUNT_FILESYSTEM:-(NONE)}" # : The following code handles migration. In 5.4.0 and earlier the nfs export : version was specified in the HACMP exports file. Starting in 5.4.1 it : became a property of the resource group. In order to make these two : methods coexist in a backwards compatible manner, if the resource group has : no NFSv4 exports configured, then get the version information from the : HACMP exports file. # EXPFILE="/usr/es/sbin/cluster/etc/exports" DARE_EVENT=reconfig_resource_acquire # : If we don\'t have NFSv4 exports configured then determine the protocol : version\(s\) from the HACMP exports file. # if [[ -z $EXPORT_FILESYSTEM_V4 && -r $EXPFILE ]] then export_v3= export_v4= for fs in $EXPORT_FILESYSTEM do # : Get the export file for the filesystem from the exports file. : Only look at the part of the line preceding comments. # export_line=$(cut -d'#' -f1 $EXPFILE | grep -w $fs) # : The line is of the format: filesystem -option1,option2,... : This will give "option1 option2 ..." # options=$(IFS=, set -- $(print -- "$export_line" | cut -d- -f2-); print $*) # : Each option can be of the format name=value, or just name. : We only care about the vers option. # # : Have we seen the vers option? # vers_missing=1 # : Loop through all of the export options for this export. # for option in $options do case $option in vers=*) vers_missing=0 # : Merge in the vers option. : Odd logic because version could be '34' or '43' # case $option in *2*) export_v3="$export_v3 $fs" ;; *3*) export_v3="$export_v3 $fs" ;; esac case $option in *4*) export_v4="$export_v4 $fs" ;; esac ;; esac done # : If we didn\'t find the vers option, then NFSv3 is the default. # (( vers_missing )) && export_v3="$export_v3 $fs" done EXPORT_FILESYSTEM=$export_v3 EXPORT_FILESYSTEM_V4=$export_v4 log "NFSv3 Export(s): ${EXPORT_FILESYSTEM:-(NONE)}" log "NFSv4 Export(s): ${EXPORT_FILESYSTEM_V4:-(NONE)}" fi typeset -i error=0 # errors from any check typeset -i nfs_exports_error=0 # errors from exported filesystems check typeset -i not_exported=0 # count of file systems not NFS exported typeset -i nfs_crossmounts_error=0 # errors from cross mounts check typeset -i not_crossmounted=0 # count of file systems not cross mounted typeset -i rc=0 export not_exported not_crossmounted # set during checks by subroutines # : Check the various components of support, : and accumulate count of errors # check_nfs_subsystems ; (( error += $? )) check_nfs_node ; (( error += $? )) check_nfs_exports ; (( nfs_exports_error += $? )) check_nfs_crossmounts ; (( nfs_crossmounts_error += $? )) # : Adding retry check for nfs export file system failure... # if (( $not_exported > 0 && $nfs_exports_error == 0 )) || (( $not_crossmounted > 0 && $nfs_crossmounts_error == 0 )) then # # A note on the odd use of variables: # 'not_exported' counts file systems not found to be exported # 'nfs_exports_errors' counts errors from any other cause from that check # Hence, not_exported > 0 and nfs_exports_errors == 0 means that # some file systems were found to be not exported, but no other errors # happened during the check. Both should be 0 for success. # There is a similar use of 'not_crossmounted' and 'nfs_crossmounts_error' # : This logic deals with the fact that if there is an on-going : exportfs command elsewhere in the system, the check to the : expected exports may fail. So, retry the check, if we are : not yet out of time in the monitoring interval. # typeset end_time=$(LC_ALL=C date -u +"%s") typeset -i execution_time typeset -i remaining_time typeset -i interval=5 typeset -i sleep_duration (( execution_time = $end_time - $start_time )) # Time taken so far... (( remaining_time = $monitor_interval - $execution_time )) # Remaining time to treat monitor as failed... (( remaining_time = $remaining_time - $interval )) # Assume that remaining execution of this monitor takes less than 5 seconds... (( sleep_duration = $remaining_time / 2 )) if (( $sleep_duration > 0 )) then # : Give it another try, after a wait not so long : that we will overrun the monitoring interval # sleep $sleep_duration if (( $not_exported > 0 )) then check_nfs_exports ; (( nfs_exports_error = $? )) fi if (( $not_crossmounted > 0 )) then check_nfs_crossmounts ; (( nfs_crossmounts_error = $? )) fi fi if (( $not_exported != 0 || $not_crossmounted != 0 )) then # : Out of time, or retry did not work. Check to see : if we are hung up on an exportfs being run elsewhere. # typeset -i pid ppid typeset command typeset pso typeset -L exportfs_etime typeset -i mm ss typeset -i event group ctl_limit exportfs_eseconds pso=$(ps -eo pid,ppid,command) if print -- "$pso" | grep -w 'exportfs' | read pid ppid command then # : exportfs is running. This makes checking for current exports : unreliable. However, if exportfs has been running longer than : the config too long time limit, something is very wrong. # ps -o etime -p $pid | tail -1 | read exportfs_etime tracking_file=$(clodmget -q name=clutils.log -f value -n HACMPlogs)/nfs_monitor_exportfs_running if [[ $exportfs_etime == @([0-9][0-9]:[0-9][0-9]) ]] then # : exportfs has been running for less than an hour. : Derive the config too long time limit # clchmsgtimer | tail -1 | IFS=: read event group (( $event == 0 )) && event=180 (( $group == 0 )) && group=180 ctl_limit=$(( $event + $group )) print -- "$exportfs_etime" | IFS=: read mm ss exportfs_eseconds=$(( 60*$mm + $ss )) if (( $exportfs_eseconds < $ctl_limit )) then # : Assume success because exportfs is running, which will : temporarily make a file system look unexported # not_exported=0 not_crossmounted=0 typeset calledby="exportfs" while (( $ppid != 1 )) do # : Determine the call chain, in case it might be useful : in understanding how this happened. # if print -- "$pso" | grep "^[ ]*${ppid}" | read pid ppid command then calledby="${calledby} called by ${command}" # And $command begot $calledby ... else break fi done (( $ppid == 1 )) && calledby="${calledby} called by init" # And init begot $calledby ... # : Record this unusual and interesting action # print "${PROGNAME}[$LINENO]: $(date '+%Y/%m/%d %H:%M:%S'): ${GROUPNAME}: exports assumed valid due to '${calledby}' currently running" | tee -a $tracking_file fi fi if (( $not_exported != 0 || $not_crossmounted != 0 )) then # : The exports error count has not been reset. This is : because exportfs has been running too long. The error : is allowed to stand. # print "${PROGNAME}[$LINENO]: $(date '+%Y/%m/%d %H:%M:%S'): ${GROUPNAME}: exportfs has been running for ${exportfs_etime}. Exports are considered non-functional" | tee -a $tracking_file fi fi fi fi (( error += $not_exported + $not_crossmounted )) if [[ -n $EXPORT_FILESYSTEM_V4 ]] then check_error_notification SM4_LOCK_DOWN (( error += $? )) fi if (( error > 0 || $nfs_exports_error > 0 || $nfs_crossmounts_error > 0 )) then fatal "Resource group $GROUPNAME is unavailable ($error error(s))" fi success # No errors, so must have worked