#!/bin/ksh93 # ALTRAN_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # Copyright (C) Altran ACT S.A.S. 2017,2018,2021. All rights reserved. # # ALTRAN_PROLOG_END_TAG # # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # 61haes_r720 src/43haes/usr/sbin/cluster/events/utils/cl_disk_available.sh 1.2.17.4 # # Licensed Materials - Property of IBM # # COPYRIGHT International Business Machines Corp. 1990,2015 # All Rights Reserved # # US Government Users Restricted Rights - Use, duplication or # disclosure restricted by GSA ADP Schedule Contract with IBM Corp. # # IBM_PROLOG_END_TAG # @(#) 7d4c34b 43haes/usr/sbin/cluster/events/utils/cl_disk_available.sh, 726, 2147A_aha726, Feb 05 2021 09:50 PM # # COMPONENT_NAME: hacmp.events # # FUNCTION: Given a list of disks, make sure that they are in the # 'available' state. If necessary, break reserves, and remove any # ghost disks. # # ORIGINS: 27 # ############################################################################### # # Name: cl_disk_available # Returns: # 0 Success # 1 Failure - Can't make it Available # 2 Failure - Bad args # # Check to see if a disk is available to the system, if not try to # make it available. # # Arguments: [ -s ] [ -v ] [ hdiskname .... ] # # If called with '-s' flag the script implies that it is called not from the # event script context, so skip resource manager updates # # If called with '-v' flag, when reserves are broken on disks, no new # reserve is set. This is intended for use with volume groups that will be # used in concurrent mode, but currently have a reserve by another # initiator. # # If called with a list of hdisks, make those disks available # # If called with no list of hdisks, then process by resource group, based on # environment variables set by process_resources: # RESOURCE_GROUPS - space separated list of resource groups # HDISKS - list of disks for a resource group # - comma separated list of disks within a resource group # - colon separated lists of disks for each resource group # VOLUME_GROUPS - list of owning volume group (if any) for each # hdisk. Same pattern of comma and colon separated # lists as for HDISKS # e.g. - # RESOURCE_GROUPS="curley larry moe shemp" # HDISKS="hdisk11,hdisk22:hdisk21,hdisk31:hdisk99:hdisk101,hdisk1" # VOLUME_GROUPS="vg01,vg01:vg02,vg02,::vg03,vg0" # # Environment: VERBOSE_LOGGING - "high" => set -x # # Questions? Comments? Expressions of Astonishment? mailto:hafeedbk@us.ibm.com # ############################################################################### # # Name: make_disktypes_available # # Input: 1) owning adapter for list of disks # 2) disk type - FSCSI or SCSIDISK # 3) list of disks # # Function: This routine is passed a list of disks, all on a common # adapter. # * The names of any ghost disks are collected for later removal # * Any that are not available are added to the list of disks to # be made available later # * The adapter and disk names are passed to a routine that will # open the adapter and use ioctls to break any reserves # # Output: None # # Environment: VERBOSE_LOGGING, PATH, disklist, ghostlist, fscarray, # pscarray # ############################################################################### function make_disktypes_available { typeset PS4_FUNC="make_disktypes_available" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset parent=$1 typeset disktype=$2 shift 2 typeset disks=$* # : Check given disks for possible ghost disks, which must be later removed # for disk in $disks # Check to see if each disk is in an do # available state if [[ -z $(lsdev -Cc disk -l $disk -S A -F 'status') ]] then # : The disk "'$disk'" is not Available. : Find the Available disks with the same parent and address, but different name. # It is possible for the original disk device to become # "Defined" while a new disk device becomes "Available". # Go find all those new disks, which have the same parent # and address, but different name. Add them to the global # list of ghost disks to be removed. # case $disktype in FSCSI) ghostlist=${ghostlist:+"$ghostlist "}$(cl_fscsighost $disk) ;; SCSIDISK) ghostlist=${ghostlist:+"$ghostlist "}$(clodmget -q "parent = $parent AND \ name != $disk AND \ location = $(lsdev -Cc disk -l $disk -F 'location')" -f name -n CuDv) ;; esac # : Add disk "'$disk'" to the list that must be made Available # disklist=${disklist:+"$disklist "}"MKDEV.${disk}" fi # End if disk not available done # End loop through all given disks # : Break the reserve - pass the given list of disks having the given parent # case $disktype in FSCSI ) # : Perform a LUN reset on a SCSI-3 device # cl_fscsilunreset $NO_RETAIN_RES $parent $disks ;; SCSIDISK) # : Perform a LUN reset on a SCSI-2 device, if the device supports it. Otherwise, do a target reset. # if [[ $parent == @(scsi*) ]] ; then # : It is possible that this is actually a SCSI RAID adapter in JBOD mode. : If its parent name is "sisioaN" then its really a SCSI RAID adapter. # SISIOA=$(lsdev -C -l $parent -F parent) if [[ $SISIOA == @(sisioa*) && -x /usr/bin/sisraidmgr ]] ; then # : Tell the SCSI RAID adapter that this node is going to be primary. # This has to be done to make takeover work. # : sisraidmgr cant run if the adapter is already responding to loss of its peer. : So, give it a few tries to get its act together. for (( i=0 ; i<6 ; i++ )) ; do if /usr/bin/sisraidmgr -X -l $SISIOA -o '1' then # : Successfully reset the adapter to primary. Go on to break reserves. # break elif (( i<5 )) ; then # : Adapter switch to primary unsuccessful. Wait before trying again. # Wait*retries is slightly more than a minute. # The designers think this might be enough. # sleep 15 else # : Retries exhausted. Log the error, but go on to try and break reserves. # It might work. # cl_log 10409 "$PROGNAME: sisraidmgr failed for $SISIOA" $PROGNAME $SISIOA fi done fi fi cl_pscsilunreset $NO_RETAIN_RES $parent $disks ;; ISCSI ) # : Perform a LUN reset on an iSCSI device # cl_iscsilunreset $NO_RETAIN_RES $parent $disks ;; SCSIRAID ) # : Tell the SCSI RAID adapter that this node is going to be primary. # This has to be done to make takeover work # SISIOA=$(lsdev -C -l $parent -F parent) : sisraidmgr cant run if the adapter is already responding to loss of its peer. : So, give it a few tries to get its act together. if [[ -x /usr/bin/sisraidmgr ]] ; then for (( i=0 ; i<6 ; i++ )) ; do if /usr/bin/sisraidmgr -X -l $SISIOA -o '1' then # : Successfully reset the adapter to primary. Go on to break reserves. # break elif (( i<5 )) ; then # : Adapter switch to primary unsuccessful. Wait before trying again. # Wait*retries is slightly more than a minute. # The designers think this might be enough. # sleep 15 else # : Retries exhausted. Log the error, but go on to try and break reserves. # It might work. # cl_log 10409 "$PROGNAME: sisraidmgr failed for $SISIOA" $PROGNAME $SISIOA fi done fi # : Perform a LUN reset for each disk on the adapter # for disk in $disks ; do cl_flutereset $NO_RETAIN_RES $disk done ;; SAS ) # : Tell the SAS RAID adapter that this node is going to be primary. # This has to be done to make takeover work. # # : Determine if the VGs are GMVGs # typeset RV="false" if [[ -n $(clodmget -n -f type HACMPrresmethods) ]]; then # : Replicated resource methods are defined, check for resources # if [[ -n $(clodmget -q "name = 'GMVG_REP_RESOURCE'" -f value -n HACMPresource) ]]; then # : Replicated resources exist # RV="true" fi fi SISSAS=$(lsdev -C -l $parent -F parent) : sisraidmgr cant run if the adapter is already responding to loss of its peer. : So, give it a few tries to get its act together. # Breaking disk reserve is incorrect since being GLVM the SAS disk is an internal (not shared) disk. if [[ $RV == "false" && -x /usr/bin/sissasraidmgr ]] ; then for (( i=0 ; i<6 ; i++ )) ; do if /usr/bin/sissasraidmgr -X -l $SISSAS -o '1' then # : Successfully reset the adapter to primary. Go on to break reserves # break elif (( i<5 )) ; then # : Adapter switch to primary unsuccessful. Wait before trying again. # Wait*retries is slightly more than a minute. # It might be enough. # sleep 15 else # : Retries exhausted. Log the error, but go on to try and break reserves. # It might work. # cl_log 10409 "$PROGNAME: sisraidmgr failed for $SISSAS" $PROGNAME $SISSAS fi done fi # : Perform a LUN reset for each disk on the adapter # for disk in $disks ; do cl_flutereset $NO_RETAIN_RES $disk done ;; esac } ############################################################################### # # Name: is_disk_reserved # Returns: # 0 If disk does NOT have reservation against it # 1 If disk has reservation against it # 2 If scdiskutil failed for some other reason # # Check to see if a disk is reserved by a system through use of the SCIOTUR # (Test Unit Ready) ioctl. # # Arguments: diskname # # Environment: None # ############################################################################### function is_disk_reserved { typeset PS4_FUNC="is_disk_reserved" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset disk disk=$1 # # Call scdiskutil only once per disk. It retries up to 6 times, if necessary. # scdiskutil -t /dev/$disk TUR_RC=$? if (( $TUR_RC == 0 )) then return 0 elif (( $TUR_RC == 24 )) then # : 24 is SC_RESERVATION_CONFLICT - disk "'$disk'" has a reserve on it # return 1 else # : "'scdiskutil -t /dev/$disk'" failed for some other reason TUR_RC=$TUR_RC # return 2 fi } ############################################################################### # # Name: breakres # Returns: # 0 If reservation has been broken # -1 If anything goes wrong # # Break and, if appropriate, re-establish the reservation on a disk # # Arguments: 1. Owning adapter - e.g., "scsi0" # 2. disk type - e.g., "SCSIDISK" # 3. disk name - e.g., "hdisk333" # # Environment: None # ############################################################################### function breakres { typeset PS4_FUNC="breakres" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x # # Pick up the passed disk name # typeset parent disktype disk parent=$1 disktype=$2 disk=$3 # # Process by disk type # case $disktype in SSA | FCPARRAY ) # : Perform a target reset, using openx SC_FORCED_OPEN, if needed. # is_disk_reserved $disk isdr_rc=$? if (( $isdr_rc == 0 )) ; then # : If no reserve is in place, claim the reset worked. # rsrv_rc=0 elif (( $isdr_rc == 1 )) ; then # : There is a reserve in place on "'$disk'". Reset and reserve device. # Note: cl_scdiskrsrv resets AND reserves device # cl_scdiskrsrv $NO_RETAIN_RES /dev/$disk rsrv_rc=$? elif (( $isdr_rc == 2 )) ; then # : Test unit ready failed on "'$disk'" for unknown reason, try a reset. # cl_scdiskreset /dev/$disk rsrv_rc=$? fi if (( $rsrv_rc == 255 )) ; then cl_log 205 "$PROGNAME: Failed reset/reserve for device: $disk." $PROGNAME $disk fi ;; DPO ) # : processing for vpath disks, which may have a persistent reserve in place. # if [[ -x /usr/sbin/lquerypr ]] ; then # : If support is available to directly query the persistent reserve, do so. # [[ "$VERBOSE_LOGGING" == "high" ]] && Vflags='-v -V' lquerypr $Vflags -h /dev/$disk query_rc=$? case $query_rc in 0 | 3 ) # # Either this node has a persistent reserve on this device, or # the device is already open and need not be further checked # rsrv_rc=0 ;; 1 ) # : Device "'$disk'" has a persistent reserve from another node. Pre-empt with our key. # lquerypr $Vflags -p -h /dev/$disk rsrv_rc=$? if (( $rsrv_rc == 2 )) ; then # : pre-empt can fail if the ESS is sufficiently dazed and confused, or is busy, : or is generally having problems with its queues. Use a bigger hammer. # cl_vpathreset $NO_RETAIN_RES /dev/$disk rsrv_rc=$? fi ;; * ) # : The query PR failed query_rc=$query_rc. Try a path reset. # cl_vpathreset /dev/$disk rsrv_rc=$? ;; esac else # : Check if we can read the disk directly, and if needed, break : the reserve and re-establish the reservation. # cl_vpathreset /dev/$disk rsrv_rc=$? fi ;; FLUTE ) # : For Flute, check to see if the device can be read, and if not break the reserve # cl_flutereset $NO_RETAIN_RES /dev/$disk rsrv_rc=$? ;; ARRAY ) # 7135 is_disk_reserved $disk isdr_rc=$? if (( $isdr_rc == 0 )) ; then # # If there is no reserve, we should be able to read the disk # directly and we are done. # if cl_querypv -q /dev/$disk ; then rsrv_rc=0 else # : Could not read "'$disk'". Use SC_FORCED_OPEN to break any reserves. # cl_scdiskreset $NO_RETAIN_RES /dev/$disk rsrv_rc=$? fi elif (( $isdr_rc == 1 )) ; then # : There is a reserve in place. Reset and reserve device "'$disk'". # cl_scdiskrsrv $NO_RETAIN_RES /dev/$disk rsrv_rc=$? elif (( $isdr_rc == 2 )) ; then # : Test unit ready failed on "'$disk'" for unknown reason, try a reset. # cl_scdiskreset $NO_RETAIN_RES /dev/$disk rsrv_rc=$? fi if (( $rsrv_rc == 255 )) ; then cl_log 205 "$PROGNAME: Failed reset/reserve for device: $disk." $PROGNAME $disk fi ;; SCSIDISK | FSCSI | ISCSI | SAS ) # : Processing for disk type "'$disktype'" is done in the function 'make_disktypes_available'. # ;; * ) # : We dont have special support for disk type "'$disktype'", so try the straightforward method. : This will work if the disk device driver supports openx SC_FORCED_OPEN and ioctl SCIOTUR # is_disk_reserved $disk isdr_rc=$? if (( $isdr_rc == 0 )) ; then # : If no reserve is in place, claim the reset worked # rsrv_rc=0 elif (( $isdr_rc == 1 )) ; then # : There is a reserve in place. Reset and reserve device "'$disk'". # cl_scdiskrsrv $NO_RETAIN_RES /dev/$disk rsrv_rc=$? elif (( $isdr_rc == 2 )) ; then # : Test unit ready failed for unknown reason, lets try a reset. # cl_scdiskreset /dev/$disk rsrv_rc=$? fi if (( $rsrv_rc == 255 )) ; then cl_log 205 "$PROGNAME: Failed reset/reserve for device: $disk." $PROGNAME $disk fi ;; esac return $rsrv_rc } ############################################################################### # # Name: vpath_dup # # Function: Given a vpath, look at the underlying hdisks # * collect any ghost disks for later removal # * if any underlying hdisks are not 'Available', save their # names for later processing by make_disktypes_available # # Arguments: vpath # # Environment: VERBOSE_LOGGING, PATH, fscarray, pscarray # ############################################################################### function vpath_dup { typeset PS4_FUNC="vpath_dup" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset vpath v_disklist hdiskvalue u_disk u_parent vpath=$1 # : There should be multiple available hdisks for each vpath. Get the disk names from ODM. # v_disklist=$(clodmget -q "name = $vpath AND attribute = active_hdisk" -f value -n CuAt) : For each of the hdisks underlying the vpath, ensure there are no ghost disks. for hdiskvalue in $v_disklist ; do # # First, extract the hdisk name. u_disk=${hdiskvalue%%/*} # # Check to see if the underlying disk is available. If not, it # will be made so. If the underlying disk is available, do not # add it to the array which will be passed to the 'make_disktypes_available' # function. # if [[ -n $(lsdev -Cc disk -l $u_disk -S A -F 'status') ]] then continue else # : The underlying disk "'$u_disk'" is not available for vpath "'$vpath'". # Vpath disks can be built on top of either parallel or fibre # SCSI. Figure out which disk is this, and add it to the # appropriate list of disks to be processed. # u_parent=$(lsdev -Cl $u_disk -F parent) if [[ ${u_parent} == fscsi* ]] ; then fscarray[${u_parent}]=${fscarray[${u_parent}]:+"${fscarray[${u_parent}]} "}${u_disk} else : Assume a scsi parent device pscarray[${u_parent}]=${pscarray[${u_parent}]:+"${pscarray[${u_parent}]} "}${u_disk} fi fi done # all hdisks in this vpath } ############################################################################### # # Name: makedev # # Run the mkdev command on a given disk, up to four times, to make it # available # # Returns: return code from mkdev command # # Arguments: disk name - e.g., 'hdisk108' # # Environment: None # ############################################################################### function makedev { typeset PS4_FUNC="makedev" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset disk disk=$1 # # Try up to four times to make the disk available by running the appropriate # config method # [[ $VERBOSE_LOGGING != high ]] && cl_echo 641 "$PROGNAME: Attempting to make disk device: $disk." $PROGNAME $disk ADPT=$(lsdev -Cl ${disk} -F parent) # : Issue a SCSI command to the hdisk identified by its SCSI, LUN ID via the parent : device. If the command succeeded, we assume that the disk is reachable and a : subsequent mkdev will not time out. # if [[ ${ADPT:0:5} == fscsi ]] then # : If the parent adapter was already found to be inaccessible, no point trying # for adp in $fsc_deadadapter do if [[ ${ADPT} == ${adp} ]] then return fi done cl_fscsiluntest ${ADPT} ${disk} save_rc=$? # # If cl_fscsiluntest fails with error code then check if its 251 # which means open of the parent adapter failed - e.g., fscsi0, fscsi1. # Save this information for future use. # if [[ $save_rc == 251 ]] then : cl_fscsiluntest error code 251 means open of the parent adapter failed # ${parameter :+word } # If parameter is set and is non-null then substitute word; otherwise substitute nothing. fsc_deadadapter=${fsc_deadadapter:+"$fsc_deadadapter "}${ADPT} return fi # # If this was a failure during executing the SCSI ioctl call then its a problem with # just the disk and not parent. So just return back. # if [[ $save_rc == 250 ]] then : failure during executing the SCSI ioctl call, problem is disk not parent return fi fi for ((i=0 ; i<4 ; i++)) ; do if mkdev -l $disk ; then # if it worked break # can quit now fi done } ############################################################################### # # Name: oemghostdisks # # Invoke the OEM supplied method - or the specified internal built in method # to find all the ghost disks for a given disk # # Returns; A list of hdisk names that are ghosts for the given disk, # written to standard out # # Arguements: Method name - this is either the path name for the OEM # supplied method, or the identifier for an # internal build in method # # disk name - e.g., hdisk43 # ############################################################################### function oemghostdisks { typeset PS4_FUNC="oemghostdisks" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset findghost disk msg findghost=$1 disk=$2 msg=$(dspmsg scripts.cat 6581 "Custom method %s invoked for disk %s to perform %s" $findghost $disk "ghostbusting") logger -t "PowerHA SystemMirror for AIX" $msg case $findghost in SCSI2 ) loc=$(lsdev -Cc disk -l $disk -F 'location') clodmget -q "parent = $parent AND \ name != $disk AND \ location = $loc" -f name -n CuDv ;; SCSI3 ) cl_fscsighost $disk ;; * ) $findghost $disk ;; esac } ############################################################################### # # Name: oemcheckres # # Invoke the OEM supplied method - or the specified internal built in method # to check if a reserve is held on a disk (and hence must be broken) # # Returns: 0 If disk does NOT have reservation against it # 1 If disk has reservation against it # # Arguements: Method name - this is either the path name for the OEM # supplied method, or the identifier for an # internal build in method # # disk name - e.g., hdisk43 # ############################################################################### function oemcheckres { typeset PS4_FUNC="oemcheckres" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset checkres disk msg checkres=$1 disk=$2 msg=$(dspmsg scripts.cat 6581 "Custom method %s invoked for disk %s to perform %s " $checkres $disk "checkreserves") logger -t "PowerHA SystemMirror for AIX" $msg case $checkres in SCSI_TUR ) is_disk_reserved $disk ;; * ) $checkres $disk ;; esac } ############################################################################### # # Name: oembreakres # # Invoke the OEM supplied methods - or the specified internal built in # methods - to check for a reserve on a disk, and break it if its there. # # Returns: any return code from the method # # Arguements: Break Reserve Method name - this is either the path name for # the OEM supplied method, or the identifier for an # internal build in method # # Check Reserve Method Name # # parent - name of the adapter owning the disk - e.g., scsi3 # # disk name - e.g., hdisk43 # ############################################################################### function oembreakres { typeset PS4_FUNC="oembreakres" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset breakres checkres parent disk msg breakres=$1 checkres=$2 parent=$3 disk=$4 msg=$(dspmsg scripts.cat 6581 "Custom method %s invoked for disk %s to perform %s" $breakres $disk "breakres") logger -t "PowerHA SystemMirror for AIX" $msg oemcheckres ${checkres} $disk ckres_rc=$? if (( $ckres_rc != 0 )) ; then case $breakres in TARGET ) cl_scdiskrsrv /dev/$disk ;; PSCSI ) oem_disktype=SCSIDISK pscarray[${parent}]=${pscarray[${parent}]:+"${pscarray[${parent}]} "}${disk} ;; FSCSI ) oem_disktype=FSCSI fscarray[${parent}]=${fscarray[${parent}]:+"${fscarray[${parent}]} "}${disk} ;; * ) $breakres $parent $disk ;; esac else return $ckres_rc fi } ############################################################################### # # Name: oemmakedev # # Invoke the OEM supplied method - or the specified internal built in method # to make a disk available # # Returns: any return code from the method # # Arguements: Method name - this is either the path name for the OEM # supplied method, or the identifier for an # internal build in method # # disk name - e.g., hdisk43 # ############################################################################### function oemmakedev { typeset PS4_FUNC="oemmakedev" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset makedev disk msg makedev=$1 disk=$2 msg=$(dspmsg scripts.cat 6581 "Custom method %s invoked for disk %s to perform %s" $makedev $disk "makdev") logger -t "PowerHA SystemMirror for AIX" $msg case $makedev in MKDEV ) makedev $disk ;; * ) $makedev $disk ;; esac } ############################################################################### # # Name: make_disk_available # # Function: Given a disk name, determine its type and state, and invoke # the appropriate routines to # * break any reserves # * remove any ghost disks # * make it 'Available' # # Input: disk name - e.g., hdisk43 # # Output: None # ############################################################################### function make_disk_available { typeset PS4_FUNC="make_disk_available" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x disk=$1 # : Determine the owning adapter. This will be used to determine the disk type. # parent=$(lsdev -Cc disk -l $disk -F parent) PdDvLn=$(lsdev -Cc disk -l $disk -F PdDvLn) if [[ -z $PdDvLn ]] ; then : Fetch the PdDvLn entry from ODM if lsdev cant get it. PdDvLn=$(clodmget -q "name = $disk" -f PdDvLn -n CuDv) fi # : If PdDvLn="disk/remote_disk/rpvclient" there is no parent # if [[ ${PdDvLn} != 'disk/remote_disk/rpvclient' ]] then if [[ -z "$parent" || -z "$PdDvLn" ]] ; then cl_log 35 "$PROGNAME: Undefined disk device $disk. (May have been duplicate)." $PROGNAME $disk if [[ $JUST_DISKS="false" ]] then # : update resource manager with the event status # cl_RMupdate resource_error $disk $PROGNAME fi # : If lsdev returns null for parent/PdDvLn then something is wrong with the disk, bail. # return fi fi # : Determine the disk type # The mechanism used to break the reserves is dependent on the type. # disktype="UNKNOWN" OEMDISKTYPES="/etc/cluster/disktype.lst" # # The format of this file is # # E.g., # disk/fcal/HAL9000 FSCSI # # will cause this routine to treat such as device as 'fibre SCSI' # : First check if this disk should be treated as one of the known types. # This check is done first, because it is more specific than the "known types" # checking below, which is primarily based on owning adapter type. if [[ -r $OEMDISKTYPES && -s $OEMDISKTYPES ]] ; then if oemdisktype=$(grep -w "^${PdDvLn}" $OEMDISKTYPES | cut -f2) ; then if [[ ${oemdisktype} == @(SCSIDISK|SSA|FCPARRAY|ARRAY|FSCSI|FLUTE) ]] ; then disktype=$oemdisktype fi fi fi # : Check to see if the disk has custom methods. # parallel='false' makedev='MKDEV' checkres='SCSI_TUR' breakres='' if [[ ${disktype} == 'UNKNOWN' ]] ; then # : Read the custom methods from the HACMPdisktype ODM class. This : will instatiate the following variables: : ghostdisks - routine to find ghost disks : checkres - routine to check is a reserve is held : breakres - routine to break reserves : parallel - 'true' can break reserves in parallel : makedev - routine to make device available # eval $(odmget -q "PdDvLn = $PdDvLn" HACMPdisktype | sed -n 's/ = /=/p') if [[ -n $breakres ]] ; then disktype="OEM" fi fi # if [[ ${disktype} == 'UNKNOWN' ]] then : Check for disk type Remote Physical Volume Client if [[ ${PdDvLn} == 'disk/remote_disk/rpvclient' ]] then disktype="RPV_CLIENT" fi fi if [[ ${disktype} == 'UNKNOWN' ]] ; then # : Normal disk type checking, in lieu of any overrides or added customizations # case $parent in scsi* | vscsi* ) disktype="SCSIDISK" # parallel SCSI - SCSI-2 ;; ssar ) disktype="SSA" ;; dar* ) if [[ -n $(odmget -q "name = $disk AND PdDvLn = disk/fdar/array" CuDv) ]] ; then # : Either Dolphin or Flute. They behave differently, but have nearly identical ODM entries. # daclist=$(clodmget -q "name = $parent AND attribute = all_controller" -f value -n CuAt) if [[ -z $daclist ]] ; then disktype="FCPARRAY" else # : Check for IBM microcode, necessary to work with reset logic later # firstdac=${daclist%%,*} if LC_ALL=C lscfg -vl $firstdac | grep -q 'Manufacturer\.*\.IBM' ; then disktype="FLUTE" # : Fetch the disktype entry from ODM if lscfg can not get it. # elif odmget -q "name = "$firstdac"" CuVPD | grep -q 'IBM' ; then disktype="FLUTE" fi fi else disktype="ARRAY" fi ;; dpo ) disktype="DPO" # SDD vpath disks ;; fscsi* ) disktype="FSCSI" # fibre SCSI - SCSI-3 ;; isc* ) disktype="ISCSI" # iSCSI ;; sisraid* ) disktype="SCSIRAID" # SCSI RAID adapters ;; sas* ) disktype="SAS" # SAS ;; * ) # The following code has been duplicated above under parent of # dar as it was apparent from the field that the parent in the # case of FLUTE at least, is a dar. Since we were unable to # determine for sure if the parent of a Dolphin is also a dar # or possibly something else, the code has been left here as # well to be on the safe side. if [[ -n $(odmget -q "name = $disk AND PdDvLn = disk/fdar/array" CuDv) ]] ; then # : Either Dolphin or Flute. They behave differently, but have nearly identical ODM entries. # daclist=$(clodmget -q "name = $parent AND attribute = all_controller" -f value -n CuAt) if [[ -z $daclist ]] ; then disktype="FCPARRAY" else # : Check for IBM microcode, necessary to work with reset logic later # firstdac=${daclist%%,*} if LC_ALL=C lscfg -vl $firstdac | grep -q 'Manufacturer\.*\.IBM' ; then disktype="FLUTE" # Fetch the disktype entry from ODM if lscfg can't get it. elif odmget -q "name = "$firstdac"" CuVPD | grep -q IBM ; then disktype="FLUTE" fi fi fi ;; esac fi # : Check to see if the device "'$disk'" is Available. # call lsdev with the logical name of the device "-l" and # ask to see the devices that have a status of Available "-S A" # we limit the output to the status field "-F status". # if [[ -n $(lsdev -Cc disk -l $disk -S A -F 'status') ]] ; then # : Break the reserve on an "Available" disk. Run in bg where supported. # case $disktype in SSA ) # : SSA and vpath can run in parallel # parallel='true' breakres $parent $disktype $disk & pidlist="$pidlist $!" ;; DPO ) # : Clean up ghost disks for underlying hdisks, and note vpath for processing later # vpath_dup $disk dpolist=${dpolist:+"$dpolist "}${disk} ;; OEM ) # : For OEM methods, the method provider indicates whether parallel processing is possible. # if [[ $parallel == 'true' ]] ; then oembreakres $breakres $checkres $parent $disk & pidlist="$pidlist $!" else oembreakres $breakres $checkres $parent $disk fi ;; FSCSI ) # : Collect the fibre SCSI disks by owning adapter, for later processing in parallel. # fscarray[${parent}]=${fscarray[${parent}]:+"${fscarray[${parent}]} "}${disk} ;; ISCSI ) # : Collect the ISCSI disks by owning adapter, for later processing in parallel. # iscarray[${parent}]=${iscarray[${parent}]:+"${iscarray[${parent}]} "}${disk} ;; SCSIDISK ) # : Collect the parallel SCSI disks by owning adapter, for later processing in parallel # pscarray[${parent}]=${pscarray[${parent}]:+"${pscarray[${parent}]} "}${disk} ;; SAS ) # : Collect the SAS disks by owning adapter, for later processing in parallel # sasarray[${parent}]=${sasarray[${parent}]:+"${sasarray[${parent}]} "}${disk} ;; RPV_CLIENT ) # : Do nothing if Remote Physical Volume Client disk is available. # ;; SCSIRAID ) # : Collect the RAID scsi disks by owning adapter, for later processing by adapter # rscarray[${parent}]=${rscarray[${parent}]:+"${rscarray[${parent}]} "}${disk} ;; * ) # : Everything else has to be serial # breakres $parent $disktype $disk ;; esac else # : If the disk "'$disk'" is not Available, determine if Defined. # If it is not then there is nothing we can do since we # dont have any information on the disk. # if [[ -z $(lsdev -Cc disk -l $disk -S D -F 'status') ]] then # : If the disk is not even defined then we cannot recover # cl_log 35 "$PROGNAME: Undefined disk device $disk. (May have been duplicate)." $PROGNAME $disk continue fi # : At this point we know the disk is Defined, so process by disk type # case $disktype in SSA ) # # Reconfigure the defined disk, and make it available # makedev $disk # # Check to see if it worked # if [[ -n $(lsdev -Cc disk -l $disk -S A -F 'status') ]] ; then # : Disk is now available - go break reserves # breakres $parent "SSA" $disk & pidlist="$pidlist $!" fi ;; DPO ) # : Clean up ghost disks for underlying disks, and add vpath to list # This list will be made Available and have reserves broken. # vpath_dup $disk disklist=${disklist:+"$disklist "}"MKDEV.${disk}" dpolist=${dpolist:+"$dpolist "}${disk} ;; SCSIDISK ) # : Collect the parallel SCSI disks by owning adapter, for later processing in parallel. # pscarray[${parent}]=${pscarray[${parent}]:+"${pscarray[${parent}]} "}${disk} ;; FSCSI ) # : Collect the fibre SCSI disks by owning adapter, for later processing in parallel. # fscarray[${parent}]=${fscarray[${parent}]:+"${fscarray[${parent}]} "}${disk} ;; ISCSI ) # : Collect the ISCSI disks by owning adapter, for later processing in parallel. # iscarray[${parent}]=${iscarray[${parent}]:+"${iscarray[${parent}]} "}${disk} ;; SCSIRAID ) # : Collect the RAID scsi disks by owning adapter, for later processing by adapter. # rscarray[${parent}]=${rscarray[${parent}]:+"${rscarray[${parent}]} "}${disk} ;; SAS ) # : Collect the SAS disks by owning adapter, for later processing in parallel. # sasarray[${parent}]=${sasarray[${parent}]:+"${sasarray[${parent}]} "}${disk} ;; ARRAY ) # : remove the Available disks with the same parent and address, but different name. # It is possible for the original disk device to become # "Defined" while the new disk device becomes "Available". # This needs to be corrected. Get the connection address for # the device and check to see if there is another device # available at this address. If there is then delete it. # loc=$(lsdev -Cc disk -l $disk -F 'location') duplist=$(clodmget -q "parent = $parent AND \ name != $disk AND \ location = $loc" -f name -n CuDv) brokeres='false' for ghostdisk in $duplist ; do # : reset AND reserve since ghosts are caused by reservations on another node # because a device must be reserved by another node # during boot of this node for the disk to be unavailable. # if [[ $brokeres == 'false' && \ -n $(lsdev -Cc disk -l $ghostdisk -S A -F 'status') ]] ; then breakres $parent $disktype $ghostdisk RC=$? : exit status of "'breakres $parent $disktype $ghostdisk'" is: $RC if (( $RC == 0 || $RC == 1 )) ; then brokeres='true' fi fi [[ $VERBOSE_LOGGING != high ]] && cl_echo 640 "$PROGNAME: Removing duplicate disk device: $ghostdisk." $PROGNAME $ghostdisk rmdev -l "$ghostdisk" '-d' done # : Reconfigure the defined disk, and make it available # makedev $disk if [[ $brokeres == 'false' && \ -n $(lsdev -Cc disk -l $disk -S A -F 'status') ]] ; then # : If the reserve was not broken using the available duplicate above, break it here # breakres $parent $disktype $disk fi ;; FCPARRAY | FLUTE | SAS ) # FCPARRAY and Flute disks aren't supposed to get ghost disks, # so let's just make it available, and then break any # reservations. makedev $disk # Break the reserve on an "available" disk, this is a special # case for fcparray only. if [[ -n $(lsdev -Cc disk -l $disk -S A -F 'status') ]] ; then breakres $parent $disktype $disk fi ;; OEM ) # Find the ghost disks for the given disk duplist=$(oemghostdisks ${ghostdisks} $disk) RC=$? : exit status of "'oemghostdisks ${ghostdisks} $disk'" is: $RC if (( $RC != 0 )) ; then # If something goes wrong with finding ghost disks, skip # this disk and go on to the next continue fi brokeres='false' for ghostdisk in $duplist ; do # Break the reserve using this available ghost disk, so # that when we go to make the real disk available, the # PVID can be read. if [[ $brokeres == 'false' && \ -n $(lsdev -Cc disk -l $ghostdisk -S A -F 'status') ]] ; then if [[ $parallel == 'true' ]] ; then oembreakres $breakres $checkres $parent $ghostdisk & pidlist="$pidlist $!" else oembreakres $breakres $checkres $parent $ghostdisk if (( $? != 0 )) ; then # If for some reason, breaking the reserve # failed, skip this disk and go on to the # next continue fi fi brokeres='true' # Because we may have back grounded the process # of breaking the reserve, the remaining steps # of deleting the ghost disk profile and making # the real disk available have to be queued up, # and done when that finishes. disklist="$disklist ${makedev}.${disk}" fi ghostlist=${ghostlist:+"$ghostlist "}${ghostdisk} done if [[ $brokeres == 'false' ]] ; then # If we've not queued up this processing above, now try # and make the real disk available. oemmakedev $makedev $disk if [[ -n $(lsdev -Cc disk -l $disk -S A -F 'status') ]] ; then if [[ $parallel == 'true' ]] ; then oembreakres $breakres $checkres $parent $disk & pidlist="$pidlist $!" else oembreakres $breakres $checkres $parent $disk fi else # We had a problem, exit failure cl_log 31 "$PROGNAME: Unable to make device $disk available. Check hardware connections." $PROGNAME $disk fi fi ;; RPV_CLIENT ) # : Do not do makedev for a Remote Physical Volume Client disk. The makedev : for a RPV is done in the predisk_available Replicated Resource Method. # ;; * ) # : There is no special support for disk type "'$disktype'", so treat it : similar to a generic SCSI-2 disk and hope for the best. # loc=$(lsdev -Cc disk -l $disk -F 'location') # First, look for obvious duplicates duplist=$(clodmget -q "parent = $parent AND \ name != $disk AND \ location = $loc" -f name -n CuDv) brokeres='false' for ghostdisk in $duplist; do : Attempt to break any outstanding reserve on this device # because a device must be reserved by another node # during boot of this node for the disk to be unavailable if [[ $brokeres == 'false' && \ -n $(lsdev -Cc disk -l $ghostdisk -S A -F 'status') ]] ; then breakres $parent $disktype $ghostdisk RC=$? : exit status of "'breakres $parent $disktype $ghostdisk'" is: $RC if (( $RC == 0 || $RC == 1 )) ; then brokeres='true' fi fi ghostlist=${ghostlist:+"$ghostlist "}${ghostdisk} done # Reconfigure the defined disk, and make it available makedev $disk if [[ $brokeres == 'false' && \ -n $(lsdev -Cc disk -l $disk -S A -F 'status') ]] ; then : If the reserve was not broken using the available duplicate above, break it here breakres $parent $disktype $disk fi ;; esac fi } ############################################################################### # # Name: verify_disk_availability # # This function verifies whether the disk came up. If not, an appropriate # error message is written. # # Returns: # # Arguments: # disk name - e.g., hdisk43 # ############################################################################### function verify_disk_availability { typeset PS4_FUNC="verify_disk_availability" [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset disk=$1 # PdDvLn=$(lsdev -Cc disk -l $disk -F PdDvLn) if [[ ${PdDvLn} == 'disk/remote_disk/rpvclient' ]] then # : Do not do this check for a Remote Physical Volume Client disk. # The makedev for a RPV is done in the predisk_available Replicated Resource Method. return 0 fi if [[ -z $(LC_ALL=C lsdev -Cc disk -l $disk -S A -F 'status') ]] ; then # : We were unable to bring disk "'$disk'" on line. Note the problem. # cl_log 31 "$PROGNAME: Unable to make device $disk available. Check hardware connections." $PROGNAME $disk fi # # Note that the resource manager is not updated with the status of the # individual disks. This is because loss of a disk is not necessarily # severe enough to stop the event - varyonvg may still work. # } ############################################################################### # # Main routine # ############################################################################### typeset PROGNAME=${0##*/} export PATH="$(/usr/es/sbin/cluster/utilities/cl_get_path all)" if [[ $VERBOSE_LOGGING == "high" ]]; then set -x version='%I%' fi cl_echo 33 "Starting execution of $0 with parameters: $*\n" $0 "$*" JUST_DISKS=false # Normal event processing NO_RETAIN_RES="" # Default is to retain reservations while getopts ":sv" option ; do case $option in s ) # : Skip cl_RMupdate when "'-s'" is specified. # JUST_DISKS=true ;; v ) # : Do not reserve disks in concurrent VGs when "'-v'" is specified. # NO_RETAIN_RES='-v' ;; * ) dspmsg scripts.cat 6555 "Option \"-${option} ${OPTARG}\" is not valid" "-$option $OPTARG" return 1 ;; esac done shift $(($OPTIND - 1)) ghostlist='' # Ghost disks to be deleted disklist='' # Disks to be made available pidlist="" # child processes to wait on # : Global variables to define associative arrays for types FSCSI SCSIDISK : ISCSI: This will be used to collect the disks by owning adapter, to handle : in parallel. The index is the adapter name, the element is a list of : disks on that adapter. E.g., : fscarray[fscsi0]="hdisk11 hdisk22 hdisk33" # typeset -A fscarray # Array for type FSCSI typeset -A pscarray # Array for type SCSIDISK typeset -A iscarray # Array for type ISCSI typeset -A rscarray # Array for type SCSIRAID typeset -A sasarray # Array for type SAS # : List of vpath devices to be processed after the underlying hdisks are cleaned up # typeset dpolist # Array for type dpo # : This routine can be invoked in two different fashions: : - resource groups processed serially : A list of disks are passed as arguments : - resource groups are processed in parallel : Resource groups, their disks, and owning VGs are passed as environment variables # if [[ $JUST_DISKS == false && -n $JOB_TYPE && $JOB_TYPE != "GROUP" ]] ; then # : Resource groups processed in parallel # PROC_RES=true # implies JUST_DISKS is false ON_LIST=$(print $(lsvg -L -o 2>/var/hacmp/log/lsvg.err)) [[ -e /var/hacmp/log/lsvg.err && ! -s /var/hacmp/log/lsvg.err ]] && rm /var/hacmp/log/lsvg.err # : Get a sorted list of the VGs, and add any that are varyd on in passive : mode to the list of VGs that are on line. The disks for such VGs are : already locally accessible, so do not need to have reserves broken. # SORTED_VGS=$( IFS=:, set -- $VOLUME_GROUPS ; print $* | \ tr ' ' '\n' | sort -u ) # : If a volume group is varyd off, lsvg -L produces no output # PASSIVE_VGS=$( LC_ALL=C lsvg -L $SORTED_VGS 2>/dev/null | \ sed -n 's/VOLUME GROUP: *\([^ ]*\).*/\1/p' ) ON_LIST=$(print $ON_LIST $PASSIVE_VGS) # : Process by resource group # RESOURCE_GROUPS - space separated list of resource groups # HDISKS - list of disks for a resource group # - comma separated list of disks within a resource group # - colon separated lists of disks for each resource group # VOLUME_GROUPS - list of owning volume group (if any) for each # hdisk. Same pattern of comma and colon separated # lists as for HDISKS # e.g. - # RESOURCE_GROUPS="curley larry moe shemp" # HDISKS="hdisk11,hdisk22:hdisk21,hdisk31:hdisk99:hdisk101,hdisk1" # VOLUME_GROUPS="vg01,vg01:vg02,vg02,::vg03,vg0" # _HDISKS=$HDISKS _VOLUME_GROUPS=$VOLUME_GROUPS for GROUPNAME in $RESOURCE_GROUPS ; do # : Tell the resource manager we are about to bring these disks online # export GROUPNAME ALLDISKS="All_disks" cl_RMupdate resource_acquiring $ALLDISKS $PROGNAME # : Extract the disk list and corresponding VG list for the disks for this RG # print $HDISKS | IFS=':' read LIST_OF_HDISKS_FOR_RG HDISKS print $VOLUME_GROUPS | IFS=':' read LIST_OF_VOLUME_GROUPS_FOR_RG VOLUME_GROUPS grouptype=$(clodmget -q "group = $GROUPNAME" -f startup_pref -n HACMPgroup) for disk in $(IFS=', ' set -- $LIST_OF_HDISKS_FOR_RG ; print $*) do # : Extract the name of the VG that contains disk "'$disk'". # [[ -n $LIST_OF_VOLUME_GROUPS_FOR_RG ]] && { print $LIST_OF_VOLUME_GROUPS_FOR_RG | IFS=', ' read vg LIST_OF_VOLUME_GROUPS_FOR_RG if [[ -n $vg && $ON_LIST == ?(* )$vg?( *) ]] then : If the disk is in a VG that is already varyd on, or varyd on in passive mode, : then the disks are already accessible and there is nothing to do. continue else if [[ $grouptype == "OAAN" ]] then : If VG "'$vg'" is used in a concurrent RG, reservations are not to be retained NO_RETAIN_RES='-v' fi fi # : Set the fence height for VG "'$vg'" to allow read/write. if needed. # cl_set_vg_fence_height -c $vg rw RC=$? if (( $RC != 0 )) then # : Log any error, but continue. If this is a real problem, the varyonvg will fail # rw=$(dspmsg -s 103 cspoc.cat 350 'read/write' | cut -f2 -d,) cl_log 10511 "$PROGNAME: Volume group $VG fence height could not be set to read/write" $PROGNAME $VG $rw fi } # : Otherwise, go make sure the disk is accessible and available # make_disk_available $disk done done # : Restore resource group list # HDISKS=$_HDISKS VOLUME_GROUPS=$_VOLUME_GROUPS export GROUPNAME=$RESOURCE_GROUPS else # : If not called from process_resources, arguments are needed on the command line # PROC_RES=false if (( $# == 0 )) ; then # no disks passed cl_echo 34 "$PROGNAME usage: cl_disk_available hdisk ...\n" $PROGNAME exit 2 fi if [[ $JUST_DISKS == false ]] ; then # : called during event processing, so update the resource manager. # ALLDISKS="All_disks" cl_RMupdate resource_acquiring $ALLDISKS $PROGNAME fi # : Requests from serially processed resource groups # for disk in $* do make_disk_available $disk done fi # : Handle fscsi disks, based on parent adapter # for parent in ${!fscarray[@]} ; do make_disktypes_available $parent FSCSI ${fscarray[$parent]} done # : Handle parallel scsi disks, based on parent adapter # for parent in ${!pscarray[@]} ; do make_disktypes_available $parent SCSIDISK ${pscarray[$parent]} done # : Handle iscsi disks, based on parent adapter # for parent in ${!iscarray[@]} ; do make_disktypes_available $parent ISCSI ${iscarray[$parent]} done # : Handle RAID scsi disks, based on parent adapter # for parent in ${!rscarray[@]} ; do make_disktypes_available $parent SCSIRAID ${rscarray[$parent]} done # : Handle SAS disks, based on parent adapter # for parent in ${!sasarray[@]}; do make_disktypes_available $parent SAS ${sasarray[$parent]} done if [[ -n $pidlist ]] then : wait to sync any background processes still breaking reserves wait $pidlist fi # : If there were ghost disk profiles queued up to be removed, do so now. # for ghostdisk in $ghostlist ; do [[ $VERBOSE_LOGGING != high ]] && cl_echo 640 "$PROGNAME: Removing duplicate disk device: $ghostdisk." $PROGNAME $ghostdisk rmdev -l "$ghostdisk" '-d' done # : If there were disks to make available, do so now # for diskspec in $disklist ; do # Limitations on ksh processing are worked around by saving in the # elements of 'disklist' the makedev method name catenated with the # hdisk name with a '.'. The next two statements pull the two # apart. makedev=${diskspec%.*} disk=${diskspec#*.} oemmakedev $makedev $disk done # : Having cleaned up reserves or ghost disks on the underlying hdisks, break any PR on vpath devices. # for vpath in $dpolist ; do breakres DPO DPO $vpath done # : Verify disks came on line, and update the cluster manager status as appropriate # if [[ $PROC_RES == true ]]; then for GROUPNAME in $RESOURCE_GROUPS ; do export GROUPNAME # : Extract the list of disks and corresponding VGs for the disks in this RG # print $HDISKS | IFS=':' read LIST_OF_HDISKS_FOR_RG HDISKS print $VOLUME_GROUPS | IFS=':' read LIST_OF_VOLUME_GROUPS_FOR_RG VOLUME_GROUPS for disk in $(IFS=', ' set -- $LIST_OF_HDISKS_FOR_RG ; print $*) do # : Extract the name of the volume group that contains disk "'$disk'". # print $LIST_OF_VOLUME_GROUPS_FOR_RG | IFS=', ' read vg LIST_OF_VOLUME_GROUPS_FOR_RG if [[ -z $vg || $ON_LIST != ?(* )$vg?( *) ]] ; then : check any disk that is not in a varyd-on VG verify_disk_availability $disk fi done done else # : Resource groups processed serially: check all the disks passed on the command line # for disk in $* do verify_disk_availability $disk done fi # : Update the resource manager with the disks that came on line # ALLNOERR="All_nonerror_disks" if [[ $PROC_RES == true ]]; then for GROUPNAME in $RESOURCE_GROUPS ; do cl_RMupdate resource_up $ALLNOERR $PROGNAME done elif [[ $JUST_DISKS == false ]] ; then cl_RMupdate resource_up $ALLNOERR $PROGNAME fi exit 0