#!/usr/bin/ksh93 # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # 61haes_r720 src/43haes/usr/sbin/cluster/sa/filenet/sbin/cl_filenet_pe_import.sh 1.2.1.2 # # Licensed Materials - Property of IBM # # COPYRIGHT International Business Machines Corp. 2010 # All Rights Reserved # # US Government Users Restricted Rights - Use, duplication or # disclosure restricted by GSA ADP Schedule Contract with IBM Corp. # # IBM_PROLOG_END_TAG # @(#)81 1.2.1.2 src/43haes/usr/sbin/cluster/sa/filenet/sbin/cl_filenet_pe_import.sh, hacmp, 61haes_r720, 1541B_hacmp720 10/5/15 11:57:37 ############################################################################################### # # Name: cl_filenet_pe_import # # Process Engine import function for adding and removing PowerHA SystemMirror resources. # # Syntax: # cl_filenet_pe_import [-D | -v] [-M] {-a | -r} -o # -A # -n # -u # -l # -P # -f # # Arguments: # # -D Turn on debug mode. # # -A Application Name - Smart assistant application identifier # # -v Turn on verbose mode. # # -a Add/change all necessary powerHA resources for the specified Process Engine User. # If the resources already exist, they will be removed and then added. # This is the change/modify mode. # # -r Remove all powerHA resources for the specified process engine user. # # -o process engine user owning node. This is the node that has the # highest priority. # # -n Process engine user takeover node(s). These nodes will be appended to the # instance owning node and then passed to "claddgrp -n". Takeover # nodelist is colon-separated. # # -i Process Engine user to be made highly available or # removed. This is the only option required when removing (-r) # the resources. # # -l The TCP/IP service IP label used by Process Engine to communicate with # other application tiers and/or clients. # # -M Delete pre-existing configuration (modify mode) # # -P Netmask(IPv4)/Prefix Length(IPv6) # # -f Input XML file populated with required information # # Returns: # 0 on Success (import completed without error) # 1 on Warning (import completed with warning(s)) # 2 on Failure (import failed due to pre-verification failure) # 3 on Failure (import failed due to failure in adding resource) # ############################################################################################### . /usr/es/lib/ksh93/func_include version='1.2.1.2 $Source: 61haes_r711 43haes/usr/sbin/cluster/sa/filenet/sbin/cl_filenet_pe_import.sh 3$' ############################################################################################### # Global Definitions: ############################################################################################### RECOVERY_ODM=/tmp/cl_filenet_pe_RemRes_recovery_odm RECOVERY_CMDS=/tmp/cl_filenet_pe_RemRes_recovery_cmds PROGNAME=${0##*/} PATH=$(/usr/es/sbin/cluster/utilities/cl_get_path all) FPATH_BASE=/usr/es/lib/ksh93 FPATH=$FPATH_BASE/hacmp:$FPATH_BASE/db2:$FPATH_BASE/sa:$FPATH_BASE/aix:$FPATH_BASE/aix/odm:$FPATH_BASE/db2/vg:$FPATH_BASE/util export PATH FPATH export ODMDIR=/etc/objrepos export FILENET_SA_HOME="/usr/es/sbin/cluster/sa/filenet" export FILENET_ETC_PATH="$FILENET_SA_HOME/etc" export FILENET_SA_BIN_DIR="$FILENET_SA_HOME/sbin" # # Load the common variables # . $FILENET_SA_BIN_DIR/cl_filenet_common_variables export DEBUG=${DEBUG:-"0"} export VERBOSE=${VERBOSE:-"0"} export ADD_RESOURCES=0 export REMOVE_RESOURCES=0 export INSTANCE_OWNING_NODE="" export TAKEOVER_NODES="" export PE_USER_NAME="" export NETMASK_PREFIX_LEN="" export SERVICE_LABEL="" MANUAL_CONFIG_SCHEMA="/usr/es/sbin/cluster/sa/filenet/config/cl_filenet_manual_config.xsd" FILE_COLL_PARAMETERIZED_VER_FILENET_PE_NAME="FILENET_PE_PARAMETERIZED_VER_FILE_COLL" CUSTOM_VERIFICATION_NAME="sa_process_engine_verification" TEST_ROOT=/usr/es/sbin/cluster/etc/config/verify VERIFICATION_FILENAME="" # Add or Remove export ACTION="" NUM_ARGS=$# MODIFY_MODE=false ################################################################################### # # Name: usage # # Prints usage message and exits the program. # # Arguments: none # # Global Variables: # KLIB_OUTPUT_CONSOLE # KLIB_HACMPLOG_ENTRY # # # Returns: does not return ################################################################################### function usage { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x KLIB_SA_logmsg INFO 1 1 filenetsa.cat "Usage: cl_filenet_pe_import [-D | -v] {-a | -r} -o \n\ \t-n -u \n\ \t-l -P [-M]\n\ \t-f \n\ \n\ -D Turn on debug mode.\n\ -v Turn on verbose mode.\n\ -a Add/change all necessary powerHA resources for the specified Process Engine User.\n\ If the resources already exist, they will be removed and then added.\n\ This is the change/modify mode.\n\ -r Remove all HACMP resources for the specified Process Engine User.\n\ -o Process Engine User owning node. This is the node that has the highest priority.\n\ -n Process Engine User takeover node(s). These nodes will be appended to the instance\n\ owning node and then passed to \"claddgrp -n\". Takeover nodelist is colon-separated.\n\ -u Process Engine User (owner) to be made highly available or removed. This is the\n\ only option required when removing (-r) the resources.\n\ -l The TCP/IP service IP label used by Process Engine to communicate with other application\n\ tiers and/or clients.\n\ -M Delete the pre-existing configuration (modify mode)\n\ -P Netmask(IPv4)/Prefix Length(IPv6)\n\ -f Input XML file populated with required information\n" exit 1 } ############################################################################################### # Name: RemoveResources # # Description: # Removes (-r) powerHA resources for Process Engine. This function # removes all the powerHA resources its counterpart function, # Entries added by AddResources in the HACMPsa_metadata ODM will also # be cleaned up. # # This function, for the most part, will print out a warning and # continue removing other resources should it fails removing a # particular resource. # # Syntax: # RemoveResources # # Arguments: none # # Global Variables: # PE_USER_NAME # NODE_LIST # FILENETSA_SBIN_DIR # # Returns: # 0 on SUCCESS # 1 on FAILURE ############################################################################################### function RemoveResources { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x REMOVE_ERROR=0 # set to 1 if any of the remove fails sa_type="SA_FileNetProcessEngine" key="$APPLICATION_NAME" # Determine if the instance resource group exists first prior to removing typeset group=$(clodmget -q "sa_key=$key and sa_type=$sa_type" -f group \ -d "=" HACMPgroup) group=${group//\"/} if [[ -z $group ]]; then KLIB_SA_logmsg INFO 1 2 filenetsa.cat "Process Engine Resource Group doesn't exist \n" fi # Now, we remove all entries from HACMP. # Remove Application Server from the HACMP Configuration clvt query application $PE_USER_NAME"_ApplicationServer" > /dev/null 2>&1 if (( $? == 0 )); then clvt delete application $PE_USER_NAME"_ApplicationServer" if (( $? != 0 )); then KLIB_SA_logmsg ERROR 1 3 filenetsa.cat "Failed removing Process \ Engine Application Server from PowerHA SystemMirror\n" REMOVE_ERROR=1 fi else KLIB_SA_logmsg INFO 1 4 filenetsa.cat "Process Engine is not running on any node\n" fi # Remove Service IP Label from the HACMP. clvt query service_ip $SERVICE_LABEL > /dev/null 2>&1 if (( $? == 0 )); then clvt delete service_ip $SERVICE_LABEL if (( $? != 0 )); then KLIB_SA_logmsg ERROR 1 5 filenetsa.cat "Failed removing Process \ Engine Service IP Label from PowerHA SystemMirror\n" REMOVE_ERROR=1 fi else KLIB_SA_logmsg INFO 1 6 filenetsa.cat "Process Engine Service IP label doesn't exist \n" fi # Remove Resource Group from the HACMP configuration # All of the metadata will be removed at the time the RG # metadata is removed if [[ -n $group ]]; then # # Removing the depedency, if any with # the RG we are going to remove. # We can't remove the RG, without removing # any dependency exists with the RG. # clvt delete dependency RG=$group 1>/dev/null 2>&1 # # Removing the RG from cluster configuration # clvt delete resource_group $group if (( $? != 0 )); then KLIB_SA_logmsg ERROR 1 7 filenetsa.cat "Failed removing Process \ Engine Resource Group from PowerHA SystemMirror\n" REMOVE_ERROR=1 fi fi # # Remove custom verification check # clvt query method $CUSTOM_VERIFICATION_NAME 1>/dev/null 2>&1 (( $? == 0 )) && { clvt delete method $CUSTOM_VERIFICATION_NAME (( $? != 0 )) && { REMOVE_ERROR=1 } } # # Remove file collection # clvt query file_collection $FILE_COLL_PARAMETERIZED_VER_FILENET_PE_NAME 1>/dev/null 2>&1 (( $? == 0 )) && { clvt delete file_collection $FILE_COLL_PARAMETERIZED_VER_FILENET_PE_NAME (( $? != 0 )) && { REMOVE_ERROR=1 } } # # Remove the parameterized verfication check file only if # we are adding an application, not during change/show # [[ "$MODIFY_MODE" == "false" ]] && { [[ -f $TEST_ROOT/$VERIFICATION_FILENAME ]] && /usr/bin/rm $TEST_ROOT/$VERIFICATION_FILENAME } return $REMOVE_ERROR } ############################################################################################### # # Name: StopProcessEngine # # Checks the $NODE_LIST to make sure the instance is not running anywhere. If # found running, an attempt will be made to stop it. # # Arguments: none # # Global Variables: # PE_USER_NAME # NODE_LIST # # Returns: # 0 on SUCCESS # 1 on FAILURE # ############################################################################################### function StopProcessEngine { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x #set -x export NODE2STOP="" # The product (*) of these will dictate how long we wait for the # instance to stop. We wait up to $TOTAL_SLEEP_TIME seconds for the # instance to stop. typeset -i SLEEP_TIME=5 # in seconds typeset -i SLEEP_CYCLE=120 typeset -i TOTAL_SLEEP_TIME (( TOTAL_SLEEP_TIME = $SLEEP_TIME * $SLEEP_CYCLE )) # Verify and stop the instance if running. for node in $NODE_LIST do cl_rsh $node 'echo | /usr/bin/su - $PE_USER_NAME -c "LC_ALL=C initfnsw -y status | \ grep -i \"software started\""' > /dev/null 2>&1 (( $? == 0 )) && { NODE2STOP="$node" cl_rsh $NODE2STOP "$FILENET_SA_BIN_DIR/cl_filenet_process_engine_stop $PE_USER_NAME" > /dev/null 2>&1 KLIB_SA_logmsg INFO 1 8 filenetsa.cat "Stopping Process Engine on node \"%1\$s\"\n" "$NODE2STOP" } done [[ -z $NODE2STOP ]] && { KLIB_SA_logmsg INFO 1 9 filenetsa.cat "Process Engine is not active on any node\n" return 0 } # Check to see if the instance is stopped, if not, cycle through a sleep # to give it more time. while (( $SLEEP_CYCLE > 0 )) do cl_rsh $NODE2STOP "echo | /usr/bin/su - $PE_USER_NAME -c \"LC_ALL=C initfnsw -y status | \ grep -i \"software started\"\"" > /dev/null 2>&1 if (( $? == 0 )); then KLIB_SA_logmsg INFO 1 10 filenetsa.cat "Process Engine on node \"%1\$s\" is still running\n" "$NODE2STOP" KLIB_SA_logmsg INFO 1 11 filenetsa.cat "Sleeping for %1\$s \ seconds \(up to a total of %2\$s seconds\)...\n" "$TOTAL_SLEEP_TIME" "$TOTAL_SLEEP_TIME" sleep $SLEEP_TIME (( SLEEP_CYCLE = $SLEEP_CYCLE - 1 )) else KLIB_SA_logmsg INFO 1 13 filenetsa.cat "Process Engine on node \"%1\$s\" is stopped\n" "$NODE2STOP" return 0 fi done KLIB_SA_logmsg ERROR 1 12 filenetsa.cat "Failed stopping the process engine on node \"%1\$s\"\n" "$NODE2STOP" return 1 } ############################################################################################### # # Name: CheckInputs # # Description: # # Checks a number of basic conditions and inputs to make sure nothing major is # missing before adding any resources. # # Arguments: none # # Global Variables: # PE_USER_NAME # OWNING_NODE # TAKEOVER_NODES # # Returns: # 0 on SUCCESS # 1 on FAILURE # ############################################################################################### function CheckInputs { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x # Make sure the process engine user is a valid system user. KLIB_SA_logmsg INFO 1 14 filenetsa.cat "Verifying Process Engine user \"%1\$s\" \n" "$PE_USER_NAME" lsuser -a home $PE_USER_NAME > /dev/null 2>&1 if (( $? != 0 )); then KLIB_SA_logmsg ERROR 1 15 filenetsa.cat "The Process Engine user \"%1\$s\" does not seem to be available\n" "$PE_USER_NAME" exit 1 fi # Make sure the owning node (-o) is not also part of the takeover node(s) (-n). KLIB_SA_logmsg INFO 1 19 filenetsa.cat "Verifying the owning node \"%1\$s\" is part of \ takeover node(s) \"%2\$s\" list\n" "$OWNING_NODE" "$TAKEOVER_NODES" NODE_LIST_OK=1 # assuming it's OK for i in $TAKEOVER_NODES do if [[ "$i" == "$OWNING_NODE" ]]; then NODE_LIST_OK=0 # The instance owning node cannot also be part of the takeover nodes! fi done if (( $NODE_LIST_OK == 0 )); then KLIB_SA_logmsg ERROR 1 20 filenetsa.cat "The owning node \"%1\$s\" \ cannot be part of takeover node(s) \"%2\$s\" list\n" "$OWNING_NODE" "$TAKEOVER_NODES" exit 1 fi # Make sure all cluster nodes (-o & -n) are valid HACMP nodes. KLIB_SA_logmsg INFO 1 21 filenetsa.cat "Verifying all supplied nodes \ are valid PowerHA SystemMirror nodes.\n" for node in $OWNING_NODE $TAKEOVER_NODES do KLIB_HACMP_is_known_node $node if (( $? != 0 )); then KLIB_SA_logmsg ERROR 1 22 filenetsa.cat "Node \"%1\$s\" is not a cluster node\n" "$node" KLIB_SA_logmsg ERROR 1 23 filenetsa.cat "Both instance \ owning node and takeover node(s) must be cluster nodes\n" exit 1 fi done return 0 } ############################################################################################### # # Function: GenerateRecoveryScript # # Description: Run the recovery script and recover to PIT prior to removal # # Arguments: filenet process engine user name. # # Global Variables: # RECOVERY_CMDS # RECOVERY_ODM # # Returns: 0 if successful # !0 if a failure occurred # ############################################################################################### function GenerateRecoveryScript { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset user_name=$1 typeset key="$APPLICATION_NAME" typeset group=$(clodmget -q "sa_key=$key and sa_type=SA_FileNetProcessEngine \ and group=ProcessEngine_${PE_USER_NAME}_ResourceGroup" -f group -d "=" HACMPgroup) group=${group//\"/} typeset service_label=$(clodmget -q "name=SERVICE_LABEL and group=$group" -f value -d "=" HACMPresource) service_label=${service_label//\"/} typeset application_server=$(KLIB_HACMP_get_appserver $group) echo "/usr/bin/odmdelete -o HACMPgroup -q group=$group" > $RECOVERY_CMDS echo "/usr/bin/odmdelete -o HACMPresource -q group=$group" >> $RECOVERY_CMDS echo "/usr/bin/odmdelete -o HACMPsa_metadata -q application_id=$APPLICATION_NAME" >> $RECOVERY_CMDS echo "/usr/bin/odmdelete -o HACMPserver -q name=$application_server" >> $RECOVERY_CMDS echo "/usr/bin/odmdelete -o HACMPadapter -q ip_label=$service_label" >> $RECOVERY_CMDS MONITORS=$(LC_ALL=C clvt query application $application_server | \ grep ASSOCIATEDMONITORS | \ awk -F= '{print $2}' | sed -e "s/\"//g") for app_mon in $MONITORS do echo "/usr/bin/odmdelete -o HACMPmonitor -q monitor=$app_mon" >> $RECOVERY_CMDS done echo "/usr/bin/odmdelete -o HACMPcustom -q name=$CUSTOM_VERIFICATION_NAME" >> $RECOVERY_CMDS echo "/usr/bin/odmdelete -o HACMPfilecollection -q name=$FILE_COLL_PARAMETERIZED_VER_FILENET_PE_NAME" >> $RECOVERY_CMDS echo "/usr/bin/odmdelete -o HACMPfcfile -q fc_name=$FILE_COLL_PARAMETERIZED_VER_FILENET_PE_NAME" >> $RECOVERY_CMDS echo "/usr/bin/odmdelete -o HACMPrgdependency -q group_parent=$group" >> $RECOVERY_CMDS echo "/usr/bin/odmdelete -o HACMPrgdependency -q group_child=$group" >> $RECOVERY_CMDS echo "/usr/bin/odmdelete -o HACMPrg_loc_dependency -q group_name=$group" >> $RECOVERY_CMDS /usr/bin/odmget -q group=$group HACMPgroup > $RECOVERY_ODM /usr/bin/odmget -q group=$group HACMPresource >> $RECOVERY_ODM /usr/bin/odmget -q "application_id=$APPLICATION_NAME" HACMPsa_metadata >> $RECOVERY_ODM /usr/bin/odmget -q "name=$application_server" HACMPserver >> $RECOVERY_ODM /usr/bin/odmget -q "ip_label=$service_label" HACMPadapter >> $RECOVERY_ODM for app_mon in $MONITORS do /usr/bin/odmget -q "monitor=$app_mon" HACMPmonitor >> $RECOVERY_ODM done /usr/bin/odmget -q "name=$CUSTOM_VERIFICATION_NAME" HACMPcustom >> $RECOVERY_ODM /usr/bin/odmget -q "name=$FILE_COLL_PARAMETERIZED_VER_FILENET_PE_NAME" HACMPfilecollection >> $RECOVERY_ODM /usr/bin/odmget -q "fc_name=$FILE_COLL_PARAMETERIZED_VER_FILENET_PE_NAME" HACMPfcfile >> $RECOVERY_ODM /usr/bin/odmget -q group_parent=$group HACMPrgdependency >> $RECOVERY_ODM /usr/bin/odmget -q group_child=$group HACMPrgdependency >> $RECOVERY_ODM /usr/bin/odmget -q group_name=$group HACMPrg_loc_dependency >> $RECOVERY_ODM # # Provide execute permission to both the files # /usr/bin/chmod u+x $RECOVERY_CMDS /usr/bin/chmod u+x $RECOVERY_ODM } ############################################################################################### # # Function: RunRecoveryScript # # Description: Run the recovery script and recover to PIT prior to removal # # Arguments: na # # Global Variables: # RECOVERY_CMDS # RECOVERY_ODM # # Returns: 0 if successful # !0 if a failure occurred # ############################################################################################### function RunRecoveryScript { set +u [[ "$VERBOSE_LOGGING" == "high" ]] && set -x set -u /usr/bin/ksh93 $RECOVERY_CMDS 2>/dev/null /usr/bin/odmadd $RECOVERY_ODM 2>/dev/null } ############################################################################################### # # Function: RemoveRecoveryScript # # Description: Remove the recovery script files from /tmp/ # prior to exiting # # Arguments: na # # Global Variables: # RECOVERY_CMDS # RECOVERY_ODM # # Returns: 0 if successful # !0 if a failure occurred # ############################################################################################### function RemoveRecoveryScript { set +u [[ "$VERBOSE_LOGGING" == "high" ]] && set -x set -u rm -f $RECOVERY_ODM $RECOVERY_CMDS } ############################################################################################### # # Function: AddProcessEngineUserToMetadata # # Description: Add process engine user metadata to HACMPsa_metadata # process engine user name, UID # process engine primary group, GID # Process engine user home filesystem, respective VG # process engine configuration path, respective VG # # Arguments: na # # Global Variables: # PE_USER_NAME # # Returns: N/A # ############################################################################################### function AddProcessEngineUserToMetadata { userowner=( user='' group='' uid='' gid='' ) HOMEDIR=$(lsuser -a home $PE_USER_NAME) HOMEDIR=${HOMEDIR/*=/} HOMEVG=$(KLIB_AIX_get_vg_by_path $HOMEDIR) PE_CONFIG_VG=$(KLIB_AIX_get_vg_by_path /$PE_USER_NAME/local) KLIB_AIX_get_user_info userowner $PE_USER_NAME claddsaapp -a "$APPLICATION_NAME" PROCESS_ENGINE_USER_NAME=$PE_USER_NAME claddsaapp -a "$APPLICATION_NAME" PROCESS_ENGINE_USER_PRIMARY_GROUP=${userowner.group} claddsaapp -a "$APPLICATION_NAME" PROCESS_ENGINE_HOME_FILESYSTEM=$HOMEDIR claddsaapp -a "$APPLICATION_NAME" PROCESS_ENGINE_HOME_VG=$HOMEVG claddsaapp -a "$APPLICATION_NAME" PROCESS_ENGINE_CONFIG_FILESYSTEM="/$PE_USER_NAME/local" claddsaapp -a "$APPLICATION_NAME" PROCESS_ENGINE_CONFIG_VG=$PE_CONFIG_VG claddsaapp -a "$APPLICATION_NAME" PROCESS_ENGINE_UID=${userowner.uid} claddsaapp -a "$APPLICATION_NAME" PROCESS_ENGINE_GID=${userowner.gid} } ############################################################################################### # # Name: AddResources # # Description: # Adds (-a) HACMP resources for Process Engine User. If the resources # already exist, they will be removed and then added. Entries will # also be added to the HACMPsa_metadata ODM. Application Discovery # will remove the resources constructed by this function # # Syntax: # AddResources # # Arguments: none # # Global Variables: # PE_USER_NAME # OWNING_NODE # TAKEOVER_NODES # SERVICE_LABEL # IP_NETWORK # NODE_LIST # FILENET_SA_BIN_DIR # NETMASK_PREFIX_LEN # # Returns: # 0 on SUCCESS # 1 on FAILURE # ############################################################################################### function AddResources { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset CONFIG_VG=$(KLIB_AIX_get_vg_by_path /$PE_USER_NAME/local) # First remove any pre-existing resources for this application id RemoveResources (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 41 filenetsa.cat "Removal of one or more resources \ failed.\n" return 1 } # Add an entry into the HACMPsa_metadata ODM for the application id # and SMARTASSIST_ID and COMPONENT_ID claddsaapp -a "$APPLICATION_NAME" SMARTASSIST_ID='FILENET' \ COMPONENT_ID="$COMPONENT_ID" APPLICATION_NAME="$APPLICATION_NAME" PROCESS_ENGINE_USER_NAME="$PE_USER_NAME" sa_type="SA_FileNetProcessEngine" key="$PE_USER_NAME" # Create the Service IP Label KLIB_SA_add_ServiceLabel $SERVICE_LABEL $NETMASK_PREFIX_LEN || { KLIB_SA_logmsg ERROR 1 25 filenetsa.cat "Failed creating \ PowerHA SystemMirror Service IP Label for Process Engine\n" return 1 } # Remove the application server first, in case it already exists clvt delete application "${PE_USER_NAME}_ApplicationServer" >/dev/null 2>&1 # # Create the Instance Application Server # clvt add application $PE_USER_NAME"_ApplicationServer" \ STARTSCRIPT="$FILENET_SA_BIN_DIR/cl_filenet_process_engine_start ${PE_USER_NAME}" \ STOPSCRIPT="$FILENET_SA_BIN_DIR/cl_filenet_process_engine_stop ${PE_USER_NAME}" (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 26 filenetsa.cat "Failed creating PowerHA SystemMirror \ Application Server for process engine user \"%1\$s\" \n" "$PE_USER_NAME" return 1 } # Create the Instance SQL Monitor claddappmon MONITOR_TYPE=user name="${PE_USER_NAME}_SQLMonitor" \ RESOURCE_TO_MONITOR="${PE_USER_NAME}_ApplicationServer" \ INVOCATION='longrunning' \ MONITOR_METHOD="$FILENET_SA_BIN_DIR/cl_filenet_process_engine_monitor -u $PE_USER_NAME" \ MONITOR_INTERVAL='120' \ HUNG_MONITOR_SIGNAL='9' \ STABILIZATION_INTERVAL='240' \ RESTART_COUNT='3' \ RESTART_INTERVAL='1440' \ FAILURE_ACTION='fallover' \ CLEANUP_METHOD="$FILENET_SA_BIN_DIR/cl_filenet_process_engine_stop $PE_USER_NAME" \ RESTART_METHOD="$FILENET_SA_BIN_DIR/cl_filenet_process_engine_start $PE_USER_NAME" [[ $? != 0 ]] && { KLIB_SA_logmsg ERROR 1 27 filenetsa.cat "Failed creating \ PowerHA SystemMirror SQL Monitor for Process Engine User \"%1\$s\" \n" "$PE_USER_NAME" return 1 } claddsaapp -a "$APPLICATION_NAME" \ PROCESS_ENGINE_SQL_MONITOR="${PE_USER_NAME}_SQLMonitor" || { KLIB_SA_logmsg ERROR 1 28 filenetsa.cat "Failed adding PowerHA SystemMirror \ SQL Monitor to the Meta Database\n" return 1 } # Create the Instance Process Monitor claddappmon MONITOR_TYPE=process name="${PE_USER_NAME}_ProcessMonitor" \ RESOURCE_TO_MONITOR="${PE_USER_NAME}_ApplicationServer" \ INVOCATION='longrunning' \ PROCESSES='TM_daemon vworbbroker vwior' \ PROCESS_OWNER="$PE_USER_NAME" \ INSTANCE_COUNT='1' \ STABILIZATION_INTERVAL='240' \ RESTART_COUNT='3' \ RESTART_INTERVAL='1440' \ FAILURE_ACTION='fallover' \ CLEANUP_METHOD="$FILENET_SA_BIN_DIR/cl_filenet_process_engine_stop $PE_USER_NAME" \ RESTART_METHOD="$FILENET_SA_BIN_DIR/cl_filenet_process_engine_start $PE_USER_NAME" [[ $? != 0 ]] && { KLIB_SA_logmsg ERROR 1 29 filenetsa.cat "Failed creating \ PowerHA SystemMirror Process Monitor for Process Engine User \"%1\$s\" \n" "$PE_USER_NAME" return 1 } claddsaapp -a $APPLICATION_NAME \ PROCESS_ENGINE_PROCESS_MONITOR="${PE_USER_NAME}_ProcessMonitor" || { KLIB_SA_logmsg ERROR 1 30 filenetsa.cat "Failed adding PowerHA SystemMirror \ Process Monitor to the Meta Database\n" return 1 } # Create the Process Engine Resource Group claddgrp -g "ProcessEngine_${PE_USER_NAME}_ResourceGroup" \ -n "$NODE_LIST" \ -S 'OHN' -O 'FNPN' -B 'NFB' \ -A $sa_type -K ${APPLICATION_NAME} || { KLIB_SA_logmsg ERROR 1 31 filenetsa.cat "Failed creating PowerHA SystemMirror \ Resource Group for Process Engine User \"%1\$s\"\n" "$PE_USER_NAME" return 1 } claddsaapp -a $APPLICATION_NAME \ PROCESS_ENGINE_RESOURCE_GROUP="ProcessEngine_${PE_USER_NAME}_ResourceGroup" || { KLIB_SA_logmsg ERROR 1 32 filenetsa.cat "Failed adding PowerHA SystemMirror \ Resource Group to the Meta Database\n" return 1 } # Add the resources to the resource group clvt modify resource_group "ProcessEngine_${PE_USER_NAME}_ResourceGroup" \ SERVICE_LABEL="$SERVICE_LABEL" \ APPLICATIONS="${PE_USER_NAME}_ApplicationServer" \ VOLUME_GROUP="$CONFIG_VG" \ FORCED_VARYON='false' \ VG_AUTO_IMPORT='true' || { KLIB_SA_logmsg ERROR 1 33 filenetsa.cat "Failed adding resources to Resource Group\n" return 1 } # # Adding custom verification library to PowerHA SystemMirror verification. # /usr/es/sbin/cluster/utilities/claddcustom -t verify -n $CUSTOM_VERIFICATION_NAME -I "Process engine verification" \ -v /usr/es/sbin/cluster/sa/filenet/lib/libverfilenetsa.so -m 1 (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 49 filenetsa.cat "Adding custom verification library \"%1\$s\"\n\ to PowerHA SystemMirror verification has failed\n" "/usr/es/sbin/cluster/sa/filenet/lib/libverfilenetsa.so" return 1 } # # Add process engine custom verification library to HACMPsa_metadata # claddsaapp -a $APPLICATION_NAME CUSTOM_VERIFICATION_LIBRARY="$CUSTOM_VERIFICATION_NAME" # # Add process engine user metadata to HACMPsa_metadata # AddProcessEngineUserToMetadata # # Generate parameterized verification check file, which will # include process engine user and group names and respective ids, # and the user name and ids under process engine primary group. # generateVerifyScript $COMPONENT_ID $APPLICATION_NAME (( $? != 0 )) && return 1 claddsaapp -a $APPLICATION_NAME FILE_COLLECTION="$FILE_COLL_PARAMETERIZED_VER_FILENET_PE_NAME" # # Establish dependencies # process engine RG start after DB for process engine RG # Application engine RG should start after process engine RG # Process Engine RG start after content engine RG # # # RG with resources of Application Engine and Content Engine, # if deployed under same application server. # AE_CE_RG=$(clodmget -q "name=FILENET_AE_CE_RESOURCE_GROUP" -f value -d "=" HACMPsa_metadata ) AE_CE_RG=${AE_CE_RG//\"/} [[ -z $AE_CE_RG ]] && { # # Application Engine RG # AE_RG=$(clodmget -q "name=APPLICATION_ENGINE_RESOURCE_GROUP" -f value -d "=" HACMPsa_metadata) AE_RG=${AE_RG//\"/} # # Content Engine RG # CE_RG=$(clodmget -q "name=CONTENT_ENGINE_RESOURCE_GROUP" -f value -d "=" HACMPsa_metadata) CE_RG=${CE_RG//\"/} } # # Database RG with resources of DB for Process engine and Content Engine, # if both DBs configured under same DB2 instance. # DB_RG=$(clodmget -q "name=FILENET_DB_PE_CE_RESOURCE_GROUP" -f value -d "=" HACMPsa_metadata) DB_RG=${DB_RG//\"/} [[ -z $DB_RG ]] && { # # DB for only Process Engine RG # DB_PE_RG=$(clodmget -q "name=FILENET_DB_PE_RESOURCE_GROUP" -f value -d "=" HACMPsa_metadata) DB_PE_RG=${DB_PE_RG//\"/} # # DB for only Content Engine RG # DB_CE_RG=$(clodmget -q "name=FILENET_DB_CE_RESOURCE_GROUP" -f value -d "=" HACMPsa_metadata) DB_CE_RG=${DB_CE_RG//\"/} } # # Process Engine RG # PE_RG=$(clodmget -q "name=PROCESS_ENGINE_RESOURCE_GROUP" -f value -d "=" HACMPsa_metadata) PE_RG=${PE_RG//\"/} if [[ -n $DB_RG ]]; then clrgdependency -t 'START_AFTER' -a -c "$PE_RG" -p "$DB_RG" RC=$? (( $RC == 0 )) && { KLIB_SA_logmsg INFO 1 43 filenetsa.cat "\"%1\$s (Process Engine RG)\" START AFTER \"%2\$s (DB for Content Engine \ and Process Engine RG)\" dependency has been established\n" "$PE_RG" "$DB_RG" } (( $RC != 0 )) && return 1 # # Establishing NODECOLLOCATION dependency across process engine RG and # DB for Process engine RG # clrgdependency -t 'NODECOLLOCATION' -a -l "$DB_RG $PE_RG" (( $? != 0 )) && return 1 else [[ -n "$DB_PE_RG" ]] && { clrgdependency -t 'START_AFTER' -a -c "$PE_RG" -p "$DB_PE_RG" RC=$? (( $RC == 0 )) && { KLIB_SA_logmsg INFO 1 44 filenetsa.cat "\"%1\$s (Process Engine RG)\" START AFTER \"%2\$s \ (DB for Process Engine RG)\" dependency has been established\n" "$PE_RG" "$DB_PE_RG" } (( $RC != 0 )) && return 1 # # Establishing NODECOLLOCATION dependency across process engine RG and # DB for Process engine RG # clrgdependency -t 'NODECOLLOCATION' -a -l "$DB_PE_RG $PE_RG" (( $? != 0 )) && return 1 } fi if [[ -n $AE_CE_RG ]]; then clrgdependency -t 'START_AFTER' -a -c "$PE_RG" -p "$AE_CE_RG" RC=$? (( $RC == 0 )) && { KLIB_SA_logmsg INFO 1 45 filenetsa.cat "\"%1\$s (Application Engine and Content \ Engine RG)\" START AFTER \"%2\$s (Process Engine RG)\" dependency has been established \n" "$AE_CE_RG" "$PE_RG" } (( $RC != 0 )) && return 1 else [[ -n "$CE_RG" ]] && { [[ -n "$AE_RG" ]] && [[ -n "$DB_RG" || -n "$DB_CE_RG" ]] || { clrgdependency -t 'START_AFTER' -a -c "$PE_RG" -p "$CE_RG" RC=$? (( $RC == 0 )) && { KLIB_SA_logmsg INFO 1 46 filenetsa.cat "\"%1\$s (Content Engine RG)\" START AFTER \ \"%2\$s (Process Engine RG)\" dependency has been established \n" "$CE_RG" "$PE_RG" } (( $RC != 0 )) && return 1 } } [[ -n "$AE_RG" ]] && { [[ -n "$CE_RG" ]] && [[ -n "$DB_RG" || -n "$DB_CE_RG" ]] && { clrgdependency -t 'START_AFTER' -d -c "$PE_RG" -p "$CE_RG" (( $? != 0 )) && return 1 } clrgdependency -t 'START_AFTER' -a -c "$AE_RG" -p "$PE_RG" RC=$? (( $RC == 0 )) && { KLIB_SA_logmsg INFO 1 47 filenetsa.cat "\"%1\$s (Application Engine RG)\" START AFTER \ \"%2\$s (Process Engine RG)\" dependency has been established \n" "$AE_RG" "$PE_RG" } (( $RC != 0 )) && return 1 } fi return 0 } ############################################################################################### # # Function: # getVerifyProcessEngineUID_GID_Test # # Purpose: # Prints a script that will validate the Process Engine user, group, and # users under primary group have same user and group ID on all nodes # that participate in the current application # # Arguments: # process engine user # # Returns: # n/a # ############################################################################################### function getVerifyProcessEngineUID_GID_Test { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset pe_user=$1 typeset LIST_OF_USERS typeset PRIMARY_GROUP_USER_NAME typeset uid typeset gid typeset user typeset -i num=1 PRIMARY_GROUP_USER_NAME=$(lsuser -a pgrp $pe_user) PRIMARY_GROUP_USER_NAME=${PRIMARY_GROUP_USER_NAME/*=/} LIST_OF_USERS=$(lsgroup -a users $PRIMARY_GROUP_USER_NAME) LIST_OF_USERS=${LIST_OF_USERS/*=/} LIST_OF_USERS=$(echo $LIST_OF_USERS | sed 's/,/ /g') # # Generates a test script that will: # # have Process Engine User and Group # Users under process engine user primary group # echo "\n#\n# Perform validation for user: $pe_user\n#" uid=$(/usr/sbin/lsuser -a id $pe_user | awk -F= '{ print $2 }') echo "HAVerify.User.severity = \"ERROR\"" echo "HAVerify.User.exists[0].name=\"$pe_user\"" echo "HAVerify.User.exists[0].UID=$uid" gid=$(/usr/sbin/lsgroup -a id $PRIMARY_GROUP_USER_NAME | awk -F= '{ print $2 }') echo "HAVerify.Group.severity = \"ERROR\"" echo "HAVerify.Group.exists[0].name=\"$PRIMARY_GROUP_USER_NAME\"" echo "HAVerify.Group.exists[0].GID=$gid" for user in $LIST_OF_USERS; do [[ "$user" != "root" && "$user" != "$pe_user" ]] && { uid=$(/usr/sbin/lsuser -a id $user | awk -F= '{ print $2 }') echo "HAVerify.User.severity = \"ERROR\"" echo "HAVerify.User.exists[$num].name=\"$user\"" echo "HAVerify.User.exists[$num].UID=$uid" (( num = $num + 1 )) } done } ############################################################################################### # Function: # getVerifyHeader # # Purpose: # Generate the verification parameterized script header # # Arguments: # (1) Application Name # # Output: # Script output to stdout # # Returns: # n/a # ############################################################################################### function getVerifyHeader { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x typeset app_name=$1 echo "#\n# PowerHA SystemMirror / Process Engine" echo "# Application Verificiation script generated for: $cid" echo "# Date: $(date)\n#\n" print -n "Component.Name.DefaultName = \"" print -n "Process Engine" echo "\"" echo "Component.MsgCat.ID = 2" echo "Component.MsgCat.Set = 8" echo "Component.MsgCat.Catalog = \"filenetsa.cat\"" echo "Component.Nodes = \"S=FILENET:A=$app_name\"" } ############################################################################################### # Function: # generateVerifyScript # # Purpose: # Generates the cluster verification parameterized script for # process engine user, then pushes the file across to other nodes # in the cluster # # Arguments: # (1) Component Identifier # (2) Application Name # # Returns: # 0 on success # 1 on failure # ############################################################################################### function generateVerifyScript { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x #set -x typeset cid=$1 typeset app_name=$2 typeset localnode=$(get_local_nodename) VERIFICATION_FILENAME=${app_name}"_"${cid}".ver" [[ -z $app_name || -z $cid ]] && return 1 { getVerifyHeader $app_name getVerifyProcessEngineUID_GID_Test $PE_USER_NAME } > $TEST_ROOT/$VERIFICATION_FILENAME for node in $NODE_LIST; do [[ "$node" != "$localnode" ]] && { cl_rcp $TEST_ROOT/$VERIFICATION_FILENAME \ $node:$TEST_ROOT/$VERIFICATION_FILENAME } done clvt add file_collection $FILE_COLL_PARAMETERIZED_VER_FILENET_PE_NAME \ FILES="$TEST_ROOT/$VERIFICATION_FILENAME" \ ISPROPOGATEDFILEDURINGSYNC="true" ISPROPOGATEAUTOWHENDETECTED="true" 2>/dev/null return 0 } ################################################################################# # # Name: importConfigFromFile # # Description: # This function will read the supplid config file and create powerHA # resources to configure Process Engine User for high availability. # # Arguments: # N/A # # Returns: # 0 - on success # 1 - on failure # ################################################################################# importConfigFromFile() { [[ "$VERBOSE_LOGGING" == "high" ]] && set -x PE_MANUAL_CONFIG="" COMPONENT="Process_Engine" PE_MANUAL_CONFIG=$(clsaxmlutil -s -x $CONFIG_FILE -m $MANUAL_CONFIG_SCHEMA -t $COMPONENT -a configure) [[ "$PE_MANUAL_CONFIG" != "yes" ]] && { return } PE_USER_NAME=$(LC_ALL=C clsaxmlutil -s -x $CONFIG_FILE -m $MANUAL_CONFIG_SCHEMA -t $COMPONENT | grep USER_NAME) (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 48 filenetsa.cat "Problem with XML configuration file. Ensure a valid XML file supplied.\n" exit 1 } PE_USER_NAME=$(echo $PE_USER_NAME | cut -d"=" -f2) primary_node=$(LC_ALL=C clsaxmlutil -s -x $CONFIG_FILE -m $MANUAL_CONFIG_SCHEMA -t $COMPONENT | grep PrimaryNode) (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 48 filenetsa.cat "Problem with XML configuration file. Ensure a valid XML file supplied.\n" exit 1 } primary_node=$(echo $primary_node | cut -d"=" -f2) KLIB_HACMP_is_known_node $primary_node (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 22 filenetsa.cat "Node \"%1\$s\" is not a cluster node\n" "$node" exit 1 } PRIMARY_NODE=$primary_node takover_nodes=$(LC_ALL=C clsaxmlutil -s -x $CONFIG_FILE -m $MANUAL_CONFIG_SCHEMA -t $COMPONENT | grep TakeoverNodes) (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 48 filenetsa.cat "Problem with XML configuration file. Ensure a valid XML file supplied.\n" exit 1 } takover_nodes=$(echo $takover_nodes | cut -d"=" -f2) takeover_nodes=$(echo $takover_nodes | tr ',' ' ') for tnode in $takeover_nodes do KLIB_HACMP_is_known_node $tnode (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 22 filenetsa.cat "Node \"%1\$s\" is not a cluster node\n" "$node" exit 1 } done TAKEOVER_NODES=$(echo $takeover_nodes | sed -e "s/ /\:/g") addrs=$(LC_ALL=C clsaxmlutil -s -x $CONFIG_FILE -m $MANUAL_CONFIG_SCHEMA -t $COMPONENT | grep IPAddress_or_name) (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 48 filenetsa.cat "Problem with XML configuration file. Ensure a valid XML file supplied.\n" exit 1 } addrs=$(echo $addrs | cut -d"=" -f2) SERVICE_LABEL=$addrs netmask=$(LC_ALL=C clsaxmlutil -s -x $CONFIG_FILE -m $MANUAL_CONFIG_SCHEMA -t $COMPONENT | grep Prefix_or_Netmask) (( $? != 0 )) && { KLIB_SA_logmsg ERROR 1 48 filenetsa.cat "Problem with XML configuration file. Ensure a valid XML file supplied.\n" exit 1 } NETMASK_PREFIX_LEN=$(echo $netmask | cut -d"=" -f2) APPLICATION_NAME="process_engine_user_$PE_USER_NAME" if [[ -n $NETMASK_PREFIX_LEN ]]; then /usr/es/sbin/cluster/sa/filenet/sbin/cl_filenet_pe_import -a -v -u $PE_USER_NAME \ -A $APPLICATION_NAME \ -l $SERVICE_LABEL \ -o $PRIMARY_NODE \ -n "$TAKEOVER_NODES_TOKENIZED" \ -C "$COMPONENT_ID" \ -P "$NETMASK_PREFIX_LEN" else /usr/es/sbin/cluster/sa/filenet/sbin/cl_filenet_pe_import -a -v -u $PE_USER_NAME \ -A $APPLICATION_NAME \ -l $SERVICE_LABEL \ -o $PRIMARY_NODE \ -n "$TAKEOVER_NODES_TOKENIZED" \ -C "$COMPONENT_ID" fi } ############################################################################################### # Main: ############################################################################################### umask -S u=rw,g=,o= (( $NUM_ARGS == 0 )) && usage while getopts DA:Mvaro:n:u:l:P:f: option do case $option in D) # Verbose on + Debug on VERBOSE=1 DEBUG=1 ;; A) # application name APPLICATION_NAME=$OPTARG ;; v) # Verbose on VERBOSE=1 ;; a) # Add Resources if [[ "$REMOVE_RESOURCES" == "1" ]]; then usage fi ACTION="add" ADD_RESOURCES=1 ;; r) # Remove Resources if [[ "$ADD_RESOURCES" == "1" ]]; then usage fi ACTION="remove" REMOVE_RESOURCES=1 ;; o) # Primary cluster node that owns the instance OWNING_NODE=$OPTARG ;; n) # Standby cluster node(s) that is/are capable of hosting the instance TAKEOVER_NODES="$OPTARG" TAKEOVER_NODES=$(echo $TAKEOVER_NODES | sed -e "s/\:/ /g") ;; u) # Name of instance to be made highly available PE_USER_NAME=$OPTARG ;; l) # Service IP label for instance SERVICE_LABEL=$OPTARG ;; M) # Delete the pre-existing configuration (modify mode) MODIFY_MODE=true ;; P) # Netmask or prefix length NETMASK_PREFIX_LEN=$OPTARG ;; f) # Manual configuration mode MANUAL_CONFIG=true CONFIG_FILE=$2 ;; *) usage esac done COMPONENT_ID="FILENET_PROCESS_ENGINE" # # Before handling anything else, check if we have to configure from XML # if [[ $MANUAL_CONFIG == true ]]; then if [[ ! -f $CONFIG_FILE ]]; then dspmsg -s 51 cluster.cat 26 "Unable to read the configuration file. Please ensure the correct path" return 1 fi importConfigFromFile exit 0 fi export NODE_LIST="$OWNING_NODE $TAKEOVER_NODES" if (( $ADD_RESOURCES == 0 )) && (( $REMOVE_RESOURCES == 0 )); then KLIB_SA_logmsg ERROR 1 34 filenetsa.cat "Missing required argument(s) to import\n" usage fi # Verify and, if needed, stop the instance before moving on. StopProcessEngine if (( $? != 0 )); then KLIB_SA_logmsg ERROR 1 37 filenetsa.cat "Process Engine on \ node \"%1\$s\" should be manually stopped before continuing\n" "$OWNING_NODE" exit 1 fi if [[ "$ACTION" == "add" ]] then clsapre CheckInputs if (( $? != 0 )); then KLIB_SA_logmsg ERROR 1 38 filenetsa.cat "Validation of one or more required \ resources failed\n" exit 1 fi # # Remove the configuration, in case modify operation # [[ "$MODIFY_MODE" == "true" ]] && { GenerateRecoveryScript $PE_USER_NAME set -a eval $(clquerysaapp -a $APPLICATION_NAME RESOURCE_GROUP) set +a service_ip_label=$(clodmget -q "name=SERVICE_LABEL and \ group=$RESOURCE_GROUP" -f value -d "=" HACMPresource) service_ip_label=${service_ip_label//\"/} [[ -n $service_ip_label ]] && clvt delete service_ip $service_ip_label # # Removing the depedency, if any with # the RG we are going to remove. # We can't remove the RG, without removing # any dependency exists with the RG. # clvt delete dependency RG=$RESOURCE_GROUP 1>/dev/null 2>&1 clvt delete resource_group $RESOURCE_GROUP clrmsaapp -a $APPLICATION_NAME } # # Determine if the process engine user we're attempting to add is already part of an # HACMP configuration # # Check to ensure another resource group with the same application name hasn't been # created application=$(clodmget -q "name=PROCESS_ENGINE_USER_NAME and value=$PE_USER_NAME" \ -f application_id -d "=" HACMPsa_metadata) application=${application//\"/} [[ -n $application ]] && { KLIB_SA_logmsg ERROR 1 39 filenetsa.cat "The Process Engine for User \"%1\$s\" is \ already defined to the PowerHA SystemMirror\n\ cluster configuration with application \"%2\$s\"\n" "$PE_USER_NAME" "$application" exit 1 } AddResources if (( $? != 0 )); then RemoveResources clquerysaapp -a $APPLICATION_NAME > /dev/null 2>&1 (( $? == 0 )) && { clrmsaapp -a $APPLICATION_NAME } # # When something went wrong while adding resources # during change of application, we are restoring # back the original configuration. # [[ "$MODIFY_MODE" == "true" ]] && { RunRecoveryScript RemoveRecoveryScript } KLIB_SA_logmsg ERROR 1 40 filenetsa.cat "Resource creation or adding required \ resources to resource group failed.\n" exit 1 fi [[ "$MODIFY_MODE" == "true" ]] && { RemoveRecoveryScript } clsapost -v (( $? != 0 )) && exit 1 else KLIB_SA_logmsg ERROR 1 42 filenetsa.cat "Unexpected Process Engine import action encountered\n" exit 1 fi exit 0