#! /bin/sh
# $Header$
#
#bcpyrght
#***************************************************************************
#* $Copyright: Copyright (c) 2022 Veritas Technologies LLC. All rights reserved $ *
#***************************************************************************
#ecpyrght
#
# bpend_notify.oracle_bli
#
# This script is called by NetBackup after bpbkar has completed the
# block incremental backup. It is normally used to checkpoint the
# DB and back up the control file and archived transaction logs.
#
# This script:
#	receives 5 parameters: CLIENTNAME POLICYNAME SCHEDNAME SCHEDTYPE STATUS
#	must be executable by the root user
#	should exit with 0 upon successful completion
#
# The following environment variables are also available:
#       KEYWORD          - The KEYWORD associated with the policy
#       STREAMS          - The number of policies with the same KEYWORD
#       BACKUPID         - The NetBackup image ID
#       UNIXBACKUPTIME   - The backup time in seconds since 1970
#       BACKUPTIME       - The backup time in readable form
#       BPEND_TIMEOUT    - The bpend_notify timeout value
#
# If this script will not complete within a few seconds, you should set
# the BPEND_TIMEOUT in the /usr/openv/netbackup/bp.conf file on the server.
# You should also be aware that the time taken by this script will delay
# the initiation of other client's backups.
#
# To use this script, copy it to the /usr/openv/netbackup/bin/ directory
# on a client and make the modifications as directed below. Then copy
# it to bpend_notify.POLICY where POLICY is the name of each of the
# policies that will back up a subset of the filesystems. For example, if
# policies orasid_block_incr1, orasid_block_incr2, and orasid_block_incr3
# are used, copy it to bpend_notify.orasid_block_incr1,
# bpend_notify.orasid_block_incr2, and bpend_notify.orasid_block_incr3.
#
# Once the bpend_notify.POLICY files have been created, change the mode
# bits on the files to 500 and change the owner to root.
#
# CAUTION:  writing anything to stdout or stderr can cause backup problems
# so direct any output to /dev/null or the file defined by OUTF.
#


# -----------------------------------------------------------------------------
# Replace the xxxxxx in ORACLE_DBA with the oracle administrator's user name.
# -----------------------------------------------------------------------------

#ORACLE_DBA=oracle
ORACLE_DBA=xxxxxx

# -----------------------------------------------------------------------------
# Replace xxxxxx below with the name of the policy that will do
# the DB shutdown/startup or alter tablespace operations. Normally,
# if you have 3 policies named block_incr1, block_incr2, and block_incr3,
# you would set POLICY_IN_CONTROL to block_incr1 so that you could
# increase or decrease the number of policies easily.
# -----------------------------------------------------------------------------

POLICY_IN_CONTROL=xxxxxx

# -----------------------------------------------------------------------------
# Replace the xxxxxx in ORACLE_BASE with the directory at the top of the
# Oracle software and administrative file structure. The recommended value
# from Oracle is /MOUNTPOINT/app/oracle.
# For example: ORACLE_BASE=/u01/app/oracle
# -----------------------------------------------------------------------------

ORACLE_BASE=xxxxxx
export ORACLE_BASE

# -----------------------------------------------------------------------------
# Replace the xxxxxx in ORACLE_HOME with the directory containing the Oracle
# software for a given Oracle Server release. The recommended value from
# Oracle is ${ORACLE_BASE}/product/RELEASE.
# For example: ORACLE_HOME=${ORACLE_BASE}/product/10.2
# -----------------------------------------------------------------------------

#ORACLE_HOME=${ORACLE_BASE}/product/RELEASE
ORACLE_HOME=xxxxxx
export ORACLE_HOME

# -----------------------------------------------------------------------------
# The Oracle admin command path.
# -----------------------------------------------------------------------------

#SQLCMD=${ORACLE_HOME}/bin/sqlplus
SQLCMD=xxxxxx

# ---------------------------------------------------------------------------
# Change the Oracle login.
# ---------------------------------------------------------------------------
 
SQLLOGIN="sys/change_on_install as sysdba"

 
# -----------------------------------------------------------------------------
# Change the ORACLE_SID value if it is not the same as the KEYWORD.
# -----------------------------------------------------------------------------

ORACLE_SID=$KEYWORD
export ORACLE_SID

# -----------------------------------------------------------------------------
# Replace xxxxxx below with the name of the Oracle archive logs directory.
# If you don't have an archive logs directory, you need to change the check
# of the ORACLE_LOGS variable farther below so that abort is not called.
# -----------------------------------------------------------------------------

ORACLE_LOGS=xxxxxx

# -----------------------------------------------------------------------------
# Change the Oracle init file path (if necessary).
# -----------------------------------------------------------------------------

ORACLE_INIT=${ORACLE_BASE}/admin/${ORACLE_SID}/pfile/init${ORACLE_SID}.ora

# -----------------------------------------------------------------------------
# Change the Oracle config file path (if necessary).
# -----------------------------------------------------------------------------

ORACLE_CONFIG=${ORACLE_BASE}/admin/${ORACLE_SID}/pfile/config${ORACLE_SID}.ora

# -----------------------------------------------------------------------------
# Change the Oracle control file path (if necessary). A copy of the Oracle
# control file will be written to this file so that it can be backed up.
# -----------------------------------------------------------------------------

ORACLE_CNTRL=${ORACLE_BASE}/admin/${ORACLE_SID}/pfile/cntrl${ORACLE_SID}.ora

# -----------------------------------------------------------------------------
# If you want to receive mail for a successful execution of the script,
# replace the xxxxxx for the MAIL_ADDR_SUCCESS variable with a mail address.
# If you want to receive mail for a unsuccessful execution of the script,
# replace the xxxxxx for the MAIL_ADDR_FAILURE variable with a mail address.
# -----------------------------------------------------------------------------

MAIL_ADDR_SUCCESS=xxxxxx
MAIL_ADDR_FAILURE=xxxxxx

# -----------------------------------------------------------------------------
# Change the following PSCMD variable if necessary.
# -----------------------------------------------------------------------------

PSCMD="/bin/ps -ef"

# -----------------------------------------------------------------------------
# Change the following LD_LIBRARY_PATH variable if necessary.
# -----------------------------------------------------------------------------

LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${ORACLE_HOME}/lib

if [ -d /usr/ucblib ]
then
	LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/ucblib
fi

export LD_LIBRARY_PATH

# -----------------------------------------------------------------------------
# END OF THE CONFIGURATION CHANGES.
# -----------------------------------------------------------------------------







# -----------------------------------------------------------------------------
# Restart the oracle DB (for shutdown/backup/restart method).
# -----------------------------------------------------------------------------

restart_database()
{
$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME ------------------------" >>$OUTF
$ECHO "`$TIME` $POLICYNAME restart_database() START" >>$OUTF
$ECHO "`$TIME` $POLICYNAME ------------------------" >>$OUTF
$ECHO "" >>$OUTF

/bin/rm -f $WORKF
/bin/rm -f $CMDF

/bin/touch $WORKF
/bin/chown $ORACLE_DBA $WORKF
/bin/chmod 644 $WORKF
/bin/touch $CMDF
/bin/chmod 644 $CMDF

$ECHO "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >>$CMDF
$ECHO "export LD_LIBRARY_PATH" >>$CMDF
$ECHO "$SQLCMD <<EOF" >>$CMDF
$ECHO "$SQLLOGIN" >>$CMDF
$ECHO "set echo on;" >>$CMDF
if [ "${ORACLE_INIT}" = "" ]
then
       $ECHO "startup" >> $CMDF
else
       $ECHO "startup pfile=$ORACLE_INIT" >>$CMDF
fi
$ECHO "exit" >>$CMDF
$ECHO "EOF" >>$CMDF

/bin/su $ORACLE_DBA -c "/bin/sh $CMDF" >>$WORKF 2>>$WORKF

$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME Database startup command output:" >>$OUTF
$ECHO "" >>$OUTF

/bin/cat $WORKF >>$OUTF

# Look for errors of the form "ORA-00000".

STAT=`egrep "(ORA|DBA|SQL|LCC|MGR)-[0-9]" $WORKF|wc -l`

if [ ${STAT} -eq 0 ]
then
	# No errors, but find out if the instance is really up.

	SMONPROC=`${PSCMD}|grep "smon_${ORACLE_SID}"|grep -v grep|wc -l`

	if [ ${SMONPROC} -eq 0 ]
	then
		$ECHO "" >>$OUTF
		$ECHO "`$TIME` $POLICYNAME Could not find smon_${ORACLE_SID} process." >>$OUTF
		$ECHO "`$TIME` $POLICYNAME Creating error file $ERROR_FILE" >>$OUTF
		/bin/touch "${ERROR_FILE}"
		/bin/chmod 644 "${ERROR_FILE}"
	else
		$ECHO "" >>$OUTF
		$ECHO "`$TIME` $POLICYNAME Database ${ORACLE_SID} restarted." >>$OUTF
		/bin/rm -f ${SHUTDOWN_FILE}
	fi
else
	$ECHO "" >>$OUTF
	$ECHO "`$TIME` $POLICYNAME Database ${ORACLE_SID} could not be restarted." >>$OUTF
	$ECHO "`$TIME` $POLICYNAME Creating error file $ERROR_FILE" >>$OUTF
	/bin/touch "${ERROR_FILE}"
	/bin/chmod 644 "${ERROR_FILE}"
fi

$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME ----------------------" >>$OUTF
$ECHO "`$TIME` $POLICYNAME restart_database() END" >>$OUTF
$ECHO "`$TIME` $POLICYNAME ----------------------" >>$OUTF
$ECHO "" >>$OUTF
}

# -----------------------------------------------------------------------------
# Take the database out of backup mode.
# -----------------------------------------------------------------------------

alter_tablespace_end_backup()
{
$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME -----------------------------------" >>$OUTF
$ECHO "`$TIME` $POLICYNAME alter_tablespace_end_backup() START" >>$OUTF
$ECHO "`$TIME` $POLICYNAME -----------------------------------" >>$OUTF
$ECHO "" >>$OUTF

/bin/rm -f $WORKF
/bin/rm -f $CMDF

/bin/touch $WORKF
/bin/chown $ORACLE_DBA $WORKF
/bin/chmod 644 $WORKF
/bin/touch $CMDF
/bin/chmod 644 $CMDF

$ECHO "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >>$CMDF
$ECHO "export LD_LIBRARY_PATH" >>$CMDF
$ECHO "$SQLCMD <<EOF" >>$CMDF
$ECHO "$SQLLOGIN" >>$CMDF
$ECHO "set echo on;" >>$CMDF

cat $TABLES_FILE |
while read TBLSPACE
do
$ECHO "alter tablespace $TBLSPACE end backup;" >>$CMDF
done

$ECHO "exit" >>$CMDF
$ECHO "EOF" >>$CMDF

/bin/su $ORACLE_DBA -c "/bin/sh $CMDF" >>$WORKF 2>>$WORKF

$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME Database end backup command output:" >>$OUTF
$ECHO "" >>$OUTF

/bin/cat $WORKF >>$OUTF

# Look for errors of the form "ORA-00000".

STAT=`egrep "(ORA|DBA|SQL|LCC|MGR)-[0-9]" $WORKF|wc -l`

if [ ${STAT} -ne 0 ]
then
	$ECHO "" >>$OUTF
	$ECHO "`$TIME` $POLICYNAME Could not take database ${ORACLE_SID} out of backup mode." >>$OUTF
	abort
else
	$ECHO "" >>$OUTF
	$ECHO "`$TIME` $POLICYNAME Database ${ORACLE_SID} taken out of backup mode." >>$OUTF
	/bin/rm -f $BEGIN_BACKUP_FILE
fi

$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME ---------------------------------" >>$OUTF
$ECHO "`$TIME` $POLICYNAME alter_tablespace_end_backup() END" >>$OUTF
$ECHO "`$TIME` $POLICYNAME ---------------------------------" >>$OUTF
$ECHO "" >>$OUTF
}

# -----------------------------------------------------------------------------
# Checkpoint the oracle DB.
# -----------------------------------------------------------------------------

checkpoint_database()
{
$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME ---------------------------" >>$OUTF
$ECHO "`$TIME` $POLICYNAME checkpoint_database() START" >>$OUTF
$ECHO "`$TIME` $POLICYNAME ---------------------------" >>$OUTF
$ECHO "" >>$OUTF

/bin/rm -f $WORKF
/bin/rm -f $CMDF

/bin/touch $WORKF
/bin/chown $ORACLE_DBA $WORKF
/bin/chmod 644 $WORKF
/bin/touch $CMDF
/bin/chmod 644 $CMDF

$ECHO "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >>$CMDF
$ECHO "export LD_LIBRARY_PATH" >>$CMDF
$ECHO "$SQLCMD <<EOF" >>$CMDF
$ECHO "$SQLLOGIN" >>$CMDF
$ECHO "set echo on;" >>$CMDF
$ECHO "alter system checkpoint;" >>$CMDF
$ECHO "alter system switch logfile;" >>$CMDF
$ECHO "alter database backup controlfile to '${ORACLE_CNTRL}' reuse;" >>$CMDF
$ECHO "exit;" >>$CMDF
$ECHO "EOF" >>$CMDF

/bin/su $ORACLE_DBA -c "/bin/sh $CMDF" >>$WORKF 2>>$WORKF

$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME Database checkpoint command output:" >>$OUTF
$ECHO "" >>$OUTF

/bin/cat $WORKF >>$OUTF

# Look for errors of the form "ORA-00000".

STAT=`egrep "(ORA|DBA|SQL|LCC|MGR)-[0-9]" $WORKF|wc -l`

if [ ${STAT} -eq 0 ]
then
	$ECHO "" >>$OUTF
	$ECHO "`$TIME` $POLICYNAME Database ${ORACLE_SID} checkpointed." >>$OUTF
else
	$ECHO "" >>$OUTF
	$ECHO "`$TIME` $POLICYNAME Could not checkpoint database ${ORACLE_SID}." >>$OUTF
fi

$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME -------------------------" >>$OUTF
$ECHO "`$TIME` $POLICYNAME checkpoint_database() END" >>$OUTF
$ECHO "`$TIME` $POLICYNAME -------------------------" >>$OUTF
$ECHO "" >>$OUTF
}

# -----------------------------------------------------------------------------
# Exit the script.
# -----------------------------------------------------------------------------

abort()
{
$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME END bpend_notify exit status 1" >>$OUTF
$ECHO "" >>$OUTF

# Make sure the error file is created so that other streams know of problems.

/bin/touch "${ERROR_FILE}"
/bin/chmod 644 "${ERROR_FILE}"

if [ "${MAIL_ADDR_FAILURE}" != "xxxxxx" ]
then
	# Mail the output to someone.
	# CAUTION: Some platforms do not allow the -s parameter on mail.

	if [ $SUBJECT -eq 1 ]
	then
		cat $OUTF | $MAIL -s "${POLICYNAME} bpend_notify exit 1" $MAIL_ADDR_FAILURE
	else
		cat $OUTF | $MAIL $MAIL_ADDR_FAILURE
	fi
fi

exit 1
}



# -----------------------------------------------------------------------------
# Main script starts here.
# -----------------------------------------------------------------------------

ECHO=/bin/echo

if [ -x /usr/bin/mailx ]
then
	MAIL=/usr/bin/mailx
	SUBJECT=1
elif [ -x /usr/ucb/mail ]
then
	MAIL=/usr/ucb/mail
	SUBJECT=1
elif [ -x /usr/bin/mail ]
then
	MAIL=/usr/bin/mail
	SUBJECT=0
else
	MAIL=/bin/mail
	SUBJECT=0
fi

if [ "$#" -ne 5 ]
then
	$ECHO "`$TIME` $POLICYNAME Expected 5 parameters, but got $# instead." >>$OUTF
	abort
fi

# -----------------------------------------------------------------------------
# Assign names to our parameters.
# -----------------------------------------------------------------------------

CLIENTNAME=$1
POLICYNAME=$2
SCHEDNAME=$3
SCHEDTYPE=$4
STATUS=$5

# -----------------------------------------------------------------------------
# Create some variables using the environment variables from bpbkar.
# -----------------------------------------------------------------------------

CONTROL_DIR=/usr/openv/netbackup/bin/BLOCK_INCR

SHUTDOWN_BKUP_RESTART_FILE=${CONTROL_DIR}/SHUTDOWN_BKUP_RESTART.${ORACLE_SID}
SHUTDOWN_CKPT_RESTART_FILE=${CONTROL_DIR}/SHUTDOWN_CKPT_RESTART.${ORACLE_SID}
ALTER_TABLESPACE_FILE=${CONTROL_DIR}/ALTER_TABLESPACE.${ORACLE_SID}
NODATA_CKPT_HOT_FILE=${CONTROL_DIR}/NODATA_CKPT_HOT.${ORACLE_SID}

SHUTDOWN_FILE=${CONTROL_DIR}/shutdown.${ORACLE_SID}
BEGIN_BACKUP_FILE=${CONTROL_DIR}/begin_backup.${ORACLE_SID}

START_FILE=${CONTROL_DIR}/start.${ORACLE_SID}
START_LINK=${CONTROL_DIR}/start.${ORACLE_SID}.${POLICYNAME}

POST_CKPT_FILE=${CONTROL_DIR}/post_ckpt.${ORACLE_SID}
POST_CKPT_LINK=${CONTROL_DIR}/post_ckpt.${ORACLE_SID}.${POLICYNAME}

END_FILE=${CONTROL_DIR}/end.${ORACLE_SID}
END_LINK=${CONTROL_DIR}/end.${ORACLE_SID}.${POLICYNAME}

TABLES_FILE=${CONTROL_DIR}/tablespaces.${ORACLE_SID}
ERROR_FILE=${CONTROL_DIR}/error.${KEYWORD}
WORKF=${CONTROL_DIR}/db_cmd_output.${ORACLE_SID}
OUTF=${CONTROL_DIR}/bpend_notify_output.${ORACLE_SID}
BPSTART_OUTF=${CONTROL_DIR}/bpstart_notify_output.${ORACLE_SID}
POST_CKPT_OUTF=${CONTROL_DIR}/post_checkpoint_notify_output.${ORACLE_SID}
CMDF=${CONTROL_DIR}/ora_cmd.${ORACLE_SID}

BKUP=/usr/openv/netbackup/bin/bpbackup

TIME='/bin/date +%T'

# -----------------------------------------------------------------------------
# Make sure there are no files left around from system crashes.
# -----------------------------------------------------------------------------

/bin/rm -f ${START_FILE}*
/bin/rm -f ${POST_CKPT_FILE}*

if [ ! -d $CONTROL_DIR ]
then
	/bin/mkdir $CONTROL_DIR
	/bin/chmod 1777 $CONTROL_DIR
fi

if [ "${SCHEDTYPE}" = "UBAK" -o $STREAMS -eq 0 ]
then
	# If this is not a block incremental, exit now.
	exit 0
fi

if [ ! -f $OUTF ]
then
	/bin/touch $OUTF
	/bin/chmod 644 $OUTF
fi

$ECHO "" >>$OUTF
$ECHO "===============================================================================" >>$OUTF
$ECHO "`date` $POLICYNAME START bpend_notify" >>$OUTF
$ECHO "===============================================================================" >>$OUTF
$ECHO "" >>$OUTF

$ECHO "`$TIME` $POLICYNAME   STATUS           = ${STATUS}" >>$OUTF
$ECHO "" >>$OUTF

$ECHO "`$TIME` $POLICYNAME   BACKUPID         = ${BACKUPID}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   UNIXBACKUPTIME   = ${UNIXBACKUPTIME}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   BACKUPTIME       = ${BACKUPTIME}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   CLIENTNAME       = ${CLIENTNAME}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   POLICYNAME        = ${POLICYNAME}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   SCHEDNAME        = ${SCHEDNAME}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   SCHEDTYPE        = ${SCHEDTYPE}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   KEYWORD          = ${KEYWORD}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   STREAMS          = ${STREAMS}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   BPSTART_TIMEOUT  = ${BPSTART_TIMEOUT}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   BPEND_TIMEOUT    = ${BPEND_TIMEOUT}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   POLICY_IN_CONTROL = ${POLICY_IN_CONTROL}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   ORACLE_DBA       = ${ORACLE_DBA}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   ORACLE_BASE      = ${ORACLE_BASE}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   ORACLE_HOME      = ${ORACLE_HOME}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   ORACLE_LOGS      = ${ORACLE_LOGS}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   ORACLE_SID       = ${ORACLE_SID}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   ORACLE_INIT      = ${ORACLE_INIT}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   ORACLE_CONFIG    = ${ORACLE_CONFIG}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   ORACLE_CNTRL     = ${ORACLE_CNTRL}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   SQLCMD           = ${SQLCMD}" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   SQLLOGIN         = ${SQLLOGIN}" >>$OUTF

$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME   LD_LIBRARY_PATH  = ${LD_LIBRARY_PATH}" >>$OUTF
$ECHO "" >>$OUTF

if [ "$POLICYNAME" = "$POLICY_IN_CONTROL" ]
then
	if [ -f $SHUTDOWN_BKUP_RESTART_FILE ]
	then
		$ECHO "`$TIME` $POLICYNAME   METHOD           = SHUTDOWN_BKUP_RESTART" >>$OUTF
	elif [ -f $SHUTDOWN_CKPT_RESTART_FILE ]
	then
		$ECHO "`$TIME` $POLICYNAME   METHOD           = SHUTDOWN_CKPT_RESTART" >>$OUTF
	elif [ -f $ALTER_TABLESPACE_FILE ]
	then
		$ECHO "`$TIME` $POLICYNAME   METHOD           = ALTER_TABLESPACE" >>$OUTF
	elif [ -f $NODATA_CKPT_HOT_FILE ]
	then
		$ECHO "`$TIME` $POLICYNAME   METHOD           = NODATA_CKPT_HOT" >>$OUTF
	else
		$ECHO "`$TIME` $POLICYNAME   METHOD           = UNKNOWN" >>$OUTF
	fi

	$ECHO "" >>$OUTF
fi

if [ "$ORACLE_DBA" = "xxxxxx" ]
then
	$ECHO "`$TIME` $POLICYNAME The ORACLE_DBA variable is not set" >>$OUTF
	abort
fi

if [ "$POLICY_IN_CONTROL" = "xxxxxx" ]
then
	$ECHO "`$TIME` $POLICYNAME The POLICY_IN_CONTROL variable is not set" >>$OUTF
	abort
fi

if [ "$ORACLE_BASE" = "xxxxxx" ]
then
	$ECHO "`$TIME` $POLICYNAME The ORACLE_BASE variable is not set" >>$OUTF
	abort
fi

if [ "$ORACLE_HOME" = "xxxxxx" ]
then
	$ECHO "`$TIME` $POLICYNAME The ORACLE_HOME variable is not set" >>$OUTF
	abort
fi

if [ "$SQLCMD" = "xxxxxx" ]
then
	$ECHO "`$TIME` $POLICYNAME The SQLCMD variable is not set" >>$OUTF
	abort
fi

if [ "$SQLLOGIN" = "xxxxxx" ]
then
	$ECHO "`$TIME` $POLICYNAME The SQLLOGIN variable is not set" >>$OUTF
	abort
fi

if [ "${ORACLE_INIT}" != "" -a ! -f "${ORACLE_INIT}" ]
then
	$ECHO "`$TIME` $POLICYNAME ORACLE_INIT ${ORACLE_INIT} not found" >>$OUTF
	abort
fi

if [ "$POLICYNAME" = "$POLICY_IN_CONTROL" -a "$ORACLE_LOGS" = "xxxxxx" ]
then
	$ECHO "`$TIME` $POLICYNAME The ORACLE_LOGS variable is not set" >>$OUTF

	# If you don't want this to be fatal, comment out the abort call here.
	abort

	$ECHO "" >>$OUTF
fi

if [ "$POLICYNAME" = "$POLICY_IN_CONTROL" ]
then
	if [ -f $END_FILE ]
	then
		# If the control file already exists, remove it along with
		# the link files from all the streams.
		/bin/rm -f ${END_FILE}* ${END_LINK}*
	fi
	/bin/touch $END_FILE
	/bin/chmod 644 $END_FILE
else
	/bin/rm -f $END_LINK

	# Wait for the control file to be created by the "policy in control"
	# before continuing.

	SLEEP_COUNT=0

	while [ ! -f $END_FILE ]
	do
		/bin/sleep 1
		SLEEP_COUNT=`expr $SLEEP_COUNT + 1`

		if [ $SLEEP_COUNT -gt 7200 ]
		then
			# If we hit 2 hours, create the error file ourselves
			# so that we don't wait forever.

			$ECHO "`$TIME` $POLICYNAME Exceeded 7200 second wait count." >>$OUTF
			$ECHO "`$TIME` $POLICYNAME Creating error file $ERROR_FILE" >>$OUTF
			/bin/touch "${ERROR_FILE}"
			/bin/chmod 644 "${ERROR_FILE}"
			abort
		elif [ $SLEEP_COUNT -gt $BPEND_TIMEOUT ]
		then
			# If we've exceeded the timeout and the policy in
			# control hasn't started yet, start looking for the
			# error file.

			if [ -f "${ERROR_FILE}" ]
			then
				$ECHO "`$TIME` $POLICYNAME Error file $ERROR_FILE detected" >>$OUTF
				$ECHO "`$TIME` $POLICYNAME while waiting for $END_FILE file" >>$OUTF
				abort
			fi
		fi
	done
fi

# -----------------------------------------------------------------------------
# Create a hard link to the END_FILE. We will use the link count to
# decide when all the streams have been started.
# -----------------------------------------------------------------------------

if [ -f $END_FILE ]
then
	/bin/ln $END_FILE $END_LINK
	LINKCOUNT=`ls -l $END_FILE|sed 's/  / /g'|sed 's/  / /g'|cut -f2 -d" "`
else
	LINKCOUNT=0
fi

while [ $STREAMS -ge $LINKCOUNT -a ! -f "${ERROR_FILE}" ]
do
	/bin/sleep 1

	if [ -f $END_FILE ]
	then
		# The "policy in control" may remove our link if
		# the "policy in control" isn't the first one started.
		/bin/ln $END_FILE $END_LINK

		LINKCOUNT=`ls -l $END_FILE|sed 's/  / /g'|sed 's/  / /g'|cut -f2 -d" "`
	else
		LINKCOUNT=0
	fi
done

if [ "$POLICYNAME" != "$POLICY_IN_CONTROL" ]
then
	if [ -f "${ERROR_FILE}" ]
	then
		$ECHO "`$TIME` $POLICYNAME Error file $ERROR_FILE detected" >>$OUTF
		$ECHO "`$TIME` $POLICYNAME while waiting for link count of $LINKCOUNT on $END_FILE file" >>$OUTF
		abort
	fi
fi

# -----------------------------------------------------------------------------
# All streams have been reached this point, so we can now continue.
# -----------------------------------------------------------------------------

$ECHO "" >>$OUTF
$ECHO "`$TIME` $POLICYNAME All streams have completed." >>$OUTF
$ECHO "" >>$OUTF

if [ "$POLICYNAME" = "$POLICY_IN_CONTROL" ]
then
	# Restart the DB if doing a cold backup.

	if [ -f $SHUTDOWN_BKUP_RESTART_FILE -a -f $SHUTDOWN_FILE ]
	then
		# Restart the database.
		restart_database
	elif [ -f $SHUTDOWN_CKPT_RESTART_FILE -a -f $SHUTDOWN_FILE ]
	then
		# The post_checkpoint_notify script wasn't called to do the
		# restart, so we need to restart the database.
		restart_database
	elif [ -f $ALTER_TABLESPACE_FILE -a -f $BEGIN_BACKUP_FILE ]
	then
		# The post_checkpoint_notify script wasn't called to do the
		# end backup, so we need to do it.
		alter_tablespace_end_backup
	elif [ -f $NODATA_CKPT_HOT_FILE -a -f $BEGIN_BACKUP_FILE ]
	then
		# The DB is in backup mode so we need to do the end backup.
		alter_tablespace_end_backup
	fi

	# We now have all the streams at the same point.
	# Now wait for others to unlink.

	if [ -f $END_FILE ]
	then
		LINKCOUNT=`ls -l $END_FILE|sed 's/  / /g'|sed 's/  / /g'|cut -f2 -d" "`
	else
		LINKCOUNT=0
	fi

	while [ $LINKCOUNT -gt 2 ]
	do
		if [ -f "${ERROR_FILE}" ]
		then
			$ECHO "`$TIME` $POLICYNAME Error file $ERROR_FILE detected" >>$OUTF
			$ECHO "`$TIME` $POLICYNAME while waiting for link count to get down to 2 on $END_FILE file" >>$OUTF
			abort
		fi

		/bin/sleep 1

		if [ -f $END_FILE ]
		then
			LINKCOUNT=`ls -l $END_FILE|sed 's/  / /g'|sed 's/  / /g'|cut -f2 -d" "`
		else
			LINKCOUNT=0
		fi
	done

	/bin/rm -f $END_LINK
	/bin/rm -f $END_FILE
else
	# This is not the "policy in control".
	/bin/sleep 3
	/bin/rm -f $END_LINK

	# Wait for the control file to be deleted by the "policy in control"
	# before continuing.

	while [ -f $END_FILE ]
	do
		if [ -f "${ERROR_FILE}" ]
		then
			$ECHO "`$TIME` $POLICYNAME Error file $ERROR_FILE detected" >>$OUTF
			$ECHO "`$TIME` $POLICYNAME while waiting for $END_FILE to be deleted" >>$OUTF
			abort
		fi

		/bin/sleep 1
	done
fi

if [ -f "${ERROR_FILE}" ]
then
	# Something went wrong

	$ECHO "`$TIME` $POLICYNAME Error file $ERROR_FILE detected" >>$OUTF
	abort
fi

# -----------------------------------------------------------------------------
# If everything is OK, backup the archive logs and control file.
# -----------------------------------------------------------------------------

if [ ! -f "${ERROR_FILE}" -a $STATUS -eq 0 ]
then
	if [ "$POLICYNAME" = "$POLICY_IN_CONTROL" ]
	then
		# Checkpoint the database and switch archive log files.

		checkpoint_database

		# Initiate a user directed backup of the control file and
		# the archive log files.

		if [ "$ORACLE_LOGS" = "xxxxxx" ]
		then
			# The ORACLE_LOGS was not specified.
			$BKUP -k "${KEYWORD}" -c $POLICY_IN_CONTROL $ORACLE_INIT $ORACLE_CONFIG $ORACLE_CNTRL
		else
			$BKUP -k "${KEYWORD}" -c $POLICY_IN_CONTROL $ORACLE_INIT $ORACLE_CONFIG $ORACLE_CNTRL $ORACLE_LOGS
		fi
	fi
fi

if [ $STATUS -ne 0 ]
then
	# Something went wrong in bpbkar. Continue using the same log files.

	$ECHO "" >>$OUTF
	$ECHO "`$TIME` $POLICYNAME Status $STATUS passed in from bpbkar" >>$OUTF
	abort
elif [ -f "${ERROR_FILE}" ]
then
	# Continue to write to the same log files.

	$ECHO "`$TIME` $POLICYNAME Error file $ERROR_FILE detected" >>$OUTF
	abort
else
	$ECHO "" >>$OUTF
	$ECHO "`$TIME` $POLICYNAME END bpend_notify exit status 0" >>$OUTF
	$ECHO "" >>$OUTF

	if [ "$POLICYNAME" = "$POLICY_IN_CONTROL" ]
	then
		# Give the other streams time to finish.

		/bin/sleep 10

		if [ "${MAIL_ADDR_SUCCESS}" != "xxxxxx" ]
		then
			# Mail the output to someone.
			# CAUTION: Some platforms do not allow the -s parameter on mail.

			if [ $SUBJECT -eq 1 ]
			then
				cat $OUTF | $MAIL -s "${POLICYNAME} bpend_notify exit 0" $MAIL_ADDR_SUCCESS
			else
				cat $OUTF | $MAIL $MAIL_ADDR_SUCCESS
			fi
		fi

		# Save 5 cycles of log files.

		/bin/rm -f ${OUTF}.old.5
		/bin/rm -f ${BPSTART_OUTF}.old.5
		/bin/rm -f ${POST_CKPT_OUTF}.old.5

		if [ -f ${OUTF}.old.4 ]
		then
			/bin/mv ${OUTF}.old.4 ${OUTF}.old.5
			/bin/mv ${BPSTART_OUTF}.old.4 ${BPSTART_OUTF}.old.5
			/bin/mv ${POST_CKPT_OUTF}.old.4 ${POST_CKPT_OUTF}.old.5
		fi

		if [ -f ${OUTF}.old.3 ]
		then
			/bin/mv ${OUTF}.old.3 ${OUTF}.old.4
			/bin/mv ${BPSTART_OUTF}.old.3 ${BPSTART_OUTF}.old.4
			/bin/mv ${POST_CKPT_OUTF}.old.3 ${POST_CKPT_OUTF}.old.4
		fi

		if [ -f ${OUTF}.old.2 ]
		then
			/bin/mv ${OUTF}.old.2 ${OUTF}.old.3
			/bin/mv ${BPSTART_OUTF}.old.2 ${BPSTART_OUTF}.old.3
			/bin/mv ${POST_CKPT_OUTF}.old.2 ${POST_CKPT_OUTF}.old.3
		fi

		if [ -f ${OUTF}.old.1 ]
		then
			/bin/mv ${OUTF}.old.1 ${OUTF}.old.2
			/bin/mv ${BPSTART_OUTF}.old.1 ${BPSTART_OUTF}.old.2
			/bin/mv ${POST_CKPT_OUTF}.old.1 ${POST_CKPT_OUTF}.old.2
		fi

		/bin/mv ${OUTF} ${OUTF}.old.1
		/bin/mv ${BPSTART_OUTF} ${BPSTART_OUTF}.old.1
		/bin/mv ${POST_CKPT_OUTF} ${POST_CKPT_OUTF}.old.1
	fi

	exit 0
fi
