#!/bin/ksh
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# bos72F src/bos/usr/sbin/triton/scripts/snap_triton.sh 1.3 
#  
# Licensed Materials - Property of IBM 
#  
# Restricted Materials of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2014,2017 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 

# @(#)01        1.1  src/bos/usr/sbin/triton/scripts/snap_triton.sh, dascache, bos720, 1510A_720 12/19/14 14:16:51

###############################################################################
# Name:         snap_triton
#
# Usage:        Invoked from command line to collect & compress triton logs,
#               config info, and dumps, compressing them into a tarball
#
#
# Change History:
#   Modifier               Date                      Change ID
#   -------                ----                      ---------
#   Juan J. Ruiz           11/14/2013                290086
#      Initial creation
#
###############################################################################

#set -x
# GLOBALS

if [ $# -le 0 ]
then
   echo "Usage: $0 <DumpDirectoryPath>"
   exit 1
fi
DUMPDIR="$1/pfc_snap"
SNAPDIR="$1"
SNAPPKG=triton_snap_${HOST}_$$
PFCDLOG="/var/adm/ras/pfc/pfcd*"
CMPTRACEDIR="$DUMPDIR/pfc_cmptrace"

if [[ ! -e ${SNAPDIR} ]] ; then
   mkdir -p ${SNAPDIR}
fi

if [[ ! -e ${DUMPDIR} ]] ; then
   mkdir -p ${DUMPDIR}
fi

if [[ ! -e ${CMPTRACEDIR} ]] ; then
   mkdir -p ${CMPTRACEDIR}
fi

MAX_OLD_LOGS=8
MIN_OLD_LOGS=4

ls -l /dev/pfcdd* > /dev/null 2>&1
if [ $? = 0 ]
then
      ETCDD_STATE="Available"
else
      ETCDD_STATE="NotAvailable"
fi     

get_pfc_levels () {
   printf "------------------------------------------------------------------\n"
   echo "Levels of bos.pfcdd.rte" 
   echo "lslpp -l bos.pfcdd.rte"
   lslpp -l bos.pfcdd.rte
   printf "------------------------------------------------------------------\n\n"
   
   printf "------------------------------------------------------------------\n"
   echo "ls -l /dev/pfcdd*"
   ls -l /dev/pfcdd*
   printf "------------------------------------------------------------------\n\n"

   printf "------------------------------------------------------------------\n"
   echo "lssrc -s pfcdaemon"
   lssrc -s pfcdaemon
   printf "------------------------------------------------------------------\n\n" 
}

get_err_info () {

errpt > ${DUMPDIR}/errpt_summary
errpt -a > ${DUMPDIR}/errpt_detail

}
get_triton_kdb_info () {

## NOTE: This needs to be optimized.  Once we have lists of thread slots, we
## should invoke kdb with a list of thread slots, otherwise there is a separate
## kdb invocation for every single thread.... takes 5 mins to run
   echo "oslevel info"
   oslevel
   oslevel -r

## collect thread lists from kdb
   URHth=`echo "th -n URH"|kdb|grep URH|grep pvthread`
   PDMth=`echo "th -n pdm"|kdb|grep pdm|grep pvthread`
   TSCth=`echo "th -n ttddScan"|kdb|grep ttddScan|grep pvthread`
   TSSth=`echo "th -n ttStateS"|kdb|grep ttStateS|grep pvthread`
   TCAth=`echo "th -n tt-cache"|kdb|grep tt-cache|grep pvthread`
   CDRth=`echo "th -n cdir_asy"|kdb|grep cdir_asy|grep pvthread`
   ETCth=`echo "th -n etc"|kdb|grep etc|grep pvthread`

## display kernext threads summary
   echo "URH Kernel Threads: \n$URHth"
   echo "PDM Kernel Threads: \n$PDMth"
   echo "TT Scan Kernel Threads: \n$TSCth"
   echo "TT Statesave Kernel Threads: \n$TSSth"
   echo "TT Cache Kernel Threads: \n$TCAth"
   echo "CDIR Kernel Threads: \n$CDRth"
   echo "ETC Kernel Threads: \n$ETCth"

## display stack for each thread
   echo "Detail info for each thread:"
   TRIth=$URHth"\n"$PDMth"\n"$TSCth"\n"$TSSth"\n"$TCAth"\n"$CDRth"\n"$ETCth
   ifs="$IFS"
IFS='
'
   for thread in `echo "$TRIth"` ; do 
      tid=`echo "$thread" | awk '{print $5}'`
      slot=`echo "$thread" | awk '{print $2}'`
      name=`echo "$thread" | awk '{print $3}'` 
      echo "TID:$tid SLOT:$slot NAME:$name"
      stack=`echo "f $slot" | kdb -script | egrep "\\[|\\]|STACK|$slot"`
      echo "$stack"
   done

   IFS="$ifs"
## display deadlock info
   echo "WARNING: kdb dla command may not present all deadlocks present in the\n system.  Use this info to aid in deadlock debug, not to rule out a deadlock"

   dlaInfo=`echo dla | kdb -script`
   echo "$dlaInfo"

   echo "\nTriton threads holding locks:"
   thLock=`echo 'th -lk' | kdb -script | grep -E "URH|pdm_wq|ttddScam|tt-cache|cdor_asy|etcdaemo"`
   echo "$thLock"
}


# rm prev files
echo "Removing previous triton logs..."
rm -fr ${DUMPDIR}/* 2> /dev/null

# Prune old tarballs to manage space
cd ${SNAPDIR}
ls triton_snap*gz > /tmp/snap.$$ 2>&1
if [[ $? -eq 0 ]]; then
   # triton snaps are present
   set -A LOG_LIST `cat /tmp/snap.$$`
   echo "${#LOG_LIST[*]} Triton logs are present"

   if [[ ${#LOG_LIST[*]} -ge $MAX_OLD_LOGS ]] ; then
      integer ndx=0
      integer fcount=$MIN_OLD_LOGS
      while [ fcount -lt ${#LOGFILE_LIST[*]} ] ; do
         echo "deleting ${COREFILE_LIST[$ndx]}"
         rm ${COREFILE_LIST[$ndx]}
         ndx=$ndx+1
         fcount=$fcount+1
      done
   fi

else
   echo "No Triton snaps are currently present"
fi
rm /tmp/snap.$$

echo "Populating snap pkg..."

#1 
get_pfc_levels > ${DUMPDIR}/triton_levels

#2
get_err_info 

#3 
if [ "${ETCDD_STATE}" = "Available" ] ; then
   pfcras -a dump_statesave
else 
   echo "pfcdd in ${ETCDD_STATE} state, dump not available" > ${DUMPDIR}/pfcdump99
fi
# now copy all available dumps
echo "copying dumps..."
lastdump=`ls -tr  /var/adm/ras/etcdump* |tail -1` 
echo "Latest dump is $lastdump" >  ${DUMPDIR}/latest_dump
cp /var/adm/ras/etcdump* ${DUMPDIR}
echo "copying pfcd.log..."
	cp ${PFCDLOG} ${DUMPDIR}

	#4 
	echo "collecting triton client config..."
	if [ "${ETCDD_STATE}" = "Available" ] ; then
	   echo "\npfcras -a dump_stats:\n" >> ${DUMPDIR}/triton_config
	   pfcras -a dump_stats >> ${DUMPDIR}/triton_config 2>&1 
	   sleep 1; sync;
	   echo "pfcras -a dump_dir:" > ${DUMPDIR}/triton_dump_dir
	   ( pfcras -a dump_dir >> ${DUMPDIR}/triton_dump_dir 2>&1 ) &
	   sleep 1; sync;
           echo "collecting component traces ..."
           /usr/sbin/ctctrl -r -D -d "${CMPTRACEDIR}" -c triton_pfcdd 
           /usr/sbin/ctctrl -r -D -d "${CMPTRACEDIR}" -c triton_reg64 
           /usr/sbin/ctctrl -r -D -d "${CMPTRACEDIR}" -c triton_urh
           /usr/sbin/ctctrl -r -D -d "${CMPTRACEDIR}" -c ttcache64
           /usr/sbin/ctctrl -r -D -d "${CMPTRACEDIR}" -c etcdd        
	else
	   echo "pfcdd in ${ETCDD_STATE} state, config not available" > ${DUMPDIR}/triton_config
	fi

	#5
	if [ "${ETCDD_STATE}" = "Available" ] ; then
	   echo "collecting triton thread info..."
	   get_triton_kdb_info >  ${DUMPDIR}/triton_kdb_info
	fi

	cd ${SNAPDIR}
	echo "Creating tarball ${SNAPDIR}/${SNAPPKG}.tar.gz ..."
	tar -cf ${SNAPDIR}/${SNAPPKG}.tar ./pfc_snap 
	gzip ${SNAPDIR}/${SNAPPKG}.tar  
	cd -
rm -rf ${SNAPDIR}/pfc_snap