#!/bin/ksh
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# rspc720 src/rspc/adde/v2/user/methods/hw_prod/mlxent/mlxdebug.sh 1.1 
#  
# Licensed Materials - Property of IBM 
#  
# Restricted Materials of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2015 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)89  1.1  src/rspc/adde/v2/user/methods/hw_prod/mlxent/mlxdebug.sh, pcimlxent, rspc720, 1522A_720 5/6/15 15:28:00
#   COMPONENT_NAME: pcimlxent 
#
#               
#
#   ORIGINS: 27
#
#
# FILE NAME: mlxdebug 
#
# FILE DESCRIPTION: High-level shell command to collect debug information
# for the PCIe2 RoCE Converged Network Adapter   
#
#   See Usage message for explanation of parms
#       --help              # This message...
#       --output            # Output directory.
#       --no_trace_decode   # Do not decode traces.
#
#
# RETURN VALUE DESCRIPTION:
#                             0         Successful
#                             non-zero  Unsuccessful
#
#
#
#
# GLOBAL VARIABLES: none
#

#set -x #debug output

cmds=$*   # All input parameters as a single string
nargs=$#  # Number of arguments passed in

# debug
#-------------
#echo ${cmds}
#echo ${nargs}


# Set the environmetal variables
DNAME=mlx
DEVICE=${DNAME}ent
DRIVER=${DEVICE}dd
BASEFILE=${DEVICE}.output
INPUTDIR=/tmp/ibmsupt

separator() {
    print "============================================================" >> ${OUTFILE}
} # separator()

banner_message() {
    print "\n\n\n" >> ${OUTFILE}
    separator
    banner $BANMSG >> ${OUTFILE}
    separator
} # banner_message()

get_ofdt() {
    LOCALFILE=${OUTDIR}/ofdt.out
    # prefix the file contents with the following string to trigger 
    # automatic syntax highlighting in text viewers such as gvim
    echo "#!/bin/sh" > ${LOCALFILE}
    errpt -a >> ${LOCALFILE}
} # get_ofdt()

get_errorlog() {
    BANMSG="ERRORS"
    banner_message

    cmd="errpt"
    exec_command

    LOCALFILE=${OUTDIR}/errpt-a
    # prefix the file contents with the following string to trigger 
    # automatic syntax highlighting in text viewers such as gvim
    echo "#!/bin/sh" > ${LOCALFILE}
    errpt -a >> ${LOCALFILE}
} # get_errorlog()

exec_command() {
    print "\n"     >> ${OUTFILE}
    separator
    print "# $cmd" >> ${OUTFILE}
    $cmd           >> ${OUTFILE}
} # exec_command()

get_system_info() {
    BANMSG="SYSTEM"
    banner_message

    cmd="emgr -l"
    exec_command

    cmd="hostname"
    exec_command

    cmd="lsconf"
    exec_command

    cmd="date"
    exec_command

    cmd="uptime"
    exec_command
    
    cmd="uname -a"
    exec_command

    cmd="uname -M"
    exec_command

    cmd="cat /usr/lpp/bos/aix_release.level"
    exec_command

    cmd="oslevel"
    exec_command

    cmd="oslevel -r"
    exec_command

    cmd="bootinfo -r"
    exec_command

    cmd="bosdebug -L"
    exec_command
    
    cmd="mount"
    exec_command

    cmd="lsattr -El proc0"
    exec_command

    cmd="lsattr -El mem0"
    exec_command

    cmd="lsattr -El sys0 -a realmem"
    exec_command
    
    cmd="vmstat"
    exec_command

    cmd="lsps -a"
    exec_command
    
    cmd="df -m"
    exec_command

    cmd="env"
    exec_command
    echo "\"" >> ${OUTFILE} # syntax highlight workaround '
} # get_system_info()

get_network_info() {
    BANMSG="DEVICES"
    banner_message

    cmd="lsdev -Cc adapter"
    exec_command

    cmd="lscfg -vl ${ENT}"
    exec_command

    cmd="lsattr -El ${ENT}"
    exec_command
    echo "\ndo" >> ${OUTFILE} # syntax highlight workaround '

    # dump the driver and auxiliary binaries file info
    TARGETS="/usr/lib/drivers/pci/${DRIVER}
             /usr/lib/lib${DNAME}-rdmav2.so
             /usr/lib/methods/cfg${DEVICE}
             /usr/bin/entstat.${DEVICE}"
    for cnt in ${TARGETS}; do
        TFILE=${cnt}
        cmd="cksum ${TFILE}"
        exec_command

        cmd="what ${TFILE}"
        exec_command
    done

    # display the component trace settings for network devices
    cmd="ctctrl -q -t network_ethernet -r"
    exec_command

    cmd="netstat -in"
    exec_command

    # get configured interfaces.  This covers both enX and etX types.
    INUMBER=`echo ${ENT}  | cut -c 4-6`
    IFACE=`netstat -in | egrep -i e[n,t]${INUMBER} | egrep -v link  | awk '{print $1}'`

    for cnt in ${IFACE}; do
        INTERFACE=${cnt}
        cmd="lsattr -El ${INTERFACE}"
        exec_command
    done

    cmd="ibv_devices"
    exec_command

    cmd="ibv_devinfo"
    exec_command

    cmd="ibv_devinfo -v"
    exec_command

} # get_network_info()

get_statistics() {
    BANMSG="STATISTICS"
    banner_message

    cmd="entstat -d ${ENT}"
    exec_command

    cmd="netstat -s -s" # only display non-0 counters for readability
    exec_command

    cmd="netstat -s" 
    exec_command
    echo "'" >> ${OUTFILE} # syntax highlight workaround '

    cmd="netstat -m"
    exec_command
} # get_statistics()

ctrace_decode() {
    # Skip trace decoding if requested
    if [[ $TRACE_DECODE -ne 0 ]]; then
        # Directory to contain RAW trace buffers
        RAWDIR=${OUTDIR}/ctrace_raw
        mkdir -p ${RAWDIR}
    
        # dump the raw trace buffers
        INSTANCE=${DRIVER}.dd.${ENT}
        ctctrl -D -c ${INSTANCE} -r -d  ${RAWDIR}
    
        # Directory to contain decoded trace files
        CTRACEDIR=${OUTDIR}/ctrace_decode
        mkdir -p ${CTRACEDIR}
    
        # Decode each tracefile found
        RAWLIST=`ls ${RAWDIR} | egrep -v trc_ct.master`
        for cnt in ${RAWLIST}; do
            RFILE=${cnt}
            DFILE=${CTRACEDIR}/${RFILE}.decode
    
            # syntax highlight workaround
            echo "#!/bin/sh" > ${DFILE} 
            trcrpt ${RAWDIR}/${RFILE} >> ${DFILE}
        done
    
        # Coalesce all traces into one sequential decoded file
        COALESCEDFILE=${CTRACEDIR}/ctrace.${ENT}.complete.decode
        echo "#!/bin/sh"       >  ${COALESCEDFILE}
        trcrpt -m ${RAWDIR}/* >>  ${COALESCEDFILE}
    
        #XXX: Need to implement a complete ENT only coalesced file
        #COALESCEDFILE=${CTRACEDIR}/ctrace.${ENT}.complete.ent.decode
        #echo "#!/bin/sh"       >  ${COALESCEDFILE}
    
        #XXX: Need to implement a complete RDMA only coalesced file
        #COALESCEDFILE=${CTRACEDIR}/ctrace.${ENT}.complete.rdma.decode
        #echo "#!/bin/sh"       >  ${COALESCEDFILE}
    fi
} # ctrace_decode()

results_to_tar() {
    TARFILE=${DEVICE}.debug.tar
    # tar up the files and redirect output messages to avoid information
    # overload on the console.
    tar -cvf ${BASEDIR}/${TARFILE}  ${BASEDIR} > /dev/null
    print "\n"
    print "Compressed Files:"
    print "================="
    print "Data location: $INPUTDIR"
    ls -lF ${BASEDIR} | egrep ${TARFILE}
} # results_to_tar()

display_output_files() {
    print "\n"
    print "Adapters Processed:"
    print "==================="
    pwd
    ls -lF ${BASEDIR}/*
    print "\n"
} # display_output_files()


display_usage() {
    print "=================================================================="
    print "Invalid input parameters: $cmds"
    print ""
    print "Sample: mlxdebug.sh --output /tmp "
    print ""
    print "display_usage:"
    print "   --help              # This message..."
    print "   --output            # Output directory."
    print "   --no_trace_decode   # Do not decode traces."
    print "=================================================================="
    print " "
} # display_usage()

config_options() {
    # Iterate over the input parameters/options
    TRACE_DECODE=1
    EXPECTING_DIR=0

    for i in $cmds; do
        if [[ ${EXPECTING_DIR} -eq 1 ]]; then 
            INPUTDIR="$i"
            EXPECTING_DIR=0
            continue
        fi

        case $i in
            --output)
                EXPECTING_DIR=1
                ;;
            --no_trace_decode)
                TRACE_DECODE=0  # disable decoding of traces for speed
                ;;
            "") 
                # No input parameters:
                # Do nothing, this will walk all of the directories 
                # Searching for files to build..
                display_usage
                echo ""
                exit 1111
                ;;
            *|-help|-h|--help|-?)
                # default case, invalid input or help...
                display_usage
                echo ""
                exit 2222
                ;;
        esac
    done
    
    # set our base working directory for all debug data
    BASEDIR=${INPUTDIR}/${DNAME}debug

} # config_options()

main() {
    set -x
    config_options

    for HBA in $(lsdev -C|grep RoCE|grep hba|awk '{print $1}'); do
        ENTS=`lsdev -p ${HBA} | awk '{print $1}'`
        for PORT in ${ENTS}; do
            # Filter out roceX devices since they are specific to AIXIB which
            # is outside of the scope of this debug tool
            echo ${PORT} | egrep -i "roce"
            if [[ $? -eq 0 ]]; then
                continue
            fi

            # Set the environmental varaible specific to this ENT
            ENT=${PORT}
    
            # for each adapter instance create a dedicated debug directory
            OUTDIR=${BASEDIR}/${HBA}/${ENT}/
            mkdir -p ${OUTDIR}
    
            # create the device specific output file
            OUTFILE=${OUTDIR}/${BASEFILE}
    
            # prefix the file contents with the following string to trigger 
            # automatic syntax highlighting in text viewers such as gvim
            echo "#!/bin/sh" > ${OUTFILE}
    
            # the following functions make use of the specific adapter 
            # variables define above.
            ctrace_decode
            get_network_info
            get_statistics
            get_system_info
            get_ofdt       # new file
            get_errorlog   # new file
        done
    done

    # display the adapters processed
    display_output_files

    # tar it all up for easy service/transport and display location to
    # standard out
    results_to_tar
}
main "$@"