#!/usr/bin/perl
#  ALTRAN_PROLOG_BEGIN_TAG                                                    
#  This is an automatically generated prolog.                                  
#                                                                              
#  Copyright (C) Altran ACT S.A.S. 2019,2021.  All rights reserved.  
#                                                                              
#  ALTRAN_PROLOG_END_TAG                                                      
#                                                                              
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
# 61haes_r714 src/43haes/usr/sbin/cluster/utilities/clavan.sh 1.15 
#  
# Licensed Materials - Property of IBM 
#  
# Restricted Materials of IBM 
#  
# COPYRIGHT International Business Machines Corp. 2001,2007 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# $Id$
# @(#)  1f0ef5a 43haes/usr/sbin/cluster/utilities/clavan.sh, 726, 2147A_aha726, Sep 02 2021 01:46 AM 
######################################################################
#                                                                    #
# Module: clavan                                                     #
#                                                                    #
# Purpose:                                                           #
#   clavan - Application Availability Analysis tool                  #
#                                                                    #
# Syntax:                                                            #
# To query the current date and time:                                #
#   clavan -c                                                        #
#                                                                    #
# To display uptime statistics for an application:                   #
#   clavan -a appname -b begin_dt -e end_dt                          #
#                                                                    #
# Flags:                                                             #
#   -a appname   Specifies the name of the application whose uptime  #
#                statistics are to be displayed.                     #
#   -c           Query current date and time.  Date and time are     #
#                returned in a format suitable for use as default    #
#                values in the Application Availability Analysis     #
#                SMIT panel.  This flag may not be used with any     #
#                other flags.                                        #
#   -b begin_dt  Specifies the date and time at which analysis       #
#                should begin.                                       #
#   -e end_dt    Specifies the date and time at which analysis       #
#                should end.                                         #
#   The format of the begin_dt and end_dt arguments is as follows:   #
#       YYYY:MM:DD:hh:mm:ss                                          #
#   Where:                                                           #
#       YYYY = year                                                  #
#       MM   = month (01..12)                                        #
#       DD   = day of month (01..31)                                 #
#       hh   = hour of day (00..23)                                  #
#       mm   = minute of hour (00..59)                               #
#       ss   = second of minute (00..59)                             #
#                                                                    #
# Operands:                                                          #
#   None.                                                            #
#                                                                    #
# Description:                                                       #
#                                                                    #
# Exit Values:                                                       #
#    0 - success                                                     #
#   -1 - error                                                       #
#                                                                    #
# Examples:                                                          #
#   To analyze the time that application "myapp" was available       #
#   during during the 5-day period of Monday through Friday, May     #
#   1 through 5, in the year 2000, enter:                            #
#     clavan -a myapp -b 2000:05:01:00:00:00 \                       #
#                     -e 2000:05:06:00:00:00                         #
#                                                                    #
#--------------------------------------------------------------------#
# Inputs:                                                            #
#   stdin - command line arguments                                   #
#                                                                    #
# Outputs:                                                           #
#   stdout - formatted uptime analysis report                        #
#   stderr - any error message.                                      #
#                                                                    #
# External Ref:                                                      #
#   Commands: clgetaddr, clnodename, cl_rsh, odmget, ping            #
#   Modules:                                                         #
#   Perl library routines: Getopt::Std                               #
#                          Time::Local                               #
#                                                                    #
# Tab Settings:                                                      #
#   4 and tabs should be expanded to spaces before saving this file. #
#   in vi:  (:set ts=4  and   :%!expand -4)                          #
#                                                                    #
######################################################################

use lib '/usr/es/lib/perl/';  #Adds a directory to perl's library path
use libcl_untaint;

#--------------------------------------------------------------------#
# Included Libraries and Extensions                                  #
#--------------------------------------------------------------------#
use locale;
use Getopt::Std; 
use Time::Local;


#--------------------------------------------------------------------#
# Globals                                                            #
#--------------------------------------------------------------------#

@clumt_log = ();              # array to hold sorted/merged log records
                              # - used by clumt_get_log_file and
                              #   clumt_analyze
# command line flags
$anl_flg = "-a";              # analysis
$beg_flg = "-b";              # begin time
$qry_flg = "-c";              # query time
$end_flg = "-e";              # end time
$beg_tag = "B";               # begin array tag
$end_tag = "E";               # end array tag

# booleans
$TRUE = 1;
$FALSE = 0;

# command names and directory paths
$CMDDIR     = "/usr/es/sbin/cluster/utilities";
$TMPFILE    = "clavan_tmplog.";   # base name, ALWAYS append timestamp
                                  # to insure uniqueness
$LOGFILE    = "clavan.log";
$CLRSH      = "cl_rsh";
$CLGETADDR  = "clgetaddr";
$CLLOG      = "cllog";
$CLLSAPPMON = "cllsappmon";
$CLLSGRP    = "cllsgrp";
$CLLSRES    = "cllsres";
$CLLSSERV   = "cllsserv";
$CLNODENAME = "clnodename";
$GREP       = "grep";

# Initialize an empty array
my @hacmpmon = ();

# event name keys - used enums from cluster.h with additional entries
# for the *_complete events
$node_up          = "TE_JOIN_NODE";
$node_up_comp     = "TE_JOIN_NODE_C";
$node_dn          = "TE_FAIL_NODE";
$node_dn_comp     = "TE_FAIL_NODE_C";
$net_up           = "TE_JOIN_NETWORK";
$net_up_comp      = "TE_JOIN_NETWORK_C";
$net_down         = "TE_FAIL_NETWORK";
$net_down_comp    = "TE_FAIL_NETWORK_C";
$swap_adptr       = "TE_SWAP_ADAPTER";
$swap_adptr_comp  = "TE_SWAP_ADAPTER_C";
$jn_stdby         = "TE_JOIN_STANDBY";
$fl_stdby         = "TE_FAIL_STANDBY";
$migrate          = "TE_MIGRATE";
$migrate_comp     = "TE_MIGRATE_C";
$recfg_rsrc_rlse  = "TE_DARE_CONFIGURATION";
$recfg_rsrc_acq   = "TE_DARE_CONFIGURATION_A";
$recfg_rsrc_comp  = "TE_DARE_CONFIGURATION_C";
$srvr_restrt      = "TE_SERVER_RESTART";
$srvr_restrt_comp = "TE_SERVER_RESTART_C";
$srvr_dn          = "TE_SERVER_DOWN";
$srvr_dn_comp     = "TE_SERVER_DOWN_C";
$rg_move          = "TE_RG_MOVE";
$rg_move_acq      = "TE_RG_MOVE_A";
$rg_move_comp     = "TE_RG_MOVE_C";
$site_up          = "TE_JOIN_SITE";
$site_up_comp     = "TE_JOIN_SITE_C";
$site_dn          = "TE_FAIL_SITE";
$site_dn_comp     = "TE_FAIL_SITE_C";

# cross-reference month strings used in clavan.log file to month numbers
# used by timelocal subroutine
%mon_xref = ("Jan", 0,
             "Feb", 1,
             "Mar", 2,
             "Apr", 3,
             "May", 4,
             "Jun", 5,
             "Jul", 6,
             "Aug", 7,
             "Sep", 8,
             "Oct", 9,
             "Nov", 10,
             "Dec", 11);

# message numbers/default text
$CLUMT = "clavan";
$PNL = ".\n";
%msg_xref = (7800, "Usage:\nTo query the current time:\n\tclavan -c\
To analyze application uptime over a range of time:\
\tclavan -a app_name -b begin_time -e end_time\
where:\n\tapp_name\tname of application to be analyzed\
\tbegin_time\tstart of time range\
\tend_time\tend of time range\
The format of the begin_time and end_time arguments is as follows:\
\tYYYY:MM:DD:hh:mm:ss\n",
             7801, "The -c flag must not be specified with other command line flags.\n",
             7802, "Missing or incorrect command line data.\n",
             7803, "The time paramenters specified with the -b and -e flags\nwere not correctly formatted.\n",
             7804, "Beginning or ending time exceeded system maximum.\n",
             7805, "Ending time must be greater than beginning time.\n",
             7806, "Not able to determine cluster nodes.\n",
             7807, "Not able to get address for node ",
             7808, "Not able to contact node ",
             7809, "Not able to determine location of clavan.log file.\n",
             7810, "Not able to obtain clavan.log file size from node ",
             7811, "Not able to perform file system space check.\n",
             7812, "Not enough file system space free to perform uptime analysis.\n",
             7813, "Not able to retrieve clavan.log file from node ",
             7814, "Not able to open combined clavan.log file.\n",
             7815, "Not able to remove temporary file.\n",
             7816, "Beginning time year field is out of range (1970-2037).\n",
             7817, "Beginning time month field is out of range (01-12).\n",
             7818, "Beginning time day field is out of range (01-31).\n",
             7819, "Beginning time hour field is out of range (00-23).\n",
             7820, "Beginning time minute field is out of range (00-59).\n",
             7821, "Beginning time second field is out of range (00-59).\n",
             7822, "Ending time year field is out of range (1970-2037).\n",
             7823, "Ending time month field is out of range (01-12).\n",
             7824, "Ending time day field is out of range (01-31).\n",
             7825, "Ending time hour field is out of range (00-23).\n",
             7826, "Ending time minute field is out of range (00-59).\n",
             7827, "Ending time second field is out of range (00-59).\n",
             7828, "Application name specified is not recognized.\n",
             7829, "Not able to obtain resource information.\n",
             7830, "Analysis begins:",
             7831, "Analysis ends:",
             7832, "Total time:",
             7833, "Uptime:",
             7834, "Downtime:",
             7835, "Amount:",
             7836, "Percentage:",
             7837, "Longest period:",
             7838, "Log records terminated before the specified ending time was reached.\n",
             7839, "Application monitoring was not active during the time period analyzed.\n",
             7840, "Application monitoring was suspended for XXXX of the time period analyzed.\n",
             7841, "Not able to obtain event data.\n",
             7842, "Not able to open recovery program file.\n",
             7843, "Possible corruption detected in event data.\n",
             7844, "Cluster services were manually restarted during the time period analyzed.\n",
             7845, "A hard node failure occurred during the time period analyzed.\n",
             7846, "Monday",
             7847, "Tuesday",
             7848, "Wednesday",
             7849, "Thursday",
             7850, "Friday",
             7851, "Saturday",
             7852, "Sunday",
             7853, "January",
             7854, "February",
             7855, "March",
             7856, "April",
             7857, "May",
             7858, "June",
             7859, "July",
             7860, "August",
             7861, "September",
             7862, "October",
             7863, "November",
             7864, "December",
             7950, "Not able to analyze application availability because both the\
start and end times specified are earlier than the first timestamp\
in the log.\n",
             7951, "Not able to analyze application availability because both the\
start and end times specified are later than the last timestamp\
in the log.\n",
             7953, "Application monitoring state was manually changed during the\ntime period analyzed.\n",
             7954, "Application monitor failed during the time period analyzed.\n",
             7955, "Application analyzed:",
             7956, "Application analyzed is part of a concurrent resource group.\n");


#--------------------------------------------------------------------#
#--------------------------------------------------------------------#
# Main Code                                                          #
#--------------------------------------------------------------------#
#--------------------------------------------------------------------#

# exit if no cmdline arguments
if (scalar(@ARGV) == 0) {
    &usage;
    exit -1;
 }

# call query or app subs if their flags are detected
if ($ARGV[0] eq $qry_flg) { exit(&clumt_query); }
if ($ARGV[0] eq $anl_flg) { exit(&clumt_app); }

# no valid cmdline arguments were given
&usage;
exit -1;

#--------------------------------------------------------------------#
# End Main Code                                                      #
#--------------------------------------------------------------------#


#--------------------------------------------------------------------#
#--------------------------------------------------------------------#
# Subroutines                                                        #
#--------------------------------------------------------------------#
#--------------------------------------------------------------------#

#--------------------------------------------------------------------#
# clumt_query                                                        #
#                                                                    #
# Description:                                                       #
#   This routine parses the argument vector to confirm that only the #
#   -c flag is present.  If this is not the case, the usage message  #
#   is issued and this routine exits with an error value.            #
#   Otherwise, the current date and time is obtained and formatted   #
#   as a colon-delimited list.  Finally, a colon-delimited list of   #
#   column headings is displayed on stdout, followed by the          #
#   previously formatted date and time list.                         #
#                                                                    #
# Inputs:                                                            #
#   @ARGV     ARGV array (global)                                    #
#                                                                    #
# Outputs:                                                           #
#   Current time in SMIT command-to-discover format:                 #
#       #ub_year:ub_month:ub_day:ub_hour:ub_min:ub_sec               #
#       YYYY:MM:DD:hh:mm:ss                                          #
#                                                                    #
# Return values:                                                     #
#    0        success                                                #
#   -1        error                                                  #
#--------------------------------------------------------------------#
sub clumt_query
{
    # return w/error if there are extra cmdline args
    if (scalar(@ARGV) > 1) {
        system("dspmsg scripts.cat 7801 '$msg_xref{7801}' $CLUMT");
        &usage;
        return -1;
    }

    # get current time
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime;

    # generate output
    print "#ub_year:ub_month:ub_day:ub_hour:ub_min:ub_sec\n";
    printf("%04d:%02d:%02d:%02d:%02d:%02d\n", $year + 1900, $mon + 1,
                                              $mday, $hour, $min, $sec);
    return 0;
}

#--------------------------------------------------------------------#
# clumt_app()                                                        #
#                                                                    #
# Description:                                                       #
#                                                                    #
#      This routine parses the argument vector,                      #
#      confirms that the -a flag is present                          #
#      and that the -b and -e flags and their arguments are present  #
#      and have correct syntax,                                      #
#      and that there are no other flags or arguments.               #
#      If there are any errors,                                      #
#      it calls usage() and returns with an error value.             #
#      It then calls clumt_get_log_file() and clumt_analyze().       #
#                                                                    #
#      This routine calls clumt_get_log_file()                       #
#        to get and collate the log files                            #
#      It calls clumt_analyze() to analyze the collated records      #
#        and to generate the desired report.                         #
#                                                                    #
# Inputs:                                                            #
#      @ARGV           ARGV array (global) which should contain:     #
#      -a appname      name of the application                       #
#      -b begin_dt     date/time at which analysis should begin      #
#      -e end_dt       date/time at which analysis should end        #
#                                                                    #
#      The arguments of the -b and -e flags have the following       #
#      format:                                                       #
#                                                                    #
#                      YYYY:MM:DD:hh:mm:ss                           #
#                                                                    #
#              where:  is:                                           #
#              YYYY    the 4-digit year (0000..9999)                 #
#              MM      the 2-digit month number (01...12)            #
#              DD      the 2-digit day of the month (01...31)        #
#              hh      the 2-digit hour of the day (00...23)         #
#              mm      the 2-digit minute within the hour (00...59)  #
#              ss      the 2-digit second within the minute (00...59)#
#                                                                    #
# Other inputs:                                                      #
#      Output from cllsserv command                                  #
#      The clavan.log files on all nodes in the cluster.             #
#                                                                    #
# Return values:                                                     #
#                                                                    #
#       0      success.                                              #
#      -1      error.                                                #
#--------------------------------------------------------------------#
sub clumt_app
{
    my $rc = 0;                        # return code
    my %opts = ();                     # hash for cmdline args
    my $app_name  = "";                # application name
    my $qw_app_name = "";              # app name in dbl quotes
    my $raw_btime = "";                # raw begin time arg
    my $raw_etime = "";                # raw end time arg
    my @temp_arr = ();                 # work array
    my @btime = ();                    # array of begin time components
    my @etime = ();                    # array of end time components
    my $btime_t = 0;                   # begin time in "time_t" format
    my $etime_t = 0;                   # end time in "time_t" format
    my $logfile = $TMPFILE;            # unique log file name

    #
    # parse command line arguments
    #

    # must massage ARGV array if called from SMIT, which can't give us
    # colon-delimited strings for the begin and end times - instead,
    # SMIT gives us each time unit as a separate string, so ARGV winds up
    # looking like:
    # -a <appname> -b YYYY :MM :DD :hh :mm :ss -e YYYY :MM :DD :hh :mm :ss
    if ($#ARGV == 15) {

        # concatenate beginning time units to get YYYY:MM:DD:hh:mm:ss
        $ARGV[3] .= $ARGV[4] . $ARGV[5] . $ARGV[6] . $ARGV[7] . $ARGV[8];

        # excise separate beginning time units
        splice(@ARGV, 4, 5);

        # concatenate ending time units to get YYYY:MM:DD:hh:mm:ss
        $ARGV[5] .= $ARGV[6] . $ARGV[7] . $ARGV[8] . $ARGV[9] . $ARGV[10];

        # chop off ending time units
        splice(@ARGV, 6);
    }

    if (!&getopts('a:b:e:', \%opts)) { # error if invalid args received
        system("dspmsg scripts.cat 7802 '$msg_xref{7802}' $CLUMT");
        &usage;
        return -1;
    }

    # this is needed because getopts isn't foolproof
    if (scalar(@ARGV)) {
        system("dspmsg scripts.cat 7802 '$msg_xref{7802}' $CLUMT");
        &usage;
        return -1;
    }

    # if we get this far, we know we were given correctly formatted
    # command line input:
    #     clavan -a <app name> -b <begin time> -e <end time>

    $app_name = untaint($opts{a});              # application name
    $raw_btime = $opts{b};             # beginning time
    $raw_etime = $opts{e};             # ending time


    # validate application name
    `$CMDDIR/$CLLSSERV | $GREP $app_name`;
    $rc = $?;

    if ($rc) {
        system("dspmsg scripts.cat 7828 '$msg_xref{7828}' $CLUMT");
        return -1;
    }


    #
    # parse and validate begin and end time values
    #

    # convert time strings to arrays
    @btime = split(/:/, $raw_btime);
    @etime = split(/:/, $raw_etime);

    # error if not exactly 6 elements in time arrays
    if (($#btime != 5) || ($#etime != 5)) {
        system("dspmsg scripts.cat 7803 '$msg_xref{7803}' $CLUMT");
        &usage;
        return -1;
    }

    # tag begin and end time arrays
    push(@btime, $beg_tag);
    push(@etime, $end_tag);

    # exit with error if any fields are out-of-range
    $rc = check_time_oor(\@btime);
    ($rc == 0) || return $rc;

    $rc = check_time_oor(\@etime);
    ($rc == 0) || return $rc;

    # convert begin/end times to "time_t" format and validate
    # that begin time is before ending time
    #    timelocal takes its args in reverse order: sec, min, hour,
    #    day, month, year; also must adjust numbering of year and month
    $btime_t = timelocal($btime[5], $btime[4], $btime[3],
                         $btime[2], $btime[1] - 1, $btime[0] - 1900);

    $etime_t = timelocal($etime[5], $etime[4], $etime[3],
                         $etime[2], $etime[1] - 1, $etime[0] - 1900);

    if (($btime_t < 0) || ($etime_t < 0)) {
        system("dspmsg scripts.cat 7804 '$msg_xref{7804}' $CLUMT");
        return -1;
    }

    if ($btime_t >= $etime_t) {
        system("dspmsg scripts.cat 7805 '$msg_xref{7805}' $CLUMT");
        return -1;
    }

    #
    # collate and analyze log records
    #

    # generate unique log file name
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime;
    $logfile .= $year . $mon . $mday . $hour . $min . $sec;

    # sort/merge log records from all nodes in cluster
    # - sorted/merged log records are stored in @clumt_log
    $rc = clumt_get_log_file($logfile);
    ($rc == 0) || return $rc;

    # analyze records and display results
    # - analysis will be performed on records in @clumt_log
    $rc = clumt_analyze($app_name, $btime_t, $etime_t);
    ($rc == 0) || return $rc;

    return 0;
}

#--------------------------------------------------------------------#
# clumt_get_log_file                                                 #
#                                                                    #
# Description:                                                       #
#                                                                    #
#      This routine first checks whether there will be adequate      #
#      space for local copies of the UMT log files and for the       #
#      temporary file.                                               #
#      If not, it is an error.                                       #
#      It then obtains a copy of the UMT log file from each node     #
#      in the cluster.                                               #
#      If for any reason it cannot, it is an error.                  #
#      It then sorts each local copy and removes duplicates.         #
#      If for any reason it cannot, it is an error.                  #
#      Finally, it merges the result into a single file.             #
#      If for any reason it cannot, it is an error.                  #
#                                                                    #
#      All errors result in a warning message and return             #
#      immediately with an "error" value.                            #
#                                                                    #
# Inputs:                                                            #
#                                                                    #
#      merged_log      name to use for temporary file                #
#                                                                    #
# Other inputs:                                                      #
#      Output of the following HAES utilities:                       #
#         clnodename, clgetaddr, cllog, cllrsh                       #
#      The clavan.log files on all nodes in the cluster.             #
#                                                                    #
# Return values:                                                     #
#                                                                    #
#       0      success.                                              #
#      -1      error.                                                #
#                                                                    #
# Outputs:                                                           #
#                                                                    #
#      @clumt_log global array populated with sorted and merged      #
#      clavan.log records from all nodes in cluster.                 #
#--------------------------------------------------------------------#
sub clumt_get_log_file
{
    my $merged_log = shift;     # get file name from input

    my @nodenames = ();         # list of nodes in the cluster
    my @log_info = ();          # clavan.log file info from ODM
    my @df_output = ();         # output from df command
    my @sorted_log = ();        # temporary - hold sorted log data
    my @temp_arr = ();          # work array
    my $path_index = 0;         # index of path in clavan.log ODM output
    my $log_file_path = "";     # path of clavan.log
    my $log_file = "";          # fully qualified path of clavan.log
    my $merged_log_path = "";   # path of merged log file
    my $tmp_size = "";          # temporary file size
    my $tot_file_size = 0;      # total clavan.log files size
    my $space_free = "";        # amount of free space in file system
    my $tmp_msg = "";           # for default msg text
    my $i = 0;
    my $rc = 0;


    #
    # get and validate cluster nodes
    #

    # get cluster node names
    @nodenames = `$CMDDIR/$CLNODENAME`;
    $rc = $?;
    unless ($rc == 0) {
        system("dspmsg scripts.cat 7806 '$msg_xref{7806}' $CLUMT");
        return $rc;
    }

    # lose newlines
    chomp(@nodenames);

    # verify that each node is reachable over the network
    foreach $nodename (@nodenames) {
        $nodename = untaint($nodename);
        $ipaddr = untaint(`$CMDDIR/$CLGETADDR -n $nodename`);
        $rc = $?;
        unless ($rc == 0) {
            $tmp_msg = $msg_xref{7807} . $nodename . $PNL;
            system("dspmsg scripts.cat 7807 '$tmp_msg' $CLUMT $nodename");
            return $rc;
        }

        `ping -c 1 $ipaddr`;
        $rc = $?;
        unless ($rc == 0) {
            $tmp_msg = $msg_xref{7808} . $nodename . $PNL;
            system("dspmsg scripts.cat 7808 '$tmp_msg' $CLUMT $nodename");
            return $rc;
        }
    }


    #
    # get location of clavan.log files
    #

    # get clavan.log file info from ODM
    @log_info = `$CMDDIR/$CLLOG -d $LOGFILE`;
    $rc = $?;

    if ($rc) {
        system("dspmsg scripts.cat 7809 '$msg_xref{7809}' $CLUMT");
        return -1;
    }

    # get clavan.log path info
    # format of output from cllog is #name:description:defaultdir:value:rfs
    # - we want the value field
    @temp_arr = split(/:/, $log_info[1]);
    $log_file_path = $temp_arr[3];
    $log_file = $log_file_path . "/" . $LOGFILE;


    #
    # check for sufficient space
    #

    # get clavan.log file sizes
    foreach $nodename (@nodenames) {
        $lgsz_ipaddr = untaint(`$CMDDIR/$CLGETADDR $nodename`);
        $log_file = untaint($log_file);       
        
        # if good return code from clgetaddr do rsh
        # if bad return code from clgetaddr fall through to error 
        if ($rc == 0) {
	   chomp($lgsz_ipaddr);
           $tmp_size = `$CMDDIR/$CLRSH $lgsz_ipaddr ls -s $log_file`;
           $rc = $?;
        } 
        # error if file size missing or cl_rsh failed
        if ($rc || ($tmp_size eq "")) {
            $tmp_msg = $msg_xref{7810} . $nodename . $PNL;
            system("dspmsg scripts.cat 7810 '$tmp_msg' $CLUMT $nodename");
            ($rc == 0) ? return -1 : return $rc;
        }

        # get just the file size
        $tmp_size =~ /\W+(\d+)/;
        $tmp_size = $1;
        $tot_file_size += $tmp_size;
    }

    # some breathing room
    $tot_file_size *= 2;

    # get file system statistics
    #untaint data
    $log_file_path = untaint($log_file_path);
    
    @df_output = `df -k $log_file_path`;
    $rc = $?;
    if ($rc) {
        system("dspmsg scripts.cat 7811 '$msg_xref{7811}' $CLUMT");
        return $rc;
    }

    # get file system free space
    $df_output[1] =~ /\w+\s+\w+\s+(\w+)/;
    $space_free = $1;

    # enough space?
    if ($space_free <= $tot_file_size) {
        system("dspmsg scripts.cat 7812 '$msg_xref{7812}' $CLUMT");
        return -1;
    }


    #
    # copy node clavan logs to merged log file on this node
    #

    # create merged log file path
    $merged_log_path = $log_file_path . "/" . $merged_log;

    # get node log files
    foreach $nodename (@nodenames) {
        $lgcp_ipaddr = `$CMDDIR/$CLGETADDR $nodename`;
        # if good return code from clgetaddr do rsh
        # if bad return code from clgetaddr fall through to error
        if ($rc == 0) {
	   chomp($lgcp_ipaddr);
           #untaint data
           $lgcp_ipaddr = untaint($lgcp_ipaddr);

           `$CMDDIR/$CLRSH $lgcp_ipaddr cat $log_file >> $merged_log_path`; 
           $rc = $?;
        }
        if ($rc) {
            $tmp_msg = $msg_xref{7813} . $nodename . $PNL;
            system("dspmsg scripts.cat 7813 '$tmp_msg' $CLUMT $nodename");
            return $rc;
        }
    }


    #
    # sort file and remove duplicate entries
    #

    # open/read/close file
    unless (open MERGEDLOG, $merged_log_path) {
        system("dspmsg scripts.cat 7814 '$msg_xref{7814}' $CLUMT");
        return -1;
    }
    $i = 0;
    while (<MERGEDLOG>) {
        $clumt_log[$i] = $_;
        $i++;
    }
    close(MERGEDLOG);

    # clean up temporary file
    `rm $merged_log_path`;
    $rc = $?;
    if ($rc) {
        system("dspmsg scripts.cat 7815 '$msg_xref{7815}' $CLUMT");
    }

    # convert for easier processing here and in clumt_analyze:
    # - before:  AAA: Mon Jul 9 10:30:17 2001: mnemonic:data:data: etc
    # - after:   994689017:: mnemonic:data:data: etc
    foreach $log_line (@clumt_log) {
        $log_line =~ /^AAA:\s+\w+\s+(\w+)\s+(\w+)\s+(\w+):(\w+):(\w+)\s+(\w+)(.*)/;
        $log_line = timelocal($5,$4,$3,$2,$mon_xref{$1},($6 - 1900)) . ":" . $7;
    }

    # sort the lines in the array by timestamp
    @sorted_log = sort logsort @clumt_log;

    # save sorted result in global log array, then empty temp array
    @clumt_log = @sorted_log;
    @sorted_log = ();

    # take out duplicate lines
    $i = 0;
    while ($i < $#clumt_log) {
        if ($clumt_log[$i] eq $clumt_log[$i+1]) {
            splice(@clumt_log, $i+1, 1);
        } else {
            $i++;
        }
    }

    # log is now fully processed and stored in @clumt_log
    return $rc;
}

#--------------------------------------------------------------------#
# clumt_analyze                                                      #
#                                                                    #
# Description:                                                       #
#                                                                    #
#      This routine reads and analyzes the merged and sorted log     #
#      UMT log records from all nodes in the cluster and generates   #
#      a report on on STDOUT on the uptime and downtime of the       #
#      specified application.                                        #
#                                                                    #
#      This routine uses the incoming app_name for the app server    #
#      and app monitor names, since HAES uses the app server name    #
#      as the app monitor name.                                      #
#                                                                    #
#      Accumulators for up and down times are kept for the           #
#      application, and separately for the application monitor.      #
#                                                                    #
#      The states (or mnemonics) recorded in the log file will be    #
#      "translated" to simple up/down application state changes.     # 
#                                                                    #
#      The resource group name will be determined from the app name. #
#                                                                    #
#      All errors encountered during analysis cause this routine     #
#      to issue an error message and return with a non-0 rc.         #
#                                                                    #
# Inputs: (validated in clumt_app)                                   #
#      app_name      name of application to analyze                  #
#      btime         beginning of period of interest - "time_t" fmt  #
#      etime         end of period of interest - "time_t" fmt        #
#                                                                    #
# Return values:                                                     #
#                                                                    #
#       0      success.                                              #
#      -1      error.                                                #
#                                                                    #
# Outputs:                                                           #
#                                                                    #
#      A report on STDOUT summarizing the uptime (and downtime)      #
#      of the specified application during the specified time period.#
#--------------------------------------------------------------------#
sub clumt_analyze
{
    # get application name, begin and end times from input
    my ($app_name, $btime, $etime) = @_;

    # log file state names
    my $UMS = "umtmonsus";           # monitor suspended
    my $UMR = "umtmonres";           # monitor resumed
    my $UMU = "umtmonstart";         # monitor started
    my $UMD = "umtmonstop";          # monitor stopped
    my $UMF = "umtmonfail";          # monitor failed
    my $UAS = "umtappstart";         # app server started (app "online")
    my $UAT = "umtappstop";          # app server stopped
    my $URN = "umtrgonln";           # resource group online
    my $URF = "umtrgoffln";          # resource group offline
    my $ULM = "umtlastmod";          # file last modified
    my $UNF = "umtnodefail";         # node failed
    my $UES = "umteventstart";       # cluster event started
    my $UEC = "umteventcomplete";    # cluster event completed

    # hash to hold event name strings - keys are defined in Globals
    my %ev_names = ($node_up, "",
                    $node_up_comp, "",
                    $node_dn, "",
                    $node_dn_comp, "",
                    $net_up, "",
                    $net_up_comp, "",
                    $net_down, "",
                    $net_down_comp, "",
                    $swap_adptr, "",
                    $swap_adptr_comp, "",
                    $jn_stdby, "",
                    $fl_stdby, "",
                    $migrate, "",
                    $migrate_comp, "",
                    $recfg_rsrc_rlse, "",
                    $recfg_rsrc_acq, "",
                    $recfg_rsrc_comp, "",
                    $srvr_restrt, "",
                    $srvr_restrt_comp, "",
                    $srvr_dn, "",
                    $srvr_dn_comp, "",
                    $rg_move, "",
                    $rg_move_acq, "",
                    $rg_move_comp, "",
                    $site_up, "",
                    $site_up_comp, "",
                    $site_dn, "",
                    $site_dn_comp, "");

    # keys for monitor, app state hashes
    my $up = "up";
    my $dn = "down";

    # hashes of time accumulators for up and down states
    my %mon_times = ($up, 0,        # app monitor states
                     $dn, 0);
    my %app_times = ($up, 0,        # app server states
                     $dn, 0);

    # hash of maximum up and down intervals
    my %max_times = ($up, 0,
                     $dn, 0);

    # analysis variables
    my $tot_time = $etime - $btime;   # total in time range
    my $tot_utime = 0;                # total uptime
    my $tot_dtime = 0;                # total downtime
    my $pct_uptime = 0;               # percent uptime of total
    my $pct_dntime = 0;               # percent downtime of total
    my $pct_amdtime = 0;              # percent downtime of app monitor
    my $prev_time = $btime;           # time at which current state began
    my $prev_mon_time = $btime;       # time at which current appmon state began
    my $curr_app_state = $dn;         # current application state
    my $curr_mon_state = $dn;         # current app monitor state
    my $new_app_state = $dn;          # new application state
    my $new_mon_state = $dn;          # new app monitor state
    my %nodes_up = ();                # nodes where app is online
    my $rg_name = "";                 # resource group name
    my $mon_name = "";                # app monitor name
    my $lr = "";                      # current log record
    my $lr_timestamp = 0;             # timestamp from log record
    my $lr_app_state = "";            # app state from log record
    my $lr_app_name = "";             # app name from log record
    my $lr_mon_name = "";             # app monitor name from log record
    my $lr_rg_name = "";              # resource group name from log record
    my $lr_node = "";                 # node name from log record
    my $lr_date = "";                 # date from log record (umtlastmod)
    my $man_clstr_up = 0;             # manual intervention flag
    my $node_failed = 0;              # hard node failure flag
    my $man_mon_sr = 0;               # appmon manual intervention flag
    my $mon_active = 0;               # appmon active flag
    my $mon_failed = 0;               # appmon failed
    my $concur_rg = 0;                # concurrent rg flag

    # output variables
    my $tDD = 0;                      # total days
    my $thh = 0;                      # total hours
    my $tmm = 0;                      # total minutes
    my $tss = 0;                      # total seconds
    my $tuDD = 0;                     # total days of uptime
    my $tuhh = 0;                     # total hours of uptime
    my $tumm = 0;                     # total minutes of uptime
    my $tuss = 0;                     # total seconds of uptime
    my $luDD = 0;                     # longest days of uptime
    my $luhh = 0;                     # longest hours of uptime
    my $lumm = 0;                     # longest minutes of uptime
    my $luss = 0;                     # longest seconds of uptime
    my $tdDD = 0;                     # total days of downtime
    my $tdhh = 0;                     # total hours of downtime
    my $tdmm = 0;                     # total minutes of downtime
    my $tdss = 0;                     # total seconds of downtime
    my $ldDD = 0;                     # longest days of downtime
    my $ldhh = 0;                     # longest hours of downtime
    my $ldmm = 0;                     # longest minutes of downtime
    my $ldss = 0;                     # longest seconds of downtime
    my $sec = 0;                      # seconds
    my $min = 0;                      # minutes
    my $mday = 0;                     # day of month
    my $mon = 0;                      # month
    my $year = 0;                     # year
    my $wday = 0;                     # day of week
    my $yday = 0;                     # day of year
    my $isdst = 0;                    # daylight savings flag

    # output strings used in default messages
    $DAYS  = " days, ";
    $HOURS = " hours, ";
    $MINS  = " minutes, ";
    $SECS  = " seconds";
    $NL    = "\n";
    $TB    = "\t";
    $TB2   = "\t\t";

    # will hold translated month strings for display
    my %mon_dsp = (1, "",
                   2, "",
                   3, "",
                   4, "",
                   5, "",
                   6, "",
                   7, "",
                   8, "",
                   9, "",
                   10, "",
                   11, "",
                   12, "");

    # will hold translated day strings for display
    my %day_dsp = (1, "",
                   2, "",
                   3, "",
                   4, "",
                   5, "",
                   6, "",
                   7, "");

    # temp variables
    my @temp_arr = ();                # work array
    my $temp_rg_name = "";            # work rg name
    my $temp_str = "";                # work string
    my $temp_str2 = "";               # work string
    my $time_interval = 0;            # for time calculations
    my $time = 0;                     # output loop variable
    my $first_timestamp;              # first timestamp from log
    my $last_timestamp;               # last timestamp from log
    my $i = 0;                        # counter variable
    my $j = 0;                        # counter variable
    my $k = 0;                        # counter variable
    my $arr_sz = 0;                   # work array size
    my $rc = 0;


    #
    # upfront tasks:  - check that start/end times are in range
    #                 - get resource group name
    #                 - verify app monitor definition
    #                 - get event names
    #

    # get first time stamp from log record
    $clumt_log[0] =~ /(\w+)::/;               
    $first_timestamp = $1;             

    # get last time stamp from log record
    $clumt_log[$#clumt_log] =~ /(\w+)::/;               
    $last_timestamp = $1;             

    # start and end times before first log entry
    if (($btime < $first_timestamp) && ($etime < $first_timestamp)) {
        system("dspmsg scripts.cat 7950 '$msg_xref{7950}' $CLUMT");
        return 1;
    }

    # start and end times after last log entry
    if (($btime > $last_timestamp) && ($etime > $last_timestamp)) {
        system("dspmsg scripts.cat 7951 '$msg_xref{7951}' $CLUMT");
        return 1;
    }

    # get resource groups sans newlines
    @temp_arr = `$CMDDIR/$CLLSGRP`;
    chomp(@temp_arr);

    # see if one of the rg's lists app as a resource
    foreach $temp_rg_name (@temp_arr) {
        # grepping for APPLICATIONS ensures that we don't pick up a
        # resource attr that happens to be the same as the app name

#untaint data        
        $temp_rg_name = untaint($temp_rg_name);
        
        $temp_str = `$CMDDIR/$CLLSRES -g $temp_rg_name -q value=$app_name | $GREP APPLICATIONS`;
        unless ($temp_str) { next; }
        $rg_name = $temp_rg_name;
	# Remove the blank spaces from the beginning and end of resource group name
	$rg_name =~ s/^\s+//;
	$rg_name =~ s/\s+$//;
    }

    # no rg's associated with app name
    unless ($rg_name) {
        system("dspmsg scripts.cat 7829 '$msg_xref{7829}' $CLUMT");
        return 1;
    }

    # set app monitor name (it's the same as the app (server) name)
    $mon_name = $app_name;
        
    # Check whether application has a application monitor defined
    @hacmpmon = `odmget -q value=$app_name HACMPmonitor | grep "monitor ="`;
    #if monitor is defined then get the monitor name
    if ( @hacmpmon ) {
        $value = @hacmpmon[0];
	# Remove the blank spaces from the beginning and end
        $value =~ s/^\s+//;
        $value =~ s/\s+$//;
        @value = split(" ",$value);
        $mon_name = @value[2];
	# Remove double quotes from the monitor name
        $mon_name =~ s/\"//g;
    }

    # load event names hash
    $rc = get_evnames(\%ev_names);
    ($rc == 0) || return $rc;


    #
    # analysis loop
    #

    foreach $lr (@clumt_log) {

        # note if app is part of concurrent rg
        if (scalar(keys %nodes_up) > 1) { $concurr_rg = 1; }

        # get time stamp from log record
        $lr =~ /(\w+)::/;
        $lr_timestamp = $1;

        # end analysis if ending time has been exceeded
        if (($lr_timestamp > $etime) || ($lr_timestamp eq $last_timestamp)) {

            # add final interval to current app state
            $time_interval = $etime - $prev_time;
            $app_times{$curr_app_state} += $time_interval;

            # save largest up/down time intervals
            if (($curr_app_state eq $up) &&
                ($time_interval > $max_times{$up})) {
                $max_times{$up} = $time_interval;
            }
            if (($curr_app_state eq $dn) &&
                ($time_interval > $max_times{$dn})) {
                $max_times{$dn} = $time_interval;
            }

            # add final interval to current monitor state
            $time_interval = $etime - $prev_mon_time;
            $mon_times{$curr_mon_state} += $time_interval;

            last;
        }

        # get state from log record
        $lr =~ /::\s+(\w+)/;
        $lr_app_state = $1;

        # app monitor processing - use separate app mon accumulators
        if (($lr_app_state eq $UMU) || ($lr_app_state eq $UMD) ||
            ($lr_app_state eq $UMS) || ($lr_app_state eq $UMR) ||
            ($lr_app_state eq $UMF)) {
            $lr =~ /::\s+\w+:(\w+):(\w+)/;
            $lr_mon_name = $1;
            $lr_node = $2;

            # skip to next record if monitor name doesn't match
            unless ($lr_mon_name eq $mon_name) { next; }

            # set app mon activity flag
            $mon_active = 1;

            # translate log state to monitor up or down state
            if (($lr_app_state eq $UMS) || ($lr_app_state eq $UMD) ||
                ($lr_app_state eq $UMF)) {
                $new_mon_state = $dn;
            } else {
                $new_mon_state = $up;
            }

            # app monitor failure
            if ($lr_app_state eq $UMF) { $mon_failed = 1; }

            # suspend or resume is manually invoked
            if (($lr_app_state eq $UMS) || ($lr_app_state eq $UMR)) {
                $man_mon_sr = 1;
            }

            # if state hasn't changed, skip to next record
            if ($curr_mon_state eq $new_mon_state) { next; }

            # add time interval if this state change occurred after
            # the begin time
            if ($lr_timestamp > $btime) {
                $time_interval = $lr_timestamp - $prev_mon_time;
                $mon_times{$curr_mon_state} += $time_interval;

                # save current time as new previous monitor time
                $prev_mon_time = $lr_timestamp;
            }

            # update monitor state
            $curr_mon_state = $new_mon_state;

            next;
        }

        # app server processing
        if (($lr_app_state eq $UAS) || ($lr_app_state eq $UAT)) {
            $lr =~ /::\s+\w+:(\w+):(\w+)/;
            $lr_app_name = $1;
            $lr_node = $2;

            # skip to next record if app name doesn't match
            unless ($lr_app_name eq $app_name) { next; }

            # translate log state to app server up or down state
            if ($lr_app_state eq $UAT) {
                delete $nodes_up{$lr_node};
                unless (scalar(keys %nodes_up)) {
                    $new_app_state = $dn;
                }
            } else {
                $nodes_up{$lr_node} = $lr_node;
                $new_app_state = $up;
            }
        }

        # resource group processing
        if (($lr_app_state eq $URN) || ($lr_app_state eq $URF)) {
            $lr =~ /::\s+\w+:(\w+):(\w+)/;
            $lr_rg_name = $1;
            $lr_node = $2;

            # skip to next record if rg name doesn't match
            unless ($lr_rg_name eq $rg_name) { next; }

            # translate log state to app server up or down state
            if ($lr_app_state eq $URF) {
                delete $nodes_up{$lr_node};      
                unless (scalar(keys %nodes_up)) {
                    $new_app_state = $dn;
                }
            } else {
                $nodes_up{$lr_node} = $lr_node;
                $new_app_state = $up;
            }
        }

        # log file last modified processing
        if ($lr_app_state eq $ULM) {
            $lr =~/::\s+\w+:\w+\s+(\w+)\s+(\w+)\s+(\w+):(\w+):(\w+)\s+(\w+):(\w+)/;
#           $lr_date = timelocal($5,$4,$3,$2,$mon_xref{$1},($6 - 1900));
            $lr_node = $7;

            # take node off of up list
            delete $nodes_up{$lr_node};

            unless (scalar(keys %nodes_up)) {

                # flip state to down if no nodes left up
                $new_app_state = $dn;
            }

            # flag if clstr services restart occurred during
            # period of interest
            if (($lr_timestamp > $btime) && ($lr_timestamp < $etime)) {
                $man_clstr_up = 1;
            }
        }

        # node failed processing
        if ($lr_app_state eq $UNF) {
            $lr =~ /::\s+\w+:(\w+)/;
            $lr_node = $1;

            # take node off of up list 
            delete $nodes_up{$lr_node};

            # flip state to down if no nodes left up
            unless (scalar(keys %nodes_up)) {
                $new_app_state = $dn;
            }

            # flag if node failure occurred during period of interest
            if (($lr_timestamp > $btime) && ($lr_timestamp < $etime)) {
                $node_failed = 1;
            }
        }

        # event processing
        if (($lr_app_state eq $UES) || ($lr_app_state eq $UEC)) {
            $lr =~ /::\s+\w+:(\w+)/;
            $event_name = $1;

            # process event data
            # pay attention to these events:
            # - server_restart/server_restart_complete (these occur if appmon
            #   is enabled and the app is terminated manually;
            #   this gives us an estimate of how long the app was down)
            #   treat server_restart as "down" and server_restart_complete
            #   as "up"
            # - others?
            if (($lr_app_state eq $UES) && 
                ($event_name eq $ev_names{$srvr_restrt})) {

                # skip to next record if app server doesn't match
		$lr =~ /::\s+\w+:\w+\s+(\w+)\s+(\w+):/;
                $lr_app_name = $2;
                $lr_node = $1;
                unless ($lr_app_name eq $app_name) { next; }

                # app no longer up on this node
                delete $nodes_up{$lr_node};

                unless (scalar(keys %nodes_up)) {
                    # treat as app down state change
                    $new_app_state = $dn;
                }
            }

            if (($lr_app_state eq $UEC) && 
                ($event_name eq $ev_names{$srvr_restrt_comp})) {

                # skip to next record if app server name doesn't match
		$lr =~ /::\s+\w+:\w+\s+(\w+)\s+(\w+):/;
                $lr_app_name = $2;
                $lr_node = $1;
                unless ($lr_app_name eq $app_name) { next; }

                # app up on this node
                $nodes_up{$lr_node} = $lr_node;

                # treat as app up state change
                $new_app_state = $up;
            }
        }
            
        # if state hasn't changed, go to next record
        if ($new_app_state eq $curr_app_state) { next; }

        # add time interval if this state change occurred after
        # the beginning time
        if ($lr_timestamp > $btime) {
            $time_interval = $lr_timestamp - $prev_time;
            $app_times{$curr_app_state} += $time_interval;

            # save current time as new "previous" time
            $prev_time = $lr_timestamp;

            # save largest up/down time intervals
            if (($curr_app_state eq $up) &&
                ($time_interval > $max_times{$up})) {
                $max_times{$up} = $time_interval;
            }
            if (($curr_app_state eq $dn) &&
                ($time_interval > $max_times{$dn})) {
                $max_times{$dn} = $time_interval;
            }
        }

        # cycle state
        $curr_app_state = $new_app_state;
    }   # end foreach $lr (@clumt_log)


    #
    # calculate results
    #

    # figure percentages
    $pct_uptime = ($app_times{$up} / $tot_time) * 100;
    $pct_dntime = ($app_times{$dn} / $tot_time) * 100;
    $pct_amdtime = ($mon_times{$dn} / $tot_time) * 100;

    # get totals in terms of days, hours, and minutes
    ($tDD, $thh, $tmm, $tss) = time_to_Dhms($etime - $btime);
    ($tuDD, $tuhh, $tumm, $tuss) = time_to_Dhms($app_times{$up});
    ($luDD, $luhh, $lumm, $luss) = time_to_Dhms($max_times{$up});
    ($tdDD, $tdhh, $tdmm, $tdss) = time_to_Dhms($app_times{$dn});
    ($ldDD, $ldhh, $ldmm, $ldss) = time_to_Dhms($max_times{$dn});


    #
    # display results
    #

    # format percentages for display
    @temp_arr = ($pct_uptime, $pct_dntime, $pct_amdtime);
    $arr_sz = @temp_arr;
    $i = 0;
    while ($i < $arr_sz) {
        $temp_arr[$i] = sprintf("%3.2f", $temp_arr[$i]);
        $temp_arr[$i] .= "\\%";
        $i++;
    }
    ($pct_uptime, $pct_dntime, $pct_amdtime) = @temp_arr;

    # format totals for display
    @temp_arr = ($tDD, $tuDD, $luDD, $tdDD, $ldDD);
    $arr_sz = @temp_arr;
    $i = 0;
    while ($i < $arr_sz) {
        $temp_arr[$i] = sprintf("%4d", $temp_arr[$i]);
        $i++;
    }
    ($tDD, $tuDD, $luDD, $tdDD, $ldDD) = @temp_arr;

    @temp_arr = ($thh, $tuhh, $luhh, $tdhh, $ldhh,
                 $tmm, $tumm, $lumm, $tdmm, $ldmm,
                 $tss, $tuss, $luss, $tdss, $ldss);
    $arr_sz = @temp_arr;
    $i = 0;
    while ($i < $arr_sz) {
        $temp_arr[$i] = sprintf("%2d", $temp_arr[$i]);
        $i++;
    }
    ($thh, $tuhh, $luhh, $tdhh, $ldhh,
     $tmm, $tumm, $lumm, $tdmm, $ldmm,
     $tss, $tuss, $luss, $tdss, $ldss) = @temp_arr;

    # load month and day string hashes with translated strings
    $i = 1;
    while ($i < 13) {
        $j = 7852 + $i;
        $k = 7845 + $i;
        $temp_str  = `dspmsg scripts.cat $j '$msg_xref{$j}'`;
        $mon_dsp{$i} = $temp_str;
        if ($i < 8) {
            $temp_str2 = `dspmsg scripts.cat $k '$msg_xref{$k}'`;
            $day_dsp{$i} = $temp_str2;
        }
        $i++;
    }

    # simple print "\n" doesn't work when called by SMIT(!) but this does
    system("dspmsg dummy.cat 1 '$NL'");

    # Analysis begins/Analysis ends
    foreach $time ($btime, $etime) {
        ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($time);
        # format time components for display
        $year += 1900; $year = sprintf("%04d", $year);
        $mon  += 1; $mon = $mon_dsp{$mon};
        $mday = sprintf("%02d", $mday);
        if ($wday == 0) { $wday = 7; }
        $wday = $day_dsp{$wday};
        $hour = sprintf("%02d", $hour);
        $min  = sprintf("%02d", $min);
        $temp_str = $TB . "   " . $wday . ", " . $mday . "-" . $mon . "-";
        $temp_str .= $year . ", " . $hour . ":" . $min; 

#untaint data
        $temp_str = untaint($temp_str);
        $wday = untaint($wday);
        $mday = untaint($mday);
        $mon = untaint($mon);
        $year = untaint($year);
        $hour = untaint($hour);
        $min = untaint($min);
        
        if ($time == $btime) {
            $temp_str = $msg_xref{7830} . $temp_str . $NL;
            system("dspmsg scripts.cat 7830 '$temp_str' $wday $mday $mon $year $hour $min");     
        } else {
            $temp_str = $msg_xref{7831} . $TB . $temp_str . $NL;
            system("dspmsg scripts.cat 7831 '$temp_str' $wday $mday $mon $year $hour $min");
        }
    }

    # name of app analyzed
    $temp_str = $msg_xref{7955} . $TB . "   " . $app_name . $NL;
    system("dspmsg scripts.cat 7955 '$temp_str' $app_name");

    # skip line before time statistics
    system("dspmsg dummy.cat 1 '$NL'");

    # Total time
    $temp_str = $msg_xref{7832} . $TB2 . $tDD . $DAYS . $thh . $HOURS .
                $tmm . $MINS . $tss . $SECS . $NL;

#untaint data
    $temp_str = untaint($temp_str);
    $tDD = untaint($tDD);
    $thh = untaint($thh);
    $tmm = untaint($tmm);
    $tss = untaint($tss);
    
    system("dspmsg scripts.cat 7832 '$temp_str' $tDD $thh $tmm $tss");

    system("dspmsg dummy.cat 1 '$NL'");

    # Uptime
    $temp_str = $msg_xref{7833} . $NL;
    system("dspmsg scripts.cat 7833 '$temp_str'");

    # Amount
    $temp_str = $TB . $msg_xref{7835} . $TB2 . $tuDD . $DAYS . $tuhh .
                $HOURS . $tumm . $MINS . $tuss . $SECS . $NL;

#untaint data
    $temp_str = untaint($temp_str);
    $tuDD = untaint($tuDD);
    $tuhh = untaint($tuhh);
    $tumm = untaint($tumm);
    $tuss = untaint($tuss);   

    system("dspmsg scripts.cat 7835 '$temp_str' $tuDD $tuhh $tumm $tuss");

    # Percentage
    $temp_str = $TB . $msg_xref{7836} . $TB . "   " .$pct_uptime . $NL;
    
    $temp_str = untaint($temp_str);
    $pct_uptime = untaint($pct_uptime);

    system("dspmsg scripts.cat 7836 '$temp_str' $pct_uptime");

    # Longest period
    $temp_str = $TB . $msg_xref{7837} . $TB . $luDD . $DAYS . $luhh . $HOURS .
               $lumm . $MINS . $luss . $SECS . $NL;
    
#untaint data
    $temp_str = untaint($temp_str);
    $luDD = untaint($luDD);
    $luhh = untaint($luhh);
    $lumm = untaint($lumm);
    $luss = untaint($luss);

    system("dspmsg scripts.cat 7837 '$temp_str' $luDD $luhh $lumm $luss");

    system("dspmsg dummy.cat 1 '$NL'");

    # Downtime
    $temp_str = $msg_xref{7834} . $NL;
    system("dspmsg scripts.cat 7834 '$temp_str'");

    # Amount
    $temp_str = $TB . $msg_xref{7835} . $TB2 . $tdDD . $DAYS . $tdhh .
                $HOURS . $tdmm . $MINS . $tdss . $SECS . $NL;
    
#untaint data
    $temp_str = untaint($temp_str);
    $tdDD = untaint($tdDD);
    $tdhh = untaint($tdhh);
    $tdmm = untaint($tdmm);
    $tdss = untaint($tdss);

    system("dspmsg scripts.cat 7835 '$temp_str' $tdDD $tdhh $tdmm $tdss");

    # Percentage
    $temp_str = $TB . $msg_xref{7836} . $TB . "   " . $pct_dntime . $NL;
 
    $temp_str = untaint($temp_str);
    $pct_dntime = untaint($pct_dntime);
   
    system("dspmsg scripts.cat 7836 '$temp_str' $pct_dntime");

    # Longest period
    $temp_str = $TB . $msg_xref{7837} . $TB . $ldDD . $DAYS . $ldhh . $HOURS .
               $ldmm . $MINS . $ldss . $SECS . $NL;
    
#untaint data
    $temp_str = untaint($temp_str);
    $ldDD = untaint($ldDD);
    $ldhh = untaint($ldhh);
    $ldmm = untaint($ldmm);
    $ldss = untaint($ldss);
    
    system("dspmsg scripts.cat 7837 '$temp_str' $ldDD $ldhh $ldmm $ldss");

    system("dspmsg dummy.cat 1 '$NL'");

    # done with analysis output, now display info messages

    # log ended before ending time
    if ($lr_timestamp < $etime) {
        system("dspmsg scripts.cat 7838 '$msg_xref{7838}'");
        system("dspmsg dummy.cat 1 '$NL'");
    }

    # app part of concurrent rg
    if ($concurr_rg) {
        system("dspmsg scripts.cat 7956 '$msg_xref{7956}'");
        system("dspmsg dummy.cat 1 '$NL'");
    }

    # application monitor messages
    unless ($mon_active) {

        # monitoring inactive
        system("dspmsg scripts.cat 7839 '$msg_xref{7839}'");
        system("dspmsg dummy.cat 1 '$NL'");

    } elsif ($mon_times{$dn} > 0) {

        # monitoring suspended
        $temp_str = $msg_xref{7840};
        $temp_str =~ s/XXXX/$pct_amdtime/;

#untainting data
	$temp_str = untaint($temp_str);
	$pct_amdtime = untaint($pct_amdtime);
 
        system("dspmsg scripts.cat 7840 '$temp_str' $pct_amdtime");
        system("dspmsg dummy.cat 1 '$NL'");

    }

    # app monitor was suspended/resumed
    if ($man_mon_sr) {
        system("dspmsg scripts.cat 7953 '$msg_xref{7953}'");
        system("dspmsg dummy.cat 1 '$NL'");
    }

    # app monitor failed
    if ($mon_failed) {
        system("dspmsg scripts.cat 7954 '$msg_xref{7954}'");
        system("dspmsg dummy.cat 1 '$NL'");
    }

    # cluster services restarted manually during period of interest
    if ($man_clstr_up) {
        system("dspmsg scripts.cat 7844 '$msg_xref{7844}'");
        system("dspmsg dummy.cat 1 '$NL'");
    }

    # hard node failure during period of interest
    if ($node_failed) {
        system("dspmsg scripts.cat 7845 '$msg_xref{7845}'");
        system("dspmsg dummy.cat 1 '$NL'");
    }

    return 0;
}


#--------------------------------------------------------------------#
# check_time_oor                                                     #
#                                                                    #
# Description:                                                       #
#                                                                    #
#      This routine checks the time date entered on the              #
#      command line to insure that none of the fields in the         #
#      time data are outside of acceptable ranges:                   #
#              YYYY    the 4-digit year (0000..9999)                 #
#              MM      the 2-digit month number (01...12)            #
#              DD      the 2-digit day of the month (01...31)        #
#              hh      the 2-digit hour of the day (00...23)         #
#              mm      the 2-digit minute within the hour (00...59)  #
#              ss      the 2-digit second within the minute (00...59)#
#                                                                    #
#      If any of the fields are out-of-range, this routine will      #
#      issue an error message and return an error to the caller.     #
#                                                                    #
# Inputs:                                                            #
#      $r_time     reference to array of time fields                 #
#                                                                    #
# Return values:                                                     #
#      0     success.                                                #
#     -1     error.                                                  #
#                                                                    #
# Outputs:                                                           #
#      Error messages when out-of-range condition is detected.       #
#                                                                    #
#--------------------------------------------------------------------#
sub check_time_oor
{
    # get input and extract time fields
    my $r_time = shift;
    my $year   = ${$r_time}[0];
    my $month  = ${$r_time}[1];
    my $day    = ${$r_time}[2];
    my $hour   = ${$r_time}[3];
    my $minute = ${$r_time}[4];
    my $second = ${$r_time}[5];
    my $tag    = ${$r_time}[6];

    # ranges
    my $zero    = 0;         # min for hour, minute, second
    my $one     = 1;         # min for month, day
    my $yr_min  = 1970;      # min for year
    my $yr_max  = 2037;      # max for year
    my $mon_max = 12;        # max for month
    my $day_max = 31;        # max for day
    my $hr_max  = 23;        # max for hour
    my $m_s_max = 59;        # max for minute, second

    # nothing fancy - check each field against acceptable range
    #               - issue message and return if out-of-range
    # use $tag to issue the proper msg for begin or end time field errors
    unless (($year >= $yr_min) && ($year <= $yr_max)) {
        if ($tag eq $beg_tag) {
            system("dspmsg scripts.cat 7816 '$msg_xref{7816}' $CLUMT");
        } else {
            system("dspmsg scripts.cat 7822 '$msg_xref{7822}' $CLUMT");
        }
        return -1;
    }
    unless (($month >= $one) && ($month <= $mon_max)) {
        if ($tag eq $beg_tag) {
            system("dspmsg scripts.cat 7817 '$msg_xref{7817}' $CLUMT");
        } else {
            system("dspmsg scripts.cat 7823 '$msg_xref{7823}' $CLUMT");
        }
        return -1;
    }
    unless (($day >= $one) && ($day <= $day_max)) {
        if ($tag eq $beg_tag) {
            system("dspmsg scripts.cat 7818 '$msg_xref{7818}' $CLUMT");
        } else {
            system("dspmsg scripts.cat 7824 '$msg_xref{7824}' $CLUMT");
        }
        return -1;
    }
    unless (($hour >= $zero) && ($hour <= $hr_max)) {
        if ($tag eq $beg_tag) {
            system("dspmsg scripts.cat 7819 '$msg_xref{7819}' $CLUMT");
        } else {
            system("dspmsg scripts.cat 7825 '$msg_xref{7825}' $CLUMT");
        }
        return -1;
    }
    unless (($minute >= $zero) && ($minute <= $m_s_max)) {
        if ($tag eq $beg_tag) {
            system("dspmsg scripts.cat 7820 '$msg_xref{7820}' $CLUMT");
        } else {
            system("dspmsg scripts.cat 7826 '$msg_xref{7826}' $CLUMT");
        }
        return -1;
    }
    unless (($second >= $zero) && ($second <= $m_s_max)) {
        if ($tag eq $beg_tag) {
            system("dspmsg scripts.cat 7821 '$msg_xref{7821}' $CLUMT");
        } else {
            system("dspmsg scripts.cat 7827 '$msg_xref{7827}' $CLUMT");
        }
        return -1;
    }

    # if we get here, everthing's hunky-dory
    return 0;
}


#--------------------------------------------------------------------#
# logsort                                                            #
#                                                                    #
# Description:                                                       #
#                                                                    #
#      Called by Perl sort routine to sort clavan log records.       #
#      Sorts by numeric timestamp.                                   #
#                                                                    #
# Inputs:                                                            #
#                                                                    #
#      None passed in explicitly.  $a and $b are defined implicitly  #
#      by sort as the two entities to be compared.                   #
#      Expected format:                                              #
#         nnnnnnnnn::mnemonic:[data]:[data]: variable text           #
#      Where:                                                        #
#         nnnnnnnnn is the numeric timestamp                         #
#         mnemonic indicates the type of log entry                   #
#         [data] is optional data specific to the type of log entry  #
#                                                                    #
# Outputs:                                                           #
#                                                                    #
#      None returned explicitly.  sort routine will receive back:    #
#         0 if $a equals $b                                          #
#        -1 if $a less than $b                                       #
#         1 if $a greater than $b                                    #
#                                                                    #
#--------------------------------------------------------------------#
sub logsort
{
    # elt 0 is numeric timestamp, elt 1 is mnemonic and text data
    my @a_arg = split(/::/, $a);
    my @b_arg = split(/::/, $b);

    # sort by timestamp
    $a_arg[0] <=> $b_arg[0];
}


#--------------------------------------------------------------------#
# get_evstrings                                                      #
#                                                                    #
# Description:                                                       #
#                                                                    #
#     The practice of odmgetting the HACMPrules data directly is     #
#     necessary here so that we can associate the "raw" event names  #
#     with the event names as stored in the recovery program files.  #
#     The cllsevpgr utility only lists the rp event names.           #
#                                                                    #
#     Obtains event names from HACMP.                                #
#     - calls odmget on HACMPrules class to get path of each         #
#       event's recovery file                                        #
#     - reads each event file to get the event names                 #
#     - saves event names in hash via reference passed in as input   #
#                                                                    #
# Inputs:                                                            #
#     r_evnames       Reference to hash to be used to store          #
#                     the event names                                #
#                                                                    #
# Outputs:                                                           #
#     0 if successful                                                #
#    -1 if failure                                                   #
#                                                                    #
#--------------------------------------------------------------------#
sub get_evnames
{
    my $r_evnames = shift;    # ref to hash which will store event names
    my @temp_arr = ();        # work array
    my @temp_ev_arr = ();     # work array - event name strings
    my $temp_elt = "";        # work array element
    my $pos = -1;             # position indicator
    my $raw_name = "";        # current "raw" event name
    my $raw_name_alt = "";    # alternate "raw" event name
    my $rp_path = "";         # recovery program file path
    my $i = 0;                # counter
    my $j = 0;                # counter
    my $k = 0;                # counter
    my $rc = 0;


    #
    # get event info from HACMPrules class
    #

    @temp_arr = `odmget HACMPrules`;
    $rc = $?;

    # problem getting event info
    unless((scalar(@temp_arr)) && ($rc == 0)) {
        system("dspmsg scripts.cat 7841 '$msg_xref{7841}' $CLUMT");
        return -1;
    }


    #
    # process output and load event names hash
    #

    while ($i < $#temp_arr) {

        # pick out name field
        $pos = index($temp_arr[$i], "name");

        # only do further processing if we are currently at the name field
        if ($pos > -1) {

            # save raw event name
            $pos = index($temp_arr[$i], "\"");
            $raw_name = substr($temp_arr[$i], $pos);
            chop($raw_name);
            $raw_name =~ s/\"//g;

            # get recovery program file path
            $j = $i + 2;
            $pos = index($temp_arr[$j], "\"");
            $rp_path = substr($temp_arr[$j], $pos);
            chop($rp_path);
            $rp_path =~ s/\"//g;

            # read recovery program file and get desired event names
            unless (open RPFILE, $rp_path) {
                system("dspmsg scripts.cat 7842 '$msg_xref{7842}' $CLUMT");
                return -1;
            }
            $k = 0;
            while(<RPFILE>) {
                # ignore comments, "barrier" lines, lines with path info
                if (/^#/) { next; }
                if (/^barrier/) { next; }
                if (/\//) { next; }

                # save unique event names - discard duplicates
                /^\w+\s+"(\w+)"/; 
                if ($k == 0) {
                    $temp_ev_arr[$k] = $1;
                    $k++;
                } else {
                    unless ($temp_ev_arr[$k-1] eq $1) {
                        $temp_ev_arr[$k] = $1;
                        $k++;
                    }
                }
            }
            close(RPFILE);

            # save unique event names in hash

            # join_standby and fail_standby have single event names
            if (($raw_name eq $jn_stdby) || ($raw_name eq $fl_stdby)) {
                $r_evnames->{$raw_name} = $temp_ev_arr[0];
            }

            # reconfig_resource has three event names
            # (release, acquire, complete)
            elsif ($raw_name eq $recfg_rsrc_rlse) {
                $r_evnames->{$raw_name} = $temp_ev_arr[0];
                $raw_name_alt = $raw_name . "_A";
                $r_evnames->{$raw_name_alt} = $temp_ev_arr[1];
                $raw_name_alt = $raw_name . "_C";
                $r_evnames->{$raw_name_alt} = $temp_ev_arr[2];
            }

            # rg_move has three event names
            # (release, acquire, complete)
            elsif ($raw_name eq $rg_move) {
                $r_evnames->{$raw_name} = $temp_ev_arr[0];
                $raw_name_alt = $raw_name . "_A";
                $r_evnames->{$raw_name_alt} = $temp_ev_arr[1];
                $raw_name_alt = $raw_name . "_C";
                $r_evnames->{$raw_name_alt} = $temp_ev_arr[2];
            }

            # all others have two names:  event, event_complete
            else {
                $raw_name_alt = $raw_name . "_C";
                $r_evnames->{$raw_name} = $temp_ev_arr[0];
                $r_evnames->{$raw_name_alt} = $temp_ev_arr[1];
            }
        }   # end if ($pos > -1)
        $i++;
    }   # end while ($i < $#temp_arr)


    #
    # if any of the hash values are empty, could indicate:
    #   - rp files corrupted
    #   - list of raw event names from HACMPrules doesn't match
    #     what this script expects
    #

    # get hash values
    @temp_arr = ();
    @temp_arr = @$r_evnames{$node_up,
                            $node_up_comp,
                            $node_dn,
                            $node_dn_comp,
                            $net_up,
                            $net_up_comp,
                            $net_down,
                            $net_down_comp,
                            $swap_adptr,
                            $swap_adptr_comp,
                            $jn_stdby,
                            $fl_stdby,
                            $migrate,
                            $migrate_comp,
                            $recfg_rsrc_rlse,
                            $recfg_rsrc_acq,
                            $recfg_rsrc_comp,
                            $srvr_restrt,
                            $srvr_restrt_comp,
                            $srvr_dn,
                            $srvr_dn_comp,
                            $rg_move,
                            $rg_move_acq,
                            $rg_move_comp,
                            $site_up,
                            $site_up_comp,
                            $site_dn,
                            $site_dn_comp};

    # check hash values
    foreach $temp_elt (@temp_arr) {
        if ($temp_elt eq "") {
            system("dspmsg scripts.cat 7843 '$msg_xref{7843}' $CLUMT");
            return -1;
        }
    }
    return 0;
}


#--------------------------------------------------------------------#
# time_to_Dhms                                                       #
#                                                                    #
# Description:                                                       #
#                                                                    #
#      This routine calculates the number of days, hours, minutes    #
#      and seconds from a time value expressed in seconds since      #
#      the epoch.                                                    #
#                                                                    #
# Inputs:                                                            #
#                                                                    #
#      $time_interval        Time interval in seconds                #
#                                                                    #
# Return values:                                                     #
#                                                                    #
#      None.                                                         #
#                                                                    #
# Outputs:                                                           #
#                                                                    #
#      $days                 Number of days in interval              #
#      $hours                Number of hours in interval             #
#      $mins                 Number of minutes in interval           #
#      $secs                 Number of seconds in interval           #
#                                                                    #
#--------------------------------------------------------------------#
sub time_to_Dhms
{
    my $time_interval = shift;
    my $days = 0;
    my $hours = 0;
    my $mins = 0;
    my $secs = 0;

    $secs = $time_interval % 60;
    $time_interval = ($time_interval - $secs) / 60;
    $mins = $time_interval % 60;
    $time_interval = ($time_interval - $mins) / 60;
    $hours = $time_interval % 24;
    $days = ($time_interval - $hours) / 24;
    
    return($days, $hours, $mins, $secs);
}


#--------------------------------------------------------------------#
# usage                                                              #
#                                                                    #
# Description:                                                       #
#                                                                    #
#      This routine displays on STDOUT                               #
#      the correct usage for this program.                           #
#                                                                    #
# Inputs:                                                            #
#                                                                    #
#      None.                                                         #
#                                                                    #
# Return values:                                                     #
#                                                                    #
#      0                                                             #
#                                                                    #
# Outputs:                                                           #
#                                                                    #
#      Usage message.                                                #
#                                                                    #
#--------------------------------------------------------------------#
sub usage
{
    system("dspmsg scripts.cat 7800 '$msg_xref{7800}'");
    return 0;
}
