#!/usr/local/bin/perl
#
# $Header: emdb/sysman/admin/scripts/has/has_metrics.pl /st_emdbsa_11.2/22 2011/03/04 23:04:35 ajdsouza Exp $
#
# has_metrics.pl
# 
# Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved. 
#
#    NAME
#      has_metrics.pl - <one-line expansion of the name>
#
#    DESCRIPTION
#      this is the script of choice for executing any result set to be passed
#      to the agent.
#      pass a tag , metric_name as argv0 to the script and provide a function name
#      for that tag in the file
#      this will cause the function to be called thru has_main which will take care 
#      of handlign errors, warnings etc.
#       if you do not do that, then you are on your own for error handling
#
#      This script instruments the metrics to get server, server group, resource,
#       resource_type , resource group , resource status for each instance
#       from crs
# 
#     check has_test for testing this script and for all the interfaces available
#
#    NOTES
#      How to test this script - Unit testing
#      1. Use the unit test feature in has::Common.pm to unit test the file
#      2. set the ENV variable HAS_TEST_MODE=capture and execute the script to capture
#       input/output from the script
#      3. Input is normally captured to /tmp/has.out
#      4. you can set ENV variable HAS_TEST_MODE_DESC to a suitable description as
#       HAS_TEST_MODE_DESC="this is my test etc.." to document the
#       test
#      5. the output should be directed to a output gold file from stdout
#      6. You can rename these files to identifiable names
#      7. Now to use this file for regression you need to set the ENV variable HAS_TEST_MODE=Regression
#      8. By default now the file input from file /tmp/has.out is used as OS command input by the script
#      9. If you want the script to use a different file name for input set ENV variable
#          HAS_TEST_FILE_PATH  to the full path of the file to be used as input
#      10.  Execute the script and capture the stdout to a log file.
#      11. Now diff between the original gold file and the log file, there should be no diff
#
#       Actual commands to capture and test
#       -----------------------------------
#       setenv HAS_TEST_MODE capture
#       setenv HAS_TEST_MODE_DESC "Test for 11gTB cluster from HAS View"
#       setenv PERL5LIB ${PERL5LIB}:${ORACLE_HOME}/perl/lib/5.8.3:${ORACLE_HOME}/perl/lib/site_perl/5.8.3:${ADE_VIEW_ROOT}/emagent/sysman/admin/scripts:${ADE_VIEW_ROOT}/emdb/sysman/admin/scripts;
#
#       perl has_metrics.pl > /tmp/hasgold.out
#ls -l /tmp/has*.out
#-rw-r--r--  1 ajdsouza g646 1307330 Oct 18 19:41 /tmp/has.out
#-rw-r--r--  1 ajdsouza g646  203432 Oct 18 19:41 /tmp/hasgold.out
#
#       setenv HAS_TEST_MODE regression
#       setenv HAS_TEST_FILE_PATH /tmp/has.out
#       perl has_metrics.pl > /tmp/hasgold2.out
#ls -l /tmp/has*.out
#-rw-r--r--  1 ajdsouza g646 1307330 Oct 18 19:41 /tmp/has.out
#-rw-r--r--  1 ajdsouza g646  203432 Oct 18 19:41 /tmp/hasgold.out
#-rw-r--r--  1 ajdsouza g646  203432 Oct 18 19:41 /tmp/hasgold2.out
#
#       diff /tmp/hasgold.out /tmp/hasgold2.out
#
# For testing in development
# --------------------------
#  setenv HAS_TEST_MODE development
#
# Making changes for regressions
# -------------------------------
#   this will cause the has_test<xxxx> functions to be be executed.
#
#  if you are adding a new metric, its a good idea to add it to has_test_metrics function
#
#  if you are adding a new sub to has::Common , its a good idea to add it to has_test_library_calls function
#  
#  makeing any change to has_test_library_calls or has_test_metrics will affect regression tests
#   you will need to capture new dat and gold regression files for tests in 
#    tvmd/tvmdh/tvmdhr/tvmdhrc/tvmdhrcsh
#
#
# NOTE : IMPORTANT
# ----------------
#    use has_init() before any function from the has::Common library 
#    use has_cleanup() after using any function from the has::Common::library
#      they will ensure that no errors and warning are written to STDERR
#      all errors , warning and debug informated is logged to trace and also saved in a stack
#      to be printed at the end of execution
#    use has_printerrors to print the warnings and errors in em_result, em_warning format
#
#
#
#    MODIFIED   (MM/DD/YY)
#    ajdsouza    03/04/11 - bug fix 10141620
#    ajdsouza    07/15/10 - Bug# 9887750 
#    ajdsouza    10/15/09 - nodeName dynamic prop support bug#8897428
#    ajdsouza    10/12/09 - invoke scanapi with dynamic flag
#    ajdsouza    10/12/09 - use has_metrics.pl to get nodename dynamic prop
#    ajdsouza    09/08/09 - set ORA_CRS_HOME if EM_CRS_TEST_CRS_HOME is set
#    ajdsouza    08/25/09 - added EM_CRS_TEST_CRS_HOME env
#    kramarat    02/17/09 - Siha listener chagnes
#    ajdsouza    08/16/08 - support server_pools, support resource_instance metric
#    rsamaved    08/16/08 - 
#    ajdsouza    04/18/08 - added metric resource_instance_alert_metric
#    kramarat    03/11/08 - use last_server
#    ajdsouza    05/21/07 - mapped to cmdb classes
#    ajdsouza    11/09/06 - Creation
#


use strict;
use warnings;

use File::Spec::Functions;
use File::Basename;
use File::Path;
use Data::Dumper;
use locale;
use Digest::MD5 qw( md5_hex );
use POSIX;
use has::Common;


$Data::Dumper::Indent = 2;
$Data::Dumper::Deepcopy = 1;
$Data::Dumper::Purity = 1;
$Data::Dumper::Sortkeys = 1;

my $oldOH;
my $DEV_DEBUG;
$DEV_DEBUG=1 if $ENV{HAS_RUN_SCRIPT_DEBUG};

BEGIN
{

 # if SRVM_TRACE is set to TRUE them do not disable SRVM_TRACE for cluvfy
 if ( exists $ENV{SRVM_TRACE} and $ENV{SRVM_TRACE} and $ENV{SRVM_TRACE} =~ /TRUE/i )
 {
   $has::Commmon::has_metric_config{ENV_VARIABLE_SRVM_TRACE}='TRUE';
   # added the extra line to get around the perl error
   #  used only once: possible typo
   $has::Commmon::has_metric_config{ENV_VARIABLE_SRVM_TRACE}='TRUE';
 }

 # trace is disabled as srvctl writes trace to stdout, enabled slectively for cluvfy
 delete $ENV{SRVM_TRACE} if $ENV{SRVM_TRACE};

 # save ORACLE_HOME and restore it back in END
 $oldOH = $ENV{ORACLE_HOME} if $ENV{ORACLE_HOME};

 # temporarly setting environment only in dev view env
 # this code is not avtive in production
 # !! DO NOT CHANGE THE CLUSTER NAME HERE WHEN YOU
 # CHECK IN AS THIS WILL BREAK REGRESSION
 #
 # IF YOU ARE CREATING A HAS VIEW CREATE IT 
 # WITH VIEW NAME <sa view_name>t
 # e.g. if sa view is sa3114 the has view should be sa3114t
 # the clustername is the has view should be newdb_cluster
 if ( $ENV{ADE_VIEW_ROOT} and not $ENV{HAS_USE_SHIPHOME} )
 {

  if ( $ENV{EM_CRS_TEST_CRS_HOME}  )
  {
    $ENV{ORA_CRS_HOME} =  $ENV{EM_CRS_TEST_CRS_HOME};
  }
  else
  {
   my $advrt;
   $advrt =  $ENV{ADE_VIEW_ROOT};
   $advrt  =~ s/_ag$//;
   $advrt  = $advrt."t";
   $ENV{ORA_CRS_HOME}="$advrt/oracle";
  }

  if ( $ENV{EM_CRS_TEST_CLUSTER_NAME}  )
  {
    $ENV{CSS_CLUSTERNAME}=$ENV{EM_CRS_TEST_CLUSTER_NAME};
  }
  else
  {
    $ENV{CSS_CLUSTERNAME}='newdb_cluster';
  }

  $ENV{CRS_HOME}="$ENV{ORA_CRS_HOME}";
  $ENV{CV_HOME}="$ENV{CRS_HOME}";
  $ENV{OCR_ROOT}="$ENV{CRS_HOME}/has_work/data.ocr";
  $ENV{OCR_LOC}="$ENV{CRS_HOME}/has_work/ocr.loc";
  $ENV{OLR_LOC}="$ENV{CRS_HOME}/has_work/olr.loc";
  $ENV{OCR_DEVELOPER_ENV}='TRUE';
  $ENV{HAS_DEVELOPMENT_ENVIRONMENT}='TRUE';
  $ENV{CV_JDKHOME}="$ENV{CRS_HOME}/jdk15";
  $ENV{ORA_ENVIRON_OPTS}='true';

  my $libs = "$ENV{CRS_HOME}/lib:$ENV{CRS_HOME}/has/lib:$ENV{CRS_HOME}/opsm/lib";

  $ENV{LD_LIBRARY_PATH}="$libs:$ENV{LD_LIBRARY_PATH}" if $ENV{LD_LIBRARY_PATH};
  $ENV{LD_LIBRARY_PATH}="$libs" unless $ENV{LD_LIBRARY_PATH};

  $ENV{PATH}="$ENV{CRS_HOME}/bin:$ENV{CRS_HOME}/has/bin:$ENV{PATH}" if $ENV{PATH};
  $ENV{PATH}="$ENV{CRS_HOME}/bin:$ENV{CRS_HOME}/has/bin" unless $ENV{PATH};

 }

 # enable autoflush as POSIX:_exit does not flush before exit
 $|=1;

}


#------------------------------------------------------------------------------
#subs declared
#------------------------------------------------------------------------------

sub has_init();
sub has_exitfail();
sub has_cleanup();

sub has_rd_cache($);
sub has_rm_cache($);
sub has_dump_to_cache_file($$;);
sub has_entities($;@);
sub has_cust_attribs($$$);
sub has_servergroup_members(\%\%$);
sub has_servergroup_resource($$$);
sub has_instances(\%\%$);
sub has_crs_nodes($;$);
sub has_instrument_metrics($;@);
sub has_crs_metric($;$);
sub has_scan_metric($;$);
sub has_cluster_health_check_metric($;$);
sub has_asm_cmdb_metric($$$$;$$);
sub has_votingdisk_metric($;$);
sub has_vip_metric($;$);
sub has_ocr_metric($;$);
sub has_eons_metric($;$);
sub has_dynamic_property($$;$);
sub hasGetDBHasManaged($;$);
sub hasGetLsnrHasManaged($;$$);
sub hasGetHasStatusMetric(;$$);
sub hasGetCRSStatusMetric(;$$$$);
sub hasGetLocalCRSStatusMetric(;$$$);
sub has_localnodeinfo_metric($;$);
sub has_get_resource_name_metric($$$;$$);
sub has_build_metric_from_env($;$);
sub has_event_resource_instance_metric($;@);
sub has_resource_instance_status($$$);

#----------------------------------------------------------------------------
# package variable to hold global values 
#---------------------------------------------------------------------------

$has::Common::has_metric_config{crs_event}{multi_function}=
[ qw ( gpnp_alert_metric gpnp_server_alert_metric ) ];

$has::Common::has_metric_config{function}{resources}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{server_groups}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{servers}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{nodes}=\&has_crs_nodes;
$has::Common::has_metric_config{function}{resource_types}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{server_group_members}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{server_group_resource}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{resource_attributes}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{resource_type_attributes}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{resource_instances}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{resource_instance_attributes}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{resource_instance_alert_metric}=\&has_event_resource_instance_metric;
$has::Common::has_metric_config{function}{resource_instance_status_metric}=\&has_event_resource_instance_metric;
$has::Common::has_metric_config{function}{gpnp_alert_metric}=\&has_event_resource_instance_metric;
$has::Common::has_metric_config{function}{css_nodes_metric}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{crs_servers}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{cluster_health_check}=\&has_cluster_health_check_metric;
$has::Common::has_metric_config{function}{gpnp_server_alert_metric}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{vip_relocation_metric}=\&has_instrument_metrics;

$has::Common::has_metric_config{function}{resource_type_dependencies}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{resource_dependencies}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{resource_instance_dependencies}=\&has_instrument_metrics;
$has::Common::has_metric_config{function}{crs_metric}=\&has_crs_metric;
$has::Common::has_metric_config{function}{css_metric}=\&has_crs_metric;
$has::Common::has_metric_config{function}{scan_metric}=\&has_scan_metric;
$has::Common::has_metric_config{function}{scan_vip_metric}=\&has_scan_metric;
$has::Common::has_metric_config{function}{asm_metric}=\&has_asm_cmdb_metric;
$has::Common::has_metric_config{function}{asm_instance_metric}=\&has_asm_cmdb_metric;
$has::Common::has_metric_config{function}{asm_diskgroup_metric}=\&has_asm_cmdb_metric;
$has::Common::has_metric_config{function}{asm_disk_metric}=\&has_asm_cmdb_metric;
$has::Common::has_metric_config{function}{asm_diskgroupused_metric}=\&has_asm_cmdb_metric;
$has::Common::has_metric_config{function}{rac_metric}=\&has_rac_cmdb_metric;
$has::Common::has_metric_config{function}{database_metric}=\&has_rac_cmdb_metric;
$has::Common::has_metric_config{function}{votingdisk_metric}=\&has_votingdisk_metric;
$has::Common::has_metric_config{function}{ocr_metric}=\&has_ocr_metric;
$has::Common::has_metric_config{function}{vip_metric}=\&has_vip_metric;
$has::Common::has_metric_config{function}{eons_metric}=\&has_eons_metric;
$has::Common::has_metric_config{function}{dynamic_property}=\&has_dynamic_property;
$has::Common::has_metric_config{function}{property}=\&has_dynamic_property;
$has::Common::has_metric_config{function}{db_has_managed}=\&hasGetDBHasManaged;
$has::Common::has_metric_config{function}{lsnr_has_managed}=\&hasGetLsnrHasManaged;
$has::Common::has_metric_config{function}{has_status}=\&hasGetHasStatusMetric;
$has::Common::has_metric_config{function}{crs_status}=\&hasGetLocalCRSStatusMetric;
$has::Common::has_metric_config{function}{local_crs_status}=\&hasGetLocalCRSStatusMetric;
$has::Common::has_metric_config{function}{nodeinfo_metric}=\&has_localnodeinfo_metric;
$has::Common::has_metric_config{function}{resource_name}=\&has_get_resource_name_metric;
$has::Common::has_metric_config{function}{listener_resource_name}=\&has_get_resource_name_metric;
$has::Common::has_metric_config{function}{listener_cmdb_metric}=\&has_build_metric_from_env;
$has::Common::has_metric_config{function}{scan_listener_cmdb_metric}=\&has_build_metric_from_env;

# metrics cached during instrumentation of this metric
$has::Common::has_metric_config{dependent_metric_functions}{resources} =
 { resource_attributes => \&has_cust_attribs,
   resource_dependencies => \&has::Common::has_dependencies,
   server_group_resource => \&has_servergroup_resource };
$has::Common::has_metric_config{dependent_metric_functions}{server_groups} =
 { server_group_members => \&has_servergroup_members };
$has::Common::has_metric_config{dependent_metric_functions}{resource_types} =
 { resource_type_attributes => \&has_cust_attribs,
   resource_type_dependencies => \&has::Common::has_dependencies };
$has::Common::has_metric_config{dependent_metric_functions}{resource_instances} =
 { resource_instance_attributes => \&has_cust_attribs,
   resource_instance_dependencies => \&has::Common::has_dependencies };
$has::Common::has_metric_config{dependent_metric_functions}{resource_instance_alert_metric} =
 { resource_instance_status_metric => \&has_resource_instance_status };
$has::Common::has_metric_config{dependent_metric_functions}{css_nodes_metric} =
 { crs_servers => \&has_resource_crs_servers ,
   gpnp_server_alert_metric => \&has_resource_crs_servers };
$has::Common::has_metric_config{dependent_metric_functions}{gpnp_alert_metric} =
 { gpnp_server_pool => \&has_gpnp_server_pool };


#------------------------------------------------------------------------------
# FUNCTION :    has_exitfail
#
# DESC
# clean up, print errors before failure exit
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
#holds the output format information
my $has_output_format;

sub has_exitfail()
{
  # log the message to the log file
  EMD_PERL_ERROR("HAS:Failed execution in fnexitfail");

  has::Common::has_handle_error('ERROR:Failed Execution');

  has::Common::has_printerrors('exit_fail');

  has_cleanup();

#graceful exit with exit status success
POSIX:_exit(1);

}

#------------------------------------------------------------------------------
# FUNCTION :    has_init
#
# DESC
# initialize 
# Perform the initialization steps
# Prepare the directory for logging
#
# ARGUMENTS
#
#-----------------------------------------------------------------------------
sub has_init()
{

 #install signal handlers for warn and die
 $SIG{'__DIE__'} = sub {  has::Common::has_handle_error( @_ ); 
                         has_exitfail() };
 $SIG{'__WARN__'} = sub { has::Common::has_handle_error( @_)};

 # initialize the library
 has::Common::hasInit();

 # Save the STDERR before redirecting it to logfile
 open(OLDERR,">&STDERR");

 # If there is an error opening a log file, redirect stderr to null
 my $devnull = File::Spec->devnull();
 open(STDERR,">$devnull");

 # read hostname and the ENV variables for target context 
 $has::Common::has_em_targettype= $ENV{EM_TARGET_TYPE} if $ENV{EM_TARGET_TYPE};
 $has::Common::has_em_targetname= $ENV{EM_TARGET_NAME} if $ENV{EM_TARGET_NAME};
 $has::Common::has_em_targetguid= $ENV{EM_TARGET_GUID} if $ENV{EM_TARGET_GUID}; 

 return 1;
}

#------------------------------------------------------------------------------
# FUNCTION :    has_cleanup
#
# DESC
# clean up, print results,print errors , successful exit
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
sub has_cleanup()
{

  # Restore STDERR
  close(STDERR);

  # Restore back the stderr fd
  open(STDERR,">&OLDERR");

  close(OLDERR);

 # cleanup the library
 has::Common::hasCleanup();

  return 1;

}

# name : has_rm_cache
# desc : clean up cached files generated by the run of this metric
#
# arg  :
#  metric_name
#
# return:
#
sub has_rm_cache($)
{

  my ($metric_name) = @_;

  # some metrics may not havea cache file
  return 1 unless $has::Common::has_metric_config{cache_file}{$metric_name};

  # Dump the metrics to the file
  my $flnm = 
   catfile($has::Common::has_metric_config{cache_dir},$has::Common::has_metric_config{cache_file}{$metric_name});

  unlink($flnm)
   or warn "WARN:Failed to remove the cached metric file $flnm, the metrics loaded may be stale\n"
    if has::Common::hasIsReadable($flnm);

  # if no dependent metric job is done
  return 1 unless $has::Common::has_metric_config{dependent_metric}{$metric_name};

  # remove each dependent metric cached file if it exists
  for my $dp_metric ( @{$has::Common::has_metric_config{dependent_metric}{$metric_name}} )
  {
    has_rm_cache($dp_metric);
  }

  return 1;

}

# name : has_rd_cache
# desc : reach the cached files generated for this metric
#
# arg  :
#  metric_name
#
# return:
#
sub has_rd_cache($)
{
  my ($metric_name) = @_;
  my $pstring;

  #my $file;
  #$file = catfile($has::Common::has_metric_config{cache_dir}, $has::Common::has_metric_config{cache_file}{$metric_name});

  # Open the file for reading , if it fails dont return an error,
  # log an error and return gracefully, so metric is blank but with errror
  $pstring = has::Common::hasReturnFileContents($has::Common::has_metric_config{cache_dir}, $has::Common::has_metric_config{cache_file}{$metric_name});

  print $pstring if $pstring;

  return 1;

}

# name : has_dump_to_cache_file
# desc : dump the results for all metrics in the result ref to the cache file 
#        for metric
#
# arg  :
#  parent metric_name
#  ref to results hash for metric
#
# return:
#
sub has_dump_to_cache_file($$;)
{

  my ($metric_name,$resref) = @_;

  for my $mn ( keys %{$resref} )
  {
    # some of the metrics do not require to be cached and printed
    next unless $has::Common::has_metric_config{cache_file}{$mn};

     has::Common::hasWriteToFile($has::Common::has_metric_config{cache_dir},$has::Common::has_metric_config{cache_file}{$mn},\@{$resref->{$mn}}) 
      or warn "WARN:Failed to cache metric $mn to cache file" and return;
  }

  return 1;
}



# name : has_entities
# desc : fn to instrument metrics for the metric_name specified
#
# arg  :
#  metric_name
#  crsHome
#  arsg to the os command
#
# return:
#
sub has_entities($;@)
{
  my ($metric_name,$crsHome,$args) = @_;

  my %reslist;
  my  %xmlvar;

  # setup the result has with the metrics and dependent metrics to be generated
  $reslist{$metric_name} = [()];
  # dependent metrics
  for my $dp_metric_name
  ( keys %{$has::Common::has_metric_config{dependent_metric_functions}{$metric_name}} )
  {
    $reslist{$dp_metric_name} = [()];
  }

  # cant use the cache , need to instrument the metric, se we begin processing
  has::Common::hasSetCRSEnv($crsHome);
  my $cmdresults = has::Common::runsystemcommand($has::Common::has_metric_config{command}{$metric_name},
        $has::Common::has_metric_config{arg}{$metric_name});
  has::Common::hasRestoreCRSEnv();

  $cmdresults = has::Common::hasCheckAndReturnEmcrspResults($cmdresults);

  warn "WARN:Failed to get results from command - $has::Common::has_metric_config{command}{$metric_name} $has::Common::has_metric_config{arg}{$metric_name}" unless $cmdresults;

  %xmlvar =  has::Common::parse_xml($cmdresults) if $cmdresults;

  if ( $cmdresults )
  {
   if  ( $metric_name =~ /^resource_types$/i )
   {
      # execute function to mark base types and base type for attribs etc
      has::Common::hasadm_resource_types_mark_base_types(\%xmlvar);  
   }

   has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&has::Common::has_entity_fn,\%reslist,$metric_name,$crsHome,$args) or 
    warn "WARN:Failed to traverse the xml while instrumenting metric $metric_name" and return;
  }


 if ( $has::Common::has_metric_config{aggregate_computation}{$metric_name} )
 {

   my @aggMetricList;

   push @aggMetricList,$metric_name;

   if ( $has::Common::has_metric_config{dependent_metric_functions}{$metric_name} )
   {
      for my $k ( keys %{$has::Common::has_metric_config{dependent_metric_functions}{$metric_name}} )
      {
       push @aggMetricList,$k;
      }
   }

   for my $agg_metric ( @aggMetricList )
   {

     delete $reslist{$agg_metric};
     delete $reslist{$agg_metric.'_key_value'};
  
       # transfer the computed composite state to each elref
       for my $key1 ( keys %{$reslist{hash_ref}{$agg_metric}{1}} )
         {
           my $elref2 = $reslist{hash_ref}{$agg_metric}{1}{$key1};
           next unless $elref2;
           next unless ref($elref2) =~ /HASH/i;
           next unless $elref2->{elem_attribs};
  
           # try to get the composite state of a resource based on states of 
           # resource instances
           if ( $agg_metric =~ /resource_instance_status_metric|resource_instance_alert_metric/ )
           {
               next unless $elref2->{resource_ref};
               my $resource_ref = $elref2->{resource_ref};
               next unless ref($resource_ref) =~ /HASH/i;
               next unless $resource_ref->{elem_attribs};
      
               next unless $resource_ref->{elem_attribs}{COMPOSITE_STATE};
      
               $elref2->{elem_attribs}{COMPOSITE_STATE}=
                 $resource_ref->{elem_attribs}{COMPOSITE_STATE};

               $elref2->{elem_attribs}{ALERT_COUNT}=
                 $resource_ref->{elem_attribs}{ALERT_COUNT} 
                 if defined $resource_ref->{elem_attribs}{ALERT_COUNT};

               $elref2->{elem_attribs}{ALERT_STATE}=
                 $resource_ref->{elem_attribs}{ALERT_STATE} 
                 if defined $resource_ref->{elem_attribs}{ALERT_STATE};
            }
  
            # build the em_results row
            has::Common::has_em_results($elref2,\%reslist,$agg_metric);
  
         }
  
  
   }

 }

 has_dump_to_cache_file($metric_name,\%reslist) or 
  warn "WARN:Failed to dump metric results to cache file while instrumenting metric $metric_name" and return;

  return 1;

}


# name : has_cust_attribs
# desc : for each element passed it filters our the custom attributes
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
# 
sub has_cust_attribs($$$)
{
  my ( $elref, $rsref, $metric_name ) = @_;
  my $parent_metric_name;

  # storage an empty value, if there are no custom attribs the metric exec 
  #  should not fail
  $rsref->{$metric_name} = [()] unless $rsref->{$metric_name};

  # this is a function for dependent metrics, get the parent metric_name
  warn "WARN:Failed, parent metric is not defined for dependent metric $metric_name\n"
   and return
    unless $has::Common::has_metric_config{parent_metric}{$metric_name};

  $parent_metric_name = $has::Common::has_metric_config{parent_metric}{$metric_name};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs};

  # build the result string, custom attribs are not in the basic_fields list 
  # of the parent metric

  # build the result string
  # call fn to build the result string
  # build the result string
  for my $pname ( keys %{$elref->{elem_attribs_properites}} )
  {

    my $pref = $elref->{elem_attribs_properites}{$pname};

    # do this special casing for resource type attributes
    # if they are not strings they should have a numberic value
    if ( $metric_name  =~ /^resource_type_attributes$/ )
    {
       if ( $pref->{type} and $pref->{type} =~ /(INTEGER|NUMBER)/i )
       {
           $pref->{default_value} = 0 unless $pref->{default_value};
           $pref->{default_value} = 0 if $pref->{default_value} =~ /_NULL|NULL/i;

           warn "WARN:has_metrics.pl: Default Value for resource type attribute $pref->{name} is $pref->{default_value} and not a $pref->{type}"
            unless $pref->{default_value} =~ /\d+/;
       }
    }

    # build the em_results row
    has::Common::has_em_results($elref,$rsref,$metric_name,$pref);

  }

  return 1;

}


# name : has_servergroup_members
# desc : for each servergroup element passed it gets the members and instruments them
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
# 
sub has_servergroup_members(\%\%$)
{
  my ( $elref, $rsref, $metric_name ) = @_;
  my $parent_metric_name;

  # storage an empty value, if there are no custom attribs the metric exec 
  #  should not fail
  $rsref->{$metric_name} = [()] unless $rsref->{$metric_name};

  # this is a function for dependent metrics, get the parent metric_name
  warn "WARN:has_metrics.pl:has_servergroup_members Failed, parent metric is not defined for dependent metric $metric_name\n"
   and return unless $has::Common::has_metric_config{parent_metric}{$metric_name};

  $parent_metric_name = $has::Common::has_metric_config{parent_metric}{$metric_name};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs}{SERVER_NAMES};

  # get the list of servers fro the servergroup
  my @servers =  split /\,/,$elref->{elem_attribs}{SERVER_NAMES};

  # get the list of active servers for this servergroup
  my %activeservers;

  # create a hash list of activeservers
  if ( $elref->{elem_attribs}{ACTIVE_SERVERS} )
  {
    for my $activeserver (  split /\,/,$elref->{elem_attribs}{ACTIVE_SERVERS} )
    {
      $activeservers{$activeserver}=1; 
    }
  }


  # build the result string, custom attribs are not in the basic_fields list 
  # of the parent metric

  # build the result string
  # call fn to build the result string
  # build the result string
  # get the member entities from the array xml
  for my $server ( @servers )
  {
   
    my $pref;

    # create a hash ref for each server member to be instrumented
    $pref->{SERVER_NAME}=$server;
    $pref->{ACTIVE_SERVER}=1 if $activeservers{$server};
    $pref->{ACTIVE_SERVER}=0 unless $pref->{ACTIVE_SERVER};

    # build the em_results row
    has::Common::has_em_results($elref,$rsref,$metric_name,$pref);

 }

 return 1;

}


# name : has_servergroup_resource
# desc : for each resource element passed it gets the server_groups and instruments them
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
# 
sub has_servergroup_resource($$$)
{
  my ( $elref, $rsref, $metric_name ) = @_;
  my $parent_metric_name;

  # storage an empty value, if there are no custom attribs the metric exec 
  #  should not fail
  $rsref->{$metric_name} = [()] unless $rsref->{$metric_name};

  # this is a function for dependent metrics, get the parent metric_name
  warn "WARN:has_metrics.pl:has_servergroup_resource Failed, parent metric is not defined for dependent metric $metric_name\n"
   and return unless $has::Common::has_metric_config{parent_metric}{$metric_name};

  $parent_metric_name = $has::Common::has_metric_config{parent_metric}{$metric_name};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs}{SERVER_GROUPS};

  # get the list of servers fro the servergroup
  my @sgs =  split /\,/,$elref->{elem_attribs}{SERVER_GROUPS};

  # build the result string, custom attribs are not in the basic_fields list 
  # of the parent metric

  # build the result string
  # call fn to build the result string
  # build the result string
  # get the member entities from the array xml
  for my $sg ( @sgs )
  {
   
    my $pref;

    $sg =~ s/^\s+|\s+$//g if $sg;

    next unless $sg;

    # create a hash ref for each server group to be instrumented
    $pref->{SERVER_GROUP_NAME}=$sg;

    # build the em_results row
    has::Common::has_em_results($elref,$rsref,$metric_name,$pref);

 }

 return 1;

}



# name : has_resource_instance_status
# desc : instrument the resource status metric from resource instance status metric
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
# 
sub has_resource_instance_status($$$)
{
  my ( $elref, $rsref, $metric_name ) = @_;
  my $parent_metric_name;

  # storage an empty value, if there are no custom attribs the metric exec 
  #  should not fail
  $rsref->{$metric_name} = [()] unless $rsref->{$metric_name};

  # this is a function for dependent metrics, get the parent metric_name
  warn "WARN:has_metrics.pl:has_resource_instance_status Failed, parent metric is not defined for dependent metric $metric_name\n"
   and return unless $has::Common::has_metric_config{parent_metric}{$metric_name};

  $parent_metric_name = $has::Common::has_metric_config{parent_metric}{$metric_name};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs}{NAME};

  my $gen_key_value = has::Common::has_gen_key_value($elref,$metric_name);

  warn "ERROR:has_metrics.pl:has_resource_instance_status, Failed to generated key value for metric $metric_name for $elref->{elem_attribs}{NAME}"
    and return 1 unless $gen_key_value;

  $elref->{elem_attribs}{GENERATED_KEY_2}=$gen_key_value;

  my $reselref;
  if  ( $rsref->{hash_ref}
        and $rsref->{hash_ref}{$metric_name}
        and $rsref->{hash_ref}{$metric_name}{1}{$gen_key_value}
        and ref($rsref->{hash_ref}{$metric_name}{1}{$gen_key_value}) =~ /HASH/ )
    {
      $reselref = $rsref->{hash_ref}{$metric_name}{1}{$gen_key_value};
    }

  $reselref = $elref unless $reselref;
  $elref->{resource_ref}=$reselref;


  $reselref->{elem_attribs}{TOTAL_COUNT}++;
  if ( $elref->{elem_attribs}{STATE} =~ /ONLINE/ )
    {
      $reselref->{elem_attribs}{ONLINE_COUNT}++;
    }
  elsif( $elref->{elem_attribs}{STATE} =~ /OFFLINE/ )
    {
      $reselref->{elem_attribs}{OFFLINE_COUNT}++;
    }
  elsif ( $elref->{elem_attribs}{STATE} =~ /UNKNOWN/ )
    {
      $reselref->{elem_attribs}{UNKNOWN_COUNT}++;
    }
  elsif ( $elref->{elem_attribs}{STATE} =~ /INTERMEDIATE/ )
    {
      $reselref->{elem_attribs}{INTERMEDIATE_COUNT}++;
    }
  else
    {
      $reselref->{elem_attribs}{UNKNOWN_COUNT}++;
    }

  # non online instance 
  if ( not $reselref->{elem_attribs}{ONLINE_COUNT}  and not $reselref->{elem_attribs}{INTERMEDIATE_COUNT} )
    {
      if ( $reselref->{elem_attribs}{OFFLINE_COUNT} )
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'COMPLETE_OFFLINE';
      }
      elsif ( $reselref->{elem_attribs}{UNKNOWN_COUNT} )
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'COMPLETE_UNKNOWN';
      }
      else
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'COMPLETE_OFFLINE';
      }

    }
  # all online instances
  elsif ( $reselref->{elem_attribs}{ONLINE_COUNT} and 
          $reselref->{elem_attribs}{ONLINE_COUNT} == $reselref->{elem_attribs}{TOTAL_COUNT}
        )
    {
      $reselref->{elem_attribs}{COMPOSITE_STATE} = 'COMPLETE_ONLINE';
    }
  # all intermediate instances
  elsif ( $reselref->{elem_attribs}{INTERMEDIATE_COUNT} and 
          $reselref->{elem_attribs}{INTERMEDIATE_COUNT} == $reselref->{elem_attribs}{TOTAL_COUNT}
        )
    {
      $reselref->{elem_attribs}{COMPOSITE_STATE} = 'COMPLETE_INTERMEDIATE';
    }
  # some online instances
  elsif ( $reselref->{elem_attribs}{ONLINE_COUNT} and 
          $reselref->{elem_attribs}{ONLINE_COUNT} < $reselref->{elem_attribs}{TOTAL_COUNT}
        )
    {

      if ( $reselref->{elem_attribs}{UNKNOWN_COUNT} )
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'PARTIALLY_UNKNOWN';
      }
      elsif ( $reselref->{elem_attribs}{OFFLINE_COUNT} )
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'PARTIALLY_OFFLINE';
      }
      elsif ( $reselref->{elem_attribs}{INTERMEDIATE_COUNT} )
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'PARTIALLY_INTERMEDIATE';
      }
      else
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'PARTIALLY_OFFLINE';
      }

    }
  # some intermediate instances
  elsif ( $reselref->{elem_attribs}{INTERMEDIATE_COUNT} and 
          $reselref->{elem_attribs}{INTERMEDIATE_COUNT} < $reselref->{elem_attribs}{TOTAL_COUNT}
        )
    {

      if ( $reselref->{elem_attribs}{UNKNOWN_COUNT} )
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'PARTIALLY_UNKNOWN';
      }
      elsif ( $reselref->{elem_attribs}{OFFLINE_COUNT} )
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'PARTIALLY_OFFLINE';
      }
      else 
      {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'PARTIALLY_INTERMEDIATE';
      }

    }
  else
   {
        $reselref->{elem_attribs}{COMPOSITE_STATE} = 'COMPLETE_UNKNOWN';
   }

  $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE}='' 
    unless $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE};

  # set the string for the message
  if ( $reselref->{elem_attribs}{COMPOSITE_STATE} =~ /COMPLETE_OFFLINE/ )
  {
   $reselref->{elem_attribs}{ALERT_STATE}='OFFLINE';
   $reselref->{elem_attribs}{ALERT_COUNT}=$reselref->{elem_attribs}{OFFLINE_COUNT};
   #$reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE} .=" ( 0 ONLINE / INTERMEDIATE )"
   #  unless $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE} =~ / \( 0 ONLINE \/ INTERMEDIATE \)/;
  }
  elsif (  $reselref->{elem_attribs}{COMPOSITE_STATE} =~ /COMPLETE_UNKNOWN/ )
  {
   $reselref->{elem_attribs}{ALERT_STATE}='UNKNOWN';
   $reselref->{elem_attribs}{ALERT_COUNT}=$reselref->{elem_attribs}{UNKNOWN_COUNT};
   #$reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE} .=" ( 0 ONLINE / INTERMEDIATE )"
   #  unless $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE} =~ / \( 0 ONLINE \/ INTERMEDIATE \)/;
  }
  elsif (  $reselref->{elem_attribs}{COMPOSITE_STATE} =~ /COMPLETE_INTERMEDIATE/ )
  {
   $reselref->{elem_attribs}{ALERT_STATE}='INTERMEDIATE';
   $reselref->{elem_attribs}{ALERT_COUNT}=$reselref->{elem_attribs}{INTERMEDIATE_COUNT};
   #$reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE}=~ s/ \( 0 ONLINE \/ INTERMEDIATE \)//g
   #  if $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE} =~ / \( 0 ONLINE \/ INTERMEDIATE \)/;
  }
  elsif ( $reselref->{elem_attribs}{COMPOSITE_STATE} =~ /PARTIALLY_OFFLINE/ )
  {
   $reselref->{elem_attribs}{ALERT_STATE}='OFFLINE';
   $reselref->{elem_attribs}{ALERT_COUNT}='';
   #$reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE}=~ s/ \( 0 ONLINE \/ INTERMEDIATE \)//g
   #  if $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE} =~ / \( 0 ONLINE \/ INTERMEDIATE \)/;
  }
  elsif (  $reselref->{elem_attribs}{COMPOSITE_STATE} =~ /PARTIALLY_UNKNOWN/ )
  {
   $reselref->{elem_attribs}{ALERT_STATE}='UNKNOWN';
   $reselref->{elem_attribs}{ALERT_COUNT}='';
   #$reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE}=~ s/ \( 0 ONLINE \/ INTERMEDIATE \)//g
   #  if $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE} =~ / \( 0 ONLINE \/ INTERMEDIATE \)/;
  }
  elsif (  $reselref->{elem_attribs}{COMPOSITE_STATE} =~ /PARTIALLY_INTERMEDIATE/ )
  {
   $reselref->{elem_attribs}{ALERT_STATE}='INTERMEDIATE';
   $reselref->{elem_attribs}{ALERT_COUNT}='';
   #$reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE}=~ s/ \( 0 ONLINE \/ INTERMEDIATE \)//g
   #  if $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE} =~ / \( 0 ONLINE \/ INTERMEDIATE \)/;
  }
  elsif (  $reselref->{elem_attribs}{COMPOSITE_STATE} =~ /COMPLETE_ONLINE/ )
  {
   $reselref->{elem_attribs}{ALERT_STATE}='ONLINE';
   $reselref->{elem_attribs}{ALERT_COUNT}=$reselref->{elem_attribs}{ONLINE_COUNT};
   #$reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE}=~ s/ \( 0 ONLINE \/ INTERMEDIATE \)//g
   #  if $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE} =~ / \( 0 ONLINE \/ INTERMEDIATE \)/;
  }

  $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE}='' 
    unless $reselref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE};

  # we want the list of resources from resource instances
  # if this name is already on the resource_instance_metric metric list skip
  if ( $rsref->{$metric_name.'_key_value'}
       and $rsref->{$metric_name.'_key_value'}{1}
       and  $rsref->{$metric_name.'_key_value'}{1}{$gen_key_value}
     )
    {
      return 1;
    }

  # build the em_results row
  has::Common::has_em_results($reselref,$rsref,$metric_name);

  # build the result string, custom attribs are not in the basic_fields list 
  # of the parent metric

  return 1;

}



# name : has_gpnp_server_pool
# desc : instrument the crs servers metric from 
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
#  crsHome being processed
# 
sub has_gpnp_server_pool($$$$)
{

  my ( $elref, $rsref, $metric_name,$crsHome ) = @_;
  my $parent_metric_name;
  my $args;

  # storage an empty value, if there are no custom attribs the metric exec 
  #  should not fail
  if ( not $rsref->{$metric_name} )
  {
    $rsref->{$metric_name} = [()];
  }

  # if server pool dependent metrics has already been instrumented
  return 1 if $rsref and ref($rsref)=~ /HASH/i and $rsref->{$metric_name} 
    and ref($rsref->{$metric_name}) =~/HASH/i and keys %{$rsref->{$metric_name}};

  # this is a function for dependent metrics, get the parent metric_name
  warn "WARN:has_metrics.pl:has_gpnp_server_pool Failed, parent metric is not defined for dependent metric $metric_name\n"
   and return unless $has::Common::has_metric_config{parent_metric}{$metric_name};

  $parent_metric_name = $has::Common::has_metric_config{parent_metric}{$metric_name};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs}{NAME};


 # get the crs server state and cache it
 if (  not $rsref or not ref($rsref) =~ /HASH/i
        or not $rsref->{has_gpnp_server_pool} )
 {
    my %gpnp_server_pool_result;
  
    # cant use the cache , need to instrument the metric, se we begin processing
    has::Common::hasSetCRSEnv($crsHome);
    my $cmdresults = has::Common::runsystemcommand($has::Common::has_metric_config{command}{$metric_name},
          $has::Common::has_metric_config{arg}{$metric_name});
    has::Common::hasRestoreCRSEnv();
  
    $cmdresults = has::Common::hasCheckAndReturnEmcrspResults($cmdresults);
  
    warn "WARN:has_metrics.pl:has_gpnp_server_pool Failed to get results from command - $has::Common::has_metric_config{command}{$metric_name} $has::Common::has_metric_config{arg}{$metric_name}" unless $cmdresults;
  
    my %xmlvar =  has::Common::parse_xml($cmdresults) if $cmdresults;
  
    if ( $cmdresults )
    {
  
     has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,
       \&has::Common::has_gpnp_server_pool_fn,\%gpnp_server_pool_result,
       $metric_name,$crsHome,$args) or 
      warn "WARN:has_metrics.pl:has_gpnp_server_pool Failed to traverse the xml while instrumenting metric $metric_name" and return;

     if ( keys %gpnp_server_pool_result 
           and $gpnp_server_pool_result{$metric_name} 
            and keys %{$gpnp_server_pool_result{$metric_name}} )
     {
       $rsref->{$metric_name} = $gpnp_server_pool_result{$metric_name};
     }
    }
  }

  return 1;

}

# name : has_resource_crs_servers
# desc : instrument the crs servers metric from 
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
#  crsHome being processed
# 
sub has_resource_crs_servers($$$$)
{


  my ( $elref, $rsref, $metric_name,$crsHome ) = @_;
  my $parent_metric_name;
  my $args;

  # storage an empty value, if there are no custom attribs the metric exec 
  #  should not fail
  if ( not $rsref->{$metric_name} )
  {
    $rsref->{$metric_name} = [()];
 
  }

  # this is a function for dependent metrics, get the parent metric_name
  warn "WARN:has_metrics.pl:has_resource_crs_servers Failed, parent metric is not defined for dependent metric $metric_name\n"
   and return unless $has::Common::has_metric_config{parent_metric}{$metric_name};

  $parent_metric_name = $has::Common::has_metric_config{parent_metric}{$metric_name};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs}{NAME};



 # get the crs server state and cache it
 if (  not $rsref or not ref($rsref) =~ /HASH/i
        or not $rsref->{crs_server_results} )
 {
    my %crs_server_results;
  
    # cant use the cache , need to instrument the metric, se we begin processing
    has::Common::hasSetCRSEnv($crsHome);
    my $cmdresults = has::Common::runsystemcommand($has::Common::has_metric_config{command}{$metric_name},
          $has::Common::has_metric_config{arg}{$metric_name});
    has::Common::hasRestoreCRSEnv();
  
    $cmdresults = has::Common::hasCheckAndReturnEmcrspResults($cmdresults);
  
    warn "WARN:Failed to get results from command - $has::Common::has_metric_config{command}{$metric_name} $has::Common::has_metric_config{arg}{$metric_name}" unless $cmdresults;
  
    my %xmlvar =  has::Common::parse_xml($cmdresults) if $cmdresults;
  
    if ( $cmdresults )
    {
  
     has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,
       \&has::Common::has_resource_crs_servers_fn,\%crs_server_results,
       $metric_name,$crsHome,$args) or 
      warn "WARN:Failed to traverse the xml while instrumenting metric $metric_name" and return;

     if ( keys %crs_server_results 
           and $crs_server_results{$metric_name} 
            and keys %{$crs_server_results{$metric_name}} )
     {
       $rsref->{crs_server_results} = $crs_server_results{$metric_name};

        my $nodelist;
        # generate the crs node list
        for my $sname ( keys %{$crs_server_results{$metric_name}} )
        {
          if ( $nodelist )
          {
            $nodelist = "$nodelist$sname\;";
          }
          else
          {
             $nodelist = "$sname\;";
          }
        }

        if ( $nodelist )
        {
          $rsref->{crs_server_CSS_NODE_LIST}=$nodelist;
          $rsref->{crs_server_CSS_NODE_LIST1}=substr($nodelist,0,2000);
          
          if ( length($nodelist) > 2000 )
          {
            $rsref->{crs_server_CSS_NODE_LIST2}=substr($nodelist,2000,2000); 
          }
        }
     }
    }
  
  }

  # get crs server state from emcrsp -e server option
  if ( $rsref and ref($rsref) =~ /HASH/i 
        and $rsref->{crs_server_results} 
        and $rsref->{crs_server_results}{$elref->{elem_attribs}{NAME}} )
  {

   for my $k ( keys %{$rsref->{crs_server_results}{$elref->{elem_attribs}{NAME}}} )
   {
     $elref->{elem_attribs}{$k} = 
       $rsref->{crs_server_results}{$elref->{elem_attribs}{NAME}}{$k};
   }

    $elref->{elem_attribs}{CSS_NODE_LIST}=$rsref->{crs_server_CSS_NODE_LIST}
     if $rsref->{crs_server_CSS_NODE_LIST};
    $elref->{elem_attribs}{CSS_NODE_LIST1}=$rsref->{crs_server_CSS_NODE_LIST1}
     if $rsref->{crs_server_CSS_NODE_LIST1};
    $elref->{elem_attribs}{CSS_NODE_LIST2}=$rsref->{crs_server_CSS_NODE_LIST2}
     if $rsref->{crs_server_CSS_NODE_LIST2};
  }
  else
  {
    return 1;
  }


  my $gen_key_value = has::Common::has_gen_key_value($elref,$metric_name);

  warn "ERROR:has_metrics.pl:has_resource_crs_servers, Failed to generated key value for metric $metric_name for $elref->{elem_attribs}{NAME}"
    and return 1 unless $gen_key_value;

  $elref->{elem_attribs}{GENERATED_KEY_2}=$gen_key_value;
  my $reselref = $elref;

  # we want the list of resources from resource instances
  # if this name is already on the resource_instance_metric metric list skip
  if ( $rsref->{$metric_name.'_key_value'}
       and $rsref->{$metric_name.'_key_value'}{1}
       and  $rsref->{$metric_name.'_key_value'}{1}{$gen_key_value}
     )
    {
      return 1;
    }

  # build the em_results row
  has::Common::has_em_results($reselref,$rsref,$metric_name);

  # build the result string, custom attribs are not in the basic_fields list 
  # of the parent metric

  return 1;
}


# name : has_instances
# desc : for each element passed it gets the instances and instruments them
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
# 
sub has_instances(\%\%$)
{
  my ( $elref, $rsref, $metric_name ) = @_;
  my $parent_metric_name;

  # storage an empty value, if there are no custom attribs the metric exec 
  #  should not fail
  $rsref->{$metric_name} = [()] unless $rsref->{$metric_name};

  # this is a function for dependent metrics, get the parent metric_name
  warn "WARN:Failed, parent metric is not defined for dependent metric $metric_name\n"
   and return
    unless $has::Common::has_metric_config{parent_metric}{$metric_name};

  $parent_metric_name = $has::Common::has_metric_config{parent_metric}{$metric_name};

  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs};

  # check if it has attributes
  return 1 unless $elref->{children} 
   and $elref->{child_elements}{instances}
    and ref($elref->{child_elements}{instances}) =~ /ARRAY/i
     and @{$elref->{child_elements}{instances}};

  my $instancesref = @{$elref->{child_elements}{instances}}[0];

  return 1 unless $instancesref->{children} and
    $instancesref->{child_elements}{instance}
     and  ref($instancesref->{child_elements}{instance}) =~ /ARRAY/i
      and @{$instancesref->{child_elements}{instance}};

  # get the instance entities from the array xml
  for my $instanceref ( @{$instancesref->{child_elements}{instance}} )
  {
     next unless $instanceref->{children} and $instanceref->{child_elements}{node};

     my $noderef = @{$instanceref->{child_elements}{node}}[0];

     next unless $instanceref->{children} and $instanceref->{child_elements}{status};

     my $statusref = @{$instanceref->{child_elements}{status}}[0];

     my $retstr = 
      "em_result=$elref->{attrs}{entity_type}|$elref->{attrs}{entity_name}|$noderef->{name}|$statusref->{name}";

     push @{$rsref->{$metric_name}},$retstr;

 }

 return 1;

}


# name : has_crs_nodes
# desc :  instrument metrics for nodes of the cluster
#
# arg  :
#  metric_name
#  args to the os command, # CRSHome
#
# return:
#
sub has_crs_nodes($;$)
{

  my ($metric_name,$crsHome ) = @_;

  my $results;
  my $noderef;
  my $clusterName;

  $noderef = has::Common::hasGetClusterNodes($crsHome);
  $clusterName = has::Common::hasGetClusterName($crsHome);

  my %rshash;

  return 1 unless $noderef and keys %{$noderef} and $noderef->{details} and keys %{$noderef->{details}};

  for my $nodeName ( keys  %{$noderef->{details}} )
  {
    my %rsxml;

    $rsxml{elem_attribs}{CLUSTER_NAME} = $clusterName;
    $rsxml{elem_attribs}{NAME} = $nodeName;
    $rsxml{elem_attribs}{TYPE} = $has::Common::has_em_targettype;
    $rsxml{elem_attribs}{SOFTWARE_VERSION} = has::Common::hasGetClusterSoftwareVersion($nodeName);

    $rsxml{elem_attribs}{CLUSTER_NAME} = $noderef->{cluster_name}
     if $noderef->{cluster_name} and not $rsxml{elem_attribs}{CLUSTER_NAME};

    if ( $noderef and $noderef->{details} and $noderef->{details}{$nodeName} )
    {
      $rsxml{elem_attribs}{HOST_NAME} = $noderef->{details}{$nodeName}{HOST_NAME} 
       if $noderef->{details}{$nodeName}{HOST_NAME};
  
      $rsxml{elem_attribs}{NODE_STATE} = $noderef->{details}{$nodeName}{NODE_STATE} 
       if $noderef->{details}{$nodeName}{NODE_STATE};
  
      $rsxml{elem_attribs}{CRS_HOME} = $noderef->{details}{$nodeName}{HOME} 
       if $noderef->{details}{$nodeName}{HOME};

      $rsxml{elem_attribs}{CRS_HOME} = $noderef->{details}{$nodeName}{crs_home} 
       if $noderef->{details}{$nodeName}{crs_home} 
       and not $rsxml{elem_attribs}{CRS_HOME};

    }

    $rsxml{elem_attribs}{CRS_HOME} = $noderef->{HOME} 
     if $noderef->{HOME} and not $rsxml{elem_attribs}{CRS_HOME};

    $rsxml{elem_attribs}{CRS_HOME} = has::Common::hasGetNodeCRSHome($nodeName,$crsHome)
     unless $rsxml{elem_attribs}{CRS_HOME};

    has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  }

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}

# name : has_crs_metric
# desc : instrument the metric for crs ,css
#          to be executed for the cluster target 
#          gets crs/css for each node as host metric
#
# arg  :
#  metric_name
#  args to the os command, # CRSHome
#
# return:
#
sub has_crs_metric($;$)
{

  my ($metric_name,$crsHome ) = @_;

  my $results;
  my $noderef;
  my $clusterName;

  $noderef = has::Common::hasGetClusterNodes($crsHome);
  $clusterName = has::Common::hasGetClusterName($crsHome);

  my %rshash;

  return 1 unless $noderef and keys %{$noderef} and $noderef->{details} and keys %{$noderef->{details}};

  for my $nodeName ( keys  %{$noderef->{details}} )
  {
    my %rsxml;

    $rsxml{elem_attribs}{CLUSTER_NAME} = $clusterName;
    $rsxml{elem_attribs}{CLUSTER_NAME} = $noderef->{cluster_name}
     if $noderef->{cluster_name} and not $rsxml{elem_attribs}{CLUSTER_NAME};

    $rsxml{elem_attribs}{TYPE} = $has::Common::has_em_targettype;

    $rsxml{elem_attribs}{NODE_NAME} = $nodeName;

    $rsxml{elem_attribs}{HOST_NAME} = $noderef->{details}{$nodeName}{HOST_NAME} 
     if $noderef->{details}{$nodeName}{HOST_NAME};

    $rsxml{elem_attribs}{CRS_HOME} = $noderef->{details}{$nodeName}{HOME} 
     if $noderef->{details}{$nodeName}{HOME};

    $rsxml{elem_attribs}{CRS_HOME} = $noderef->{HOME} 
     if $noderef->{HOME} and not $rsxml{elem_attribs}{CRS_HOME};

    $rsxml{elem_attribs}{CRS_HOME} = has::Common::hasGetNodeCRSHome($nodeName,$crsHome)
     unless $rsxml{elem_attribs}{CRS_HOME};

    has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  }

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}


# name : has_scan_metric
# desc : instrument scan metrics
#
# arg  :
#  metric_name
#  args to the os command, # CRSHome
#
# return:
#
sub has_scan_metric($;$)
{

  my ($metric_name,$crsHome ) = @_;

  my $clusterName;
  my $scanName;
  my $scanIP;
  my $scanPort;
  my %rshash;

 if ( not has::Common::hasCheckForEmcrsp($crsHome) )
 {
  if ( $crsHome )
  {
    warn "WARN:has_metrics.pl:has_scan_metric:This is a pre 11gR2 cluster, binary emcrsp is not found at $crsHome";
  }
  else
  {
    warn "WARN:has_metrics.pl:has_scan_metric:This is a pre 11gR2 cluster, binary emcrsp is not found";
  }
   return;
 }

  $clusterName = has::Common::hasGetClusterName($crsHome);
  $scanName = has::Common::hasGetScanName($crsHome);
  $scanIP = has::Common::hasGetScanIP($crsHome);
  $scanPort = has::Common::hasGetScanPort($crsHome);

  my %rsxml;
  $rsxml{elem_attribs}{CLUSTER_NAME} = $clusterName;
  $rsxml{elem_attribs}{SCAN_NAME} = $scanName;
  $rsxml{elem_attribs}{IP} = $scanIP;
  $rsxml{elem_attribs}{SCAN_PORT} = $scanPort;
  $rsxml{elem_attribs}{RESOURCE_NAME} = $scanName;

  has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}


# name : has_cluster_health_check_metric
# desc : instrument cluster health check metrics
#
# arg  :
#  metric_name
#  args to the os command, # CRSHome
#
# return:
#
sub has_cluster_health_check_metric($;$)
{

  my ($metric_name,$crsHome ) = @_;

  my %rshash;
  my $healthref;

  $healthref = has::Common::hasClusterHealthCheck('crs',$crsHome);

  return unless $healthref and ref($healthref) =~ /HASH/i and keys %$healthref;

  for my $k ( keys %{$healthref} )
  {
   my $resref = $healthref->{$k};
   next unless $resref or ref($resref) =~ /HASH/i or keys %{$resref};

   my %rsxml = %$resref;
   has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  }

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}



# name : has_asm_cmdb_metric
# desc : instrument the asm cmdb metric 
#
# arg  :
#  metric_name
#  target_name
#  oracle home
#  oracle type 
#  machine name
#  oracle name
#
# return:
#
sub has_asm_cmdb_metric($$$$;$$)
{

  my ($metric_name,$targetName,$oracleHome,$oracleType,$machineName,$oracleName) = @_;

  my %rshash;
  my %stdinArgs;
  my $password ;
  my $username ;
  my $dbOcrType;
  my $address;
  my $role;
  my $dbOracleHome;

  # Get the username,password and target database base address from
  # fetchlet environment
  %stdinArgs = has::Common::get_stdinvars();
   
   # pupulate the credentials array
   my %credentials;
   $credentials{username} = $stdinArgs{EM_TARGET_USERNAME} if $stdinArgs{EM_TARGET_USERNAME};
   $credentials{password} = $stdinArgs{EM_TARGET_PASSWORD} if $stdinArgs{EM_TARGET_PASSWORD};
   $credentials{address} = $ENV{EM_TARGET_ADDRESS} if $ENV{EM_TARGET_ADDRESS};
   $credentials{role} = $ENV{EM_TARGET_ROLE} if $ENV{EM_TARGET_ROLE};
   $credentials{ORACLE_HOME} = $ENV{EM_TARGET_ORACLE_HOME} if $ENV{EM_TARGET_ORACLE_HOME};

   my %sql;
   $sql{asm_metric} = '
        SELECT  DISTINCT cd.isCluster isCluster,
                au.db_unique_name      db_unique_name,
                vi.host_name           host_name,
                av.banner_version      activeVersion
           FROM ( select value db_unique_name ,
                         1 cnt
                    from v$parameter 
                   where name=\'db_unique_name\' 
                     AND ROWNUM = 1) au,
                ( select NVL(value,\'FALSE\') isCluster ,
                         1 cnt
                    from v$parameter 
                   where name=\'cluster_database\' 
                     AND ROWNUM = 1) cd ,
                (
                  select v.banner banner_version,
                         1 cnt
                   from  v$version v
                   where v.banner like \'Oracle%\'
                     and ROWNUM = 1
                ) av,
                (
                  select NVL(instance_number,0) instance_number,
                         host_name,
                         1 cnt
                   from  v$instance 
                ) vi
            WHERE ROWNUM = 1
              and vi.instance_number = 1
              AND vi.cnt = au.cnt(+)
              AND vi.cnt = cd.cnt(+)
              AND vi.cnt = av.cnt(+)';

  $sql{asm_instance_metric}  = '
        SELECT  DISTINCT cd.isCluster isCluster,
                au.db_unique_name    db_unique_name,
                v.version           version,
                av.banner_version    activeVersion,
                v.instance_name     instanceName,
                v.instance_id       instanceId,
                v.instance_number   instanceNumber,
                NULL                 diskDiscoveryString,
                v.host_name         host_name
           FROM ( select value db_unique_name , 1 cnt from v$parameter where name=\'db_unique_name\' AND ROWNUM = 1) au,
                ( select NVL(value,\'FALSE\') isCluster , 1 cnt from v$parameter where name=\'cluster_database\' AND ROWNUM = 1) cd ,
                ( select vi.host_name, 
                         vi.instance_name,
                         vi.instance_number,
                         vi.version,
                         to_char(gv.inst_id) instance_id ,
                         1 cnt
                  from   gv$instance gv,
                         v$instance vi
                   where vi.instance_number = gv.instance_number(+)
                ) v,
                (
                  select v.banner banner_version,
                         1 cnt
                   from  v$version v
                   where v.banner like \'Oracle%\'
                     and ROWNUM = 1
                ) av
            WHERE v.cnt = av.cnt(+)
              AND v.cnt = cd.cnt(+)
              AND v.cnt = au.cnt(+)';

  $sql{asm_diskgroup_metric}{'10gR1'}  = '
        SELECT  DISTINCT cd.isCluster,
                au.db_unique_name,
                vi.host_name,  
                dg.name          diskGroupName,
                dg.group_number  diskGroupNumber,
                dg.total_mb      diskGroupSize,
                NULL             redundancy,
                NULL             compatibility,
                dg.state         state
           FROM v$asm_diskgroup dg,
                ( select value db_unique_name from v$parameter where name=\'db_unique_name\' AND ROWNUM = 1) au,
                ( select value isCluster from v$parameter where name=\'cluster_database\' AND ROWNUM = 1) cd ,
                ( select host_name from v$instance where ROWNUM = 1) vi
          ';
  $sql{asm_diskgroup_metric}{'10gR2'}  = '
        SELECT  DISTINCT cd.isCluster,
                au.db_unique_name,
                vi.host_name,
                dg.name          diskGroupName,
                dg.group_number  diskGroupNumber,
                dg.total_mb      diskGroupSize,
                dg.type          redundancy,
                dg.compatibility compatibility,
                dg.state         state
           FROM v$asm_diskgroup dg,
                ( select value db_unique_name from v$parameter where name=\'db_unique_name\' AND ROWNUM = 1) au,
                ( select value isCluster from v$parameter where name=\'cluster_database\' AND ROWNUM = 1) cd ,
                ( select host_name from v$instance where ROWNUM = 1) vi';


  $sql{asm_disk_metric}='
        SELECT  DISTINCT cd.isCluster,
                au.db_unique_name,
                vi.host_name,
                d.group_number  diskGroupNumber,
                d.disk_number   diskNumber,
                d.path          diskPath,
                d.name          diskName,
                d.mode_status   modeStatus
           FROM v$asm_disk d,
                ( select value db_unique_name from v$parameter where name=\'db_unique_name\' AND ROWNUM = 1) au,
                ( select NVL(value,\'FALSE\') isCluster from v$parameter where name=\'cluster_database\' AND ROWNUM = 1) cd ,
                ( select host_name from v$instance where ROWNUM = 1) vi
   ';

  $sql{asm_diskgroupused_metric}{'10gR1'} = '
    SELECT  DISTINCT cd.isCluster,
           au.db_unique_name,
           vi.host_name,
           d.dg_name diskGroupName, 
           d.db_name databaseName
    FROM
          ((select dg.name as dg_name, 
                   a1.name as db_name
            from   V$ASM_ALIAS a1,  
                   V$ASM_DISKGROUP dg
            where (mod(a1.parent_index, 16777216) = 0)
              and a1.system_created = \'Y\' and a1.alias_directory = \'Y\'
              and dg.group_number = a1.group_number
              and dg.state = \'MOUNTED\'
            group by dg.name, a1.name) 
           UNION ALL
           (select dg.name as dg_name, 
                   a1.name as db_name
            from   V$ASM_ALIAS a1,  
                   V$ASM_DISKGROUP dg
            where (mod(a1.parent_index, 16777216) = 0)
              and a1.system_created = \'Y\' and a1.alias_directory = \'Y\'
              and dg.group_number = a1.group_number
              and dg.state = \'MOUNTED\'
             group by dg.name, a1.name)
           UNION ALL
           (select dg.name as dg_name,
                   a1.name as db_name
            from   V$ASM_ALIAS a1,  
                   V$ASM_DISKGROUP dg
            where (mod(a1.parent_index, 16777216) = 0)
              and a1.system_created = \'Y\' and a1.alias_directory = \'Y\'
              and dg.group_number = a1.group_number
              and dg.state = \'MOUNTED\'   
             group by dg.name, a1.name))i d,
            ( select value db_unique_name from v$parameter where name=\'db_unique_name\' AND ROWNUM = 1) au,
            ( select NVL(value,\'FALSE\') isCluster from v$parameter where name=\'cluster_database\' AND ROWNUM = 1) cd ,
            ( select host_name from v$instance where ROWNUM = 1) vi';


  $sql{asm_diskgroupused_metric}{'10gR2'} = '
     SELECT DISTINCT cd.isCluster,
            au.db_unique_name,
            vi.host_name,
            d.dg_name diskGroupName, 
            d.db_name databaseName
     FROM
          ((select dg.name as dg_name, 
                   a1.name as db_name 
            from V$ASM_ALIAS a1,  
                 V$ASM_DISKGROUP_STAT dg
            where (mod(a1.parent_index, 16777216) = 0)
              and a1.system_created = \'Y\' and a1.alias_directory = \'Y\'
              and dg.group_number = a1.group_number
              and dg.state = \'MOUNTED\'
            group by dg.name, a1.name) 
           UNION ALL
           (select dg.name as dg_name, 
                   a1.name as db_name
            from V$ASM_ALIAS a1,
                 V$ASM_DISKGROUP_STAT dg
            where (mod(a1.parent_index, 16777216) = 0)
              and a1.system_created = \'Y\' and a1.alias_directory = \'Y\'
              and dg.group_number = a1.group_number
              and dg.state = \'MOUNTED\'
             group by dg.name, a1.name)
           UNION ALL
           (select dg.name as dg_name,
                   a1.name as db_name
            from   V$ASM_ALIAS a1,  
                   V$ASM_DISKGROUP_STAT dg
            where (mod(a1.parent_index, 16777216) = 0)
              and a1.system_created = \'Y\' and a1.alias_directory = \'Y\' 
              and dg.group_number = a1.group_number
              and dg.state = \'MOUNTED\'   
             group by dg.name, a1.name)) d,
            ( select value db_unique_name from v$parameter where name=\'db_unique_name\' AND ROWNUM = 1) au,
           ( select NVL(value,\'FALSE\') isCluster from v$parameter where name=\'cluster_database\' AND ROWNUM = 1) cd ,
           ( select host_name from v$instance where ROWNUM = 1) vi
   ';


  my $sqlexecuted;
  if ( not ref($sql{$metric_name})  )
  {
       $credentials{sql} = [ ( $sql{$metric_name} ) ];
       $sqlexecuted = $sql{$metric_name};
  }
  elsif ( $ENV{EM_VERSION_CATEGORY} and  $sql{$metric_name}{$ENV{EM_VERSION_CATEGORY}} )
  {
       $credentials{sql} = [ ( $sql{$metric_name}{$ENV{EM_VERSION_CATEGORY}} ) ];
       $sqlexecuted = $sql{$metric_name}{$ENV{EM_VERSION_CATEGORY}};
  }


  # order of fields in the results
  my %fieldOrder;
  $fieldOrder{$sql{asm_metric}} = 
                      {isCluster=>1 ,
                       db_unique_name =>2,
                       host_name =>3,
                       activeVersion =>4 };

  $fieldOrder{$sql{asm_instance_metric}}=
       {
          isCluster=>1,
          db_unique_name=>2,
          version=>3,
          activeVersion=>4,
          instanceName=>5,
          instanceId=>6,
          instanceNumber=>7,
          diskDiscoveryString=>8,
          host_name=>9
        };

  $fieldOrder{$sql{asm_diskgroup_metric}{'10gR1'}}=
       {
          isCluster=>1,
          db_unique_name=>2,
          diskGroupName=>3,
          diskGroupNumber=>4,
          diskGroupSize=>5,
          redundancy=>6,
          compatibility=>7,
          state=>8,
          host_name=>9
        };

  $fieldOrder{$sql{asm_diskgroup_metric}{'10gR2'}}=
       {
          isCluster=>1,
          db_unique_name=>2,
          diskGroupName=>3,
          diskGroupNumber=>4,
          diskGroupSize=>5,
          redundancy=>6,
          compatibility=>7,
          state=>8,
          host_name=>9
        };

  $fieldOrder{$sql{asm_disk_metric}}=
       {
          isCluster=>1,
          db_unique_name=>2,
          diskGroupNumber=>3,
          diskNumber=>4,
          diskPath=>5,
          diskName=>6,
          modeStatus=>7,
          host_name=>8
        };

  $fieldOrder{$sql{asm_diskgroupused_metric}{'10gR2'}}=
       {
          isCluster=>1,
          db_unique_name=>2,
          diskGroupName=>3,
          databaseName=>4,
          host_name=>5
        };

  $fieldOrder{$sql{asm_diskgroupused_metric}{'10gR1'}}=
       {
          isCluster=>1,
          db_unique_name=>2,
          diskGroupName=>3,
          databaseName=>4,
          host_name=>5
        };

   my $results_ref = has::Common::hasGetSQLResults(\%credentials,\%fieldOrder);

   my $array_ref;

   my $clusterName = has::Common::hasGetClusterName();

   my $resourceName = has::Common::hasGetResourceNameForOracleApp($oracleHome,$oracleType,$oracleName);
   my $isCluster;
   my $hostName;
   my $dbUniqueName;
   my $name1;
   my $name2;

   $array_ref = $results_ref->{$sqlexecuted} if $results_ref and keys %{$results_ref} and $results_ref->{$sqlexecuted};

   if ( $array_ref and ref($array_ref) and ref($array_ref) =~ /LIST|ARRAY/ and @{$array_ref} )
   {
      # parse query results to check if db has has
      for my $row ( @{$array_ref} )
      {
        my %rsxml;
        my %rowhash = %{$row};

        $rsxml{elem_attribs}{IS_CLUSTER} = $rowhash{isCluster} if $rowhash{isCluster};
        $isCluster = $rsxml{elem_attribs}{IS_CLUSTER} if $rsxml{elem_attribs}{IS_CLUSTER};

        $rsxml{elem_attribs}{DB_UNIQUE_NAME} = $rowhash{db_unique_name} if $rowhash{db_unique_name};
        $dbUniqueName = $rsxml{elem_attribs}{DB_UNIQUE_NAME} if $rsxml{elem_attribs}{DB_UNIQUE_NAME};


        $rsxml{elem_attribs}{HOST_NAME} = $rowhash{host_name} if $rowhash{host_name};
        $hostName = $rsxml{elem_attribs}{HOST_NAME} if $rsxml{elem_attribs}{HOST_NAME};


        $rsxml{elem_attribs}{ACTIVE_VERSION} = $rowhash{activeVersion} if $rowhash{activeVersion};
        $rsxml{elem_attribs}{VERSION} = $rowhash{version} if $rowhash{version};

        $rsxml{elem_attribs}{INSTANCE_NUMBER} = $rowhash{instanceNumber} if defined $rowhash{instanceNumber};
        $rsxml{elem_attribs}{INSTANCE_NAME} = $rowhash{instanceName} if $rowhash{instanceName};
        $rsxml{elem_attribs}{INSTANCE_ID} = $rowhash{instanceId} if defined $rowhash{instanceId};
        $rsxml{elem_attribs}{DISK_DISCOVERY_STRING} = $rowhash{diskDiscoveryString} if defined $rowhash{diskDiscoveryString};
        $rsxml{elem_attribs}{DISKGROUP_NAME} = $rowhash{diskGroupName} if defined $rowhash{diskGroupName};
        $rsxml{elem_attribs}{DISKGROUP_NUMBER} = $rowhash{diskGroupNumber} if defined $rowhash{diskGroupNumber};
        $rsxml{elem_attribs}{DISKGROUP_SIZE} = $rowhash{diskGroupSize} if defined $rowhash{diskGroupSize};
        $rsxml{elem_attribs}{REDUNDANCY} = $rowhash{redundancy} if defined $rowhash{redundancy};
        $rsxml{elem_attribs}{COMPATIBILITY} = $rowhash{compatibility} if defined $rowhash{compatibility};
        $rsxml{elem_attribs}{STATE} = $rowhash{state} if defined $rowhash{state};
        $rsxml{elem_attribs}{DISK_NUMBER} = $rowhash{diskNumber} if defined $rowhash{diskNumber};
        $rsxml{elem_attribs}{DISK_PATH} = $rowhash{diskPath} if defined $rowhash{diskPath};
        $rsxml{elem_attribs}{DISK_NAME} = $rowhash{diskName} if defined $rowhash{diskName};
        $rsxml{elem_attribs}{MODE_STATUS} = $rowhash{modeStatus} if defined $rowhash{modeStatus};
        $rsxml{elem_attribs}{DATABASE_NAME} = $rowhash{databaseName} if defined $rowhash{databaseName};
       
        $rsxml{elem_attribs}{CLUSTER_NAME} = $clusterName;
        $rsxml{elem_attribs}{RESOURCE_NAME} = $resourceName;
        $rsxml{elem_attribs}{ROLE} = $ENV{EM_TARGET_ROLE} if $ENV{EM_TARGET_ROLE};
        $rsxml{elem_attribs}{USERNAME} = $credentials{username} if $credentials{username};
        $rsxml{elem_attribs}{SID} = $ENV{EM_TARGET_SID} if $ENV{EM_TARGET_SID};
        $rsxml{elem_attribs}{PORT} = $ENV{EM_TARGET_PORT} if $ENV{EM_TARGET_PORT};
        $rsxml{elem_attribs}{MACHINE_NAME} = $ENV{EM_TARGET_MACHINE_NAME} if $ENV{EM_TARGET_MACHINE_NAME};
        $rsxml{elem_attribs}{ORACLE_HOME} = $ENV{EM_TARGET_ORACLE_HOME} if $ENV{EM_TARGET_ORACLE_HOME};
 
        if ( $rsxml{elem_attribs}{IS_CLUSTER}=~/TRUE/ )
        {
          if ( $clusterName )
          {
            $name1 = $clusterName;
          }
          elsif ( $dbUniqueName  )
          {
            if ( $dbUniqueName =~ /$targetName/ )
            {
              $name1 = $targetName;
            }
            else
            {
              $name1 = $targetName.'_'.$dbUniqueName;
            }
          }
          else 
          {
            $name1 = $targetName;
          }
     
        }
        else
        {
          if ( $hostName )
          {
            $name1 = $hostName;
          }
          else
          {
            $name1 = $machineName;
          }
        }
     
        if ( $dbUniqueName )
        {
          $name2 = $dbUniqueName;
        }
        else
        {
          $name2 = $targetName;
        }
     
        $rsxml{elem_attribs}{NAME}=$name1.'_'.$name2;
     
        has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

      }

    }

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}




# name : has_rac_cmdb_metric
# desc : instrument the OracleClusterDatabase cmdb metric 
#
# arg  :
#  metric_name
#  target_name
#  oracle home
#  oracle type 
#  machine name
#  oracle name
#
# return:
#
sub has_rac_cmdb_metric($$$$;$$)
{

  my ($metric_name,$targetName,$oracleHome,$oracleType,$machineName,$oracleName) = @_;

  my %rshash;
  my %stdinArgs;
  my $password ;
  my $username ;
  my $address;
  my $role;
  my $dbOracleHome;

  # Get the username,password and target database base address from
  # fetchlet environment
  %stdinArgs = has::Common::get_stdinvars();
   
   # pupulate the credentials array
   my %credentials;
   $credentials{username} = $stdinArgs{EM_TARGET_USERNAME} if $stdinArgs{EM_TARGET_USERNAME};
   $credentials{password} = $stdinArgs{EM_TARGET_PASSWORD} if $stdinArgs{EM_TARGET_PASSWORD};
   $credentials{address} = $ENV{EM_TARGET_ADDRESS} if $ENV{EM_TARGET_ADDRESS};
   $credentials{role} = $ENV{EM_TARGET_ROLE} if $ENV{EM_TARGET_ROLE};
   $credentials{ORACLE_HOME} = $ENV{EM_TARGET_ORACLE_HOME} if $ENV{EM_TARGET_ORACLE_HOME};

   my %sql;
   $sql{rac_metric} = '
          SELECT NVL(dun.DBUniqueName,dn.DBName) name,
                 d.DBDomain dbDomain,
                 dn.DBName dbName,
                 DECODE(NVL(ROUND(SUBSTR(v.DBVersion,1,INSTRB(v.DBVersion,\'.\')-1)/10),-1),-1,NULL,0,NULL,dun.DBUniqueName) db_unique_name,
                 v.DBVersion version,
                 NVL(SUBSTR(v.DBVersion,1,DECODE(INSTRB(v.DBVersion,\'.\',1,2),0,LENGTH(v.DBVersion),INSTRB(v.DBVersion,\'.\',1,2)-1)),0) shortVersion,
                 o.openMode openMode,
                 s.serviceName serviceName
           FROM  
           (
                 SELECT d.open_mode openMode,
                        1 cnt
                   FROM v$database d
           ) o,
           (
                 SELECT i.version DBVersion,
                        1 cnt
                   FROM v$instance i
           ) v,
           (
                 SELECT p.value DBDomain ,
                        1 cnt
                   FROM v$parameter p
                  WHERE p.name=\'db_domain\'
           ) d,
           (
                 SELECT p.value DBName ,
                        1 cnt
                   FROM v$parameter p
                  WHERE p.name=\'db_name\'
           ) dn,
           (
                 SELECT p.value DBUniqueName ,
                        1 cnt
                   FROM v$parameter p
                  WHERE p.name=\'db_unique_name\'
           ) dun,
           (
                 SELECT p.value serviceName ,
                        1 cnt
                   FROM v$parameter p
                  WHERE p.name=\'service_names\'
           ) s
           WHERE ROWNUM = 1
             AND v.cnt = d.cnt(+)
             AND v.cnt = dn.cnt(+)
             AND v.cnt = dun.cnt(+)
             AND v.cnt = o.cnt(+)
             AND v.cnt = s.cnt(+)
     ';

    $sql{database_metric} =  $sql{rac_metric};

  my $sqlexecuted;
  if ( not ref($sql{$metric_name})  )
  {
       $credentials{sql} = [ ( $sql{$metric_name} ) ];
       $sqlexecuted = $sql{$metric_name};
  }
  elsif ( $ENV{EM_VERSION_CATEGORY} and  $sql{$metric_name}{$ENV{EM_VERSION_CATEGORY}} )
  {
       $credentials{sql} = [ ( $sql{$metric_name}{$ENV{EM_VERSION_CATEGORY}} ) ];
       $sqlexecuted = $sql{$metric_name}{$ENV{EM_VERSION_CATEGORY}};
  }


  # order of fields in the results
  my %fieldOrder;
  $fieldOrder{$sql{rac_metric}} = 
                      {name=>1 ,
                       dbDomain =>2,
                       dbName =>3,
                       db_unique_name =>4,
                       version => 5,
                       shortVersion => 6,
                       openMode => 7,
                       serviceName => 8 };

   my $results_ref = has::Common::hasGetSQLResults(\%credentials,\%fieldOrder);

   my $array_ref;

   my $clusterName = has::Common::hasGetClusterName();

   my $resourceName = has::Common::hasGetResourceNameForOracleApp($oracleHome,$oracleType,$oracleName);

   $array_ref = $results_ref->{$sqlexecuted} if $results_ref and keys %{$results_ref} and $results_ref->{$sqlexecuted};

   my $name;
   my $type;
   my $guid;


   $name = $ENV{EM_TARGET_NAME} if  $ENV{EM_TARGET_NAME};
   $type = $ENV{EM_TARGET_TYPE} if  $ENV{EM_TARGET_TYPE};
   $guid = md5_hex($name, $type) if $name and $type;

   if ( $array_ref and ref($array_ref) and ref($array_ref) =~ /LIST|ARRAY/ and @{$array_ref} )
   {
      # parse query results to check if db has has
      for my $row ( @{$array_ref} )
      {
        my %rsxml;
        my %rowhash = %{$row};

        $rsxml{elem_attribs}{VENDOR} = 'Oracle';

        $rsxml{elem_attribs}{NAME} = $rowhash{name} if $rowhash{name};
        $rsxml{elem_attribs}{NAME} = $targetName unless $rsxml{elem_attribs}{NAME};

        $rsxml{elem_attribs}{DB_DOMAIN} = $rowhash{dbDomain} if $rowhash{dbDomain};
        $rsxml{elem_attribs}{DB_DOMAIN} = has::Common::hasGetDomainName() if not $rsxml{elem_attribs}{DB_DOMAIN};
        $rsxml{elem_attribs}{DB_DOMAIN} = $ENV{EM_MACHINE_NAME} if $ENV{EM_MACHINE_NAME} and not $rsxml{elem_attribs}{DB_DOMAIN};
        $rsxml{elem_attribs}{DB_DOMAIN} = has::Common::hasGetLocalHostName() if not $rsxml{elem_attribs}{DB_DOMAIN};

        $rsxml{elem_attribs}{DB_NAME} = $rowhash{dbName} if $rowhash{dbName};
        $rsxml{elem_attribs}{DB_UNIQUE_NAME} = $rowhash{db_unique_name} if $rowhash{db_unique_name};

        $rsxml{elem_attribs}{TYPE} = $ENV{EM_TARGET_TYPE} if $ENV{EM_TARGET_TYPE};
        $rsxml{elem_attribs}{SHORT_VERSION} = $rowhash{shortVersion} if $rowhash{shortVersion};
        $rsxml{elem_attribs}{VERSION} = $rowhash{version} if $rowhash{version};
        $rsxml{elem_attribs}{VERSION_CATEGORY} =  $ENV{EM_VERSION_CATEGORY} if $ENV{EM_VERSION_CATEGORY};
        $rsxml{elem_attribs}{OPEN_MODE} = $rowhash{openMode} if $rowhash{openMode};
        $rsxml{elem_attribs}{SP_FILE} = '';
        $rsxml{elem_attribs}{CLUSTER_NAME} = $ENV{EM_CLUSTER_NAME} if $ENV{EM_CLUSTER_NAME};
        $rsxml{elem_attribs}{CLUSTER_NAME} = $clusterName;
        $rsxml{elem_attribs}{RESOURCE_NAME} = $resourceName;
        $rsxml{elem_attribs}{SERVICE_NAME} = $ENV{EM_SERVICE_NAME} if $ENV{EM_SERVICE_NAME};
        $rsxml{elem_attribs}{SERVICE_NAME} = $rowhash{serviceName} if $rowhash{serviceName} and not $ENV{EM_SERVICE_NAME};
        $rsxml{elem_attribs}{ROLE} = $ENV{EM_TARGET_ROLE} if $ENV{EM_TARGET_ROLE};
        $rsxml{elem_attribs}{USER_NAME} = $credentials{username} if $credentials{username};
        $rsxml{elem_attribs}{ORACLE_HOME} = $ENV{EM_ORACLE_HOME} if $ENV{EM_ORACLE_HOME};
	$rsxml{elem_attribs}{MASTER} = $ENV{EM_MACHINE_NAME} if $ENV{EM_MACHINE_NAME};
        $rsxml{elem_attribs}{MASTER} = has::Common::hasGetLocalHostName() if not $ENV{EM_TARGET_MACHINE_NAME};
        $rsxml{elem_attribs}{GUID} = $guid if $guid;

        has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

      }

    }

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}

# name : has_localnodeinfo_metric
# desc : instrument nodemame,clustername, node list as metrics for the local host 
#
# arg  :
#  metric_name
#  args to the os command, # CRSHome
#
# return:
#
sub has_localnodeinfo_metric($;$)
{

  my ($metric_name,$crsHome ) = @_;

  my %rshash;
  my %rsxml;
  my $localnodename;

  $rsxml{elem_attribs}{HOST_NAME} = has::Common::hasGetLocalHostName();
  $rsxml{elem_attribs}{HOST_NAME} = $has::Common::has_em_targetname if $has::Common::has_em_targetname and not $rsxml{elem_attribs}{HOST_NAME};

  $rsxml{elem_attribs}{NODE_NAME} =  has::Common::hasGetNodeName($crsHome);
  $localnodename = $rsxml{elem_attribs}{NODE_NAME} if $rsxml{elem_attribs}{NODE_NAME};

  $rsxml{elem_attribs}{CLUSTER_NAME} = has::Common::hasGetClusterName($crsHome);

  $rsxml{elem_attribs}{CRS_HOME} = $crsHome if $crsHome;
  $rsxml{elem_attribs}{CRS_HOME} = has::Common::hasGetNodeCRSHome($localnodename,$crsHome) 
   unless $rsxml{elem_attribs}{CRS_HOME};

  $rsxml{elem_attribs}{NODE_LIST} = has::Common::hasGetNodeList($crsHome);
  $rsxml{elem_attribs}{VERSION} = has::Common::hasGetClusterVersion($crsHome);

  has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}



# name : has_votingdisk_metric
# desc : instrument the voting disk metric
#
# arg  :
#  metric_name
#  args to the os command, # CRSHome
#
# return:
#
sub has_votingdisk_metric($;$)
{

  my ($metric_name,$crsHome ) = @_;

  my $results;
  my $vdref;
  my $clusterName;

  $vdref = has::Common::hasGetVotingDiskInformation($crsHome);
  $clusterName = has::Common::hasGetClusterName($crsHome);

  my %rshash;

  for my $location ( keys %{$vdref} )
  {
    my %rsxml;

    $rsxml{elem_attribs}{CLUSTER_NAME} = $clusterName;
    $rsxml{elem_attribs}{LOCATION} = $location;

    has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  }

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}


# name : has_vip_metric
# desc : instrument vip metric for each node
#
# arg  :
#  metric_name
#  args to the os command, # CRSHome
#
# return:
#
sub has_vip_metric($;$)
{

  my ($metric_name,$crsHome ) = @_;

  my $results;
  my $vipref;
  my $clusterName;

 if ( not has::Common::hasCheckForEmcrsp($crsHome) )
 {
  if ( $crsHome )
  {
    warn "WARN:has_metrics.pl:has_vip_metric:This is a pre 11gR2 cluster, binary emcrsp is not found at $crsHome";
  }
  else
  {
    warn "WARN:has_metrics.pl:has_vip_metric:This is a pre 11gR2 cluster, binary emcrsp is not found";
  }
   return;
 }

  $vipref = has::Common::hasGetVIPInformation($crsHome);
  $clusterName = has::Common::hasGetClusterName($crsHome);

  my %rshash;

  for my $id ( keys %{$vipref} )
  {
    my %rsxml;
    my $ip;


    $rsxml{elem_attribs}{CLUSTER_NAME} = $clusterName;
    $rsxml{elem_attribs}{NAME}         = $vipref->{$id}{NAME} if $vipref->{$id}{NAME};
    $rsxml{elem_attribs}{NODE_NAME}    = $vipref->{$id}{NODE_NAME} if $vipref->{$id}{NODE_NAME};
    $rsxml{elem_attribs}{IP}           = $vipref->{$id}{VIP_IP} if $vipref->{$id}{VIP_IP};
    $rsxml{elem_attribs}{RESOURCE_NAME}= $vipref->{$id}{NAME} if $vipref->{$id}{NAME};

    has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  }

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}


# name : has_ocr_metric
# desc : instrument the ocr metric
#
# arg  :
#  metric_name
#  args to the os command, # CRSHome
#
# return:
#
sub has_ocr_metric($;$)
{

  my ($metric_name,$crsHome ) = @_;

  my $results;
  my $ocref;
  my $clusterName;

  $ocref = has::Common::hasGetOCRInformation($crsHome);
  $clusterName = has::Common::hasGetClusterName($crsHome);

  my %rshash;

  for my $location ( keys %{$ocref} )
  {
    my %rsxml;

    $rsxml{elem_attribs}{CLUSTER_NAME} = $clusterName;
    $rsxml{elem_attribs}{LOCATION} = $location;
    $rsxml{elem_attribs}{MIRRORS} = $ocref->{$location}{MIRRORS} 
     if defined $ocref->{$location}{MIRRORS};

    has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  }

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}


# name : has_eons_metric
# desc : instrument eons metric
#
# arg  :
#  metric_name
#  args to the os command, # CRSHome
#
# return:
#
sub has_eons_metric($;$)
{

  my ($metric_name,$crsHome ) = @_;

  my $results;
  my $eonsref;
  my $clusterName;

 if ( not has::Common::hasCheckForEmcrsp($crsHome) )
 {
  if ( $crsHome )
  {
    warn "WARN:has_metrics.pl:has_eons_metric:This is a pre 11gR2 cluster, binary emcrsp is not found at $crsHome";
  }
  else
  {
    warn "WARN:has_metrics.pl:has_eons_metric:This is a pre 11gR2 cluster, binary emcrsp is not found";
  }
   return;
 }

  $eonsref = has::Common::hasGetEONSInformation($crsHome);
  $clusterName = has::Common::hasGetClusterName($crsHome);

  my %rshash;

  for my $id ( keys %{$eonsref} )
  {
    my %rsxml;
    my $eonsPort;
    my $eonsName;

    $eonsPort = $eonsref->{$id}{EONS_PORT} if $eonsref->{$id}{EONS_PORT};
    $eonsPort = 0 unless $eonsPort;

    $eonsName = $eonsref->{$id}{NAME} if $eonsref->{$id}{NAME};
    $eonsName = 'UNKNOWN' unless $eonsName;

    $rsxml{elem_attribs}{CLUSTER_NAME} = $clusterName;
    $rsxml{elem_attribs}{NAME} = $eonsName;
    $rsxml{elem_attribs}{NODE_NAME} = $eonsref->{$id}{NODE_NAME} if $eonsref->{$id}{NODE_NAME};
    $rsxml{elem_attribs}{EONS_PORT} = $eonsPort if $eonsPort;
    $rsxml{elem_attribs}{RESOURCE_NAME} = $eonsName;

    has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  }

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}


# name : has_dynamic_property
# desc : instrument dynamic properties for _ delimted list
#
# arg  :
#  metric_name
#  list of dynamic properties __ delimted
#  # CRSHome
#
# return:
#
sub has_dynamic_property($$;$)
{

  my ($metric_name,$prop_list,$crsHome,$arg1 ) = @_;

  my @props;
  my %rshash;
  my %rsxml;
  my $eonsref;
  my $i = 0;
  my $dynamic=0;

  $dynamic = 1 if $metric_name =~ /^dynamic_property$/;

  @props = split/__/,$prop_list;

  for my $prop ( @props )
  {
    $prop =~ s/^\s+|\s+$// if $prop;

    next unless $prop;

    # build the dynamic list of properties along with their order
    # this is used by has::Common::has_em_results
    $i++;
    $has::Common::has_metric_config{basic_fields}{$metric_name}{$i}=$prop;
 
    if  ( $prop =~ /^eonsport$/i )
    {
        $eonsref = has::Common::hasGetEONSInformation($crsHome) unless $eonsref and keys %{$eonsref};
      
        for my $id ( keys %{$eonsref} )
        {
          $rsxml{elem_attribs}{$prop} = $eonsref->{$id}{EONS_PORT} if $prop =~ /eonsport/i and $eonsref->{$id}{EONS_PORT};
          
          last if $rsxml{elem_attribs} and $rsxml{elem_attribs}{$prop};
      
        }

      }
      elsif ( $prop =~ /^isscan$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasIsListenerScan($crsHome,$arg1);
         $rsxml{elem_attribs}{$prop} = '0' unless $rsxml{elem_attribs}{$prop};
      }
      elsif ( $prop =~ /^scanname$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetScanName($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^scanport$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetScanPort($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^clustername$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetClusterName($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^hasname$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetHasName($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^vendor$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetClusterVendor($crsHome,$dynamic);
      }
      elsif ( $prop =~ /isvendorcw/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetIsVendorCluster($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^version$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetClusterSoftwareVersion('',$crsHome,$dynamic);
      }
      elsif ( $prop =~ /^activeversion$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetClusterActiveVersion($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^has_releaseversion$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetHasReleaseVersion($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^ocrtype$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetOcrType($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^hostname$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetLocalHostName();
      }
      elsif ( $prop =~ /^crshome$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetCRSHome($crsHome);
      }
      elsif ( $prop =~ /^nodename$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetNodeName($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^has_nodename$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetHasNodeName($crsHome,$dynamic);
      }
      elsif ( $prop =~ /^nodelist$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetNodeList($crsHome);
      }
      elsif ( $prop =~ /^nodestatus$/i )
      {
         $rsxml{elem_attribs}{$prop} = has::Common::hasGetNodeStatus($crsHome,$arg1);
          my $ocrtype = has::Common::hasGetOcrType($crsHome,$dynamic);
          if ( $ocrtype =~ /has/ )
          {
            $rsxml{elem_attribs}{$prop}  = 'has';
          }
      }
      elsif ( $prop =~ /^isemcrsp$/i )
      {
         my $isemcrsp = has::Common::hasCheckForEmcrsp($crsHome);
         $rsxml{elem_attribs}{$prop} = $isemcrsp if $isemcrsp;
         $rsxml{elem_attribs}{$prop} = 0 unless $isemcrsp;
      }
      else
      {
         # this is a case of unknown property
         $rsxml{elem_attribs}{$prop} = undef;
         die "has_metrics.pl:has_dynamic_property:ERROR:Property $prop is not supported\n";
      }

  }

  has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
    last;
  }

  return 1;
 
}


# name : has_get_resource_name_metric
# desc : use srvctl to get resource name for a given database,listener, service,scan, asm, eons
#
# arg  :
#  metric name
#  oracleHome of the resource
#  type of resource (database,listener,service,scanlistener,asm,eons)
#  oracle name of the reesource
#
# return:
#
sub has_get_resource_name_metric($$$;$$)
{

  my ($metric_name,$oracleHome,$oracleType,$oracleName,$oracleName2 ) = @_;

  my %rshash;
  my %rsxml;
  my $localnodename;

  # if database get db unique name before calling srvctl 
  if ( $oracleType =~ /database|oracle_database|rac_database/ )
  {

   my %stdinArgs;

   # Get the username,password and target database base address from
   # fetchlet environment
   %stdinArgs = has::Common::get_stdinvars();
   
   # pupulate the credentials array
   my %credentials;
   $credentials{username} = $stdinArgs{EM_TARGET_USERNAME} if $stdinArgs{EM_TARGET_USERNAME};
   $credentials{password} = $stdinArgs{EM_TARGET_PASSWORD} if $stdinArgs{EM_TARGET_PASSWORD};
   $credentials{address} = $ENV{EM_TARGET_ADDRESS} if $ENV{EM_TARGET_ADDRESS};
   $credentials{role} = $ENV{EM_TARGET_ROLE} if $ENV{EM_TARGET_ROLE};
   $credentials{ORACLE_HOME} = $ENV{EM_TARGET_ORACLE_HOME} if $ENV{EM_TARGET_ORACLE_HOME};

   my $sql;
   $sql = "
           SELECT database_name \"db_name\"
           FROM
           (
             SELECT name,
                    value database_name 
             FROM   v\$parameter
             WHERE (name = \'db_unique_name\'
                    OR name = \'db_name\')
              AND value IS NOT NULL
              ORDER BY name DESC
            )
            WHERE ROWNUM = 1";

   $credentials{sql} = [ ( $sql ) ];

   # order of fields in the results
   my %fieldOrder;
   $fieldOrder{$sql} = 
                      {
                       db_name =>1
                       };

   my $results_ref = has::Common::hasGetSQLResults(\%credentials,\%fieldOrder);

   my $array_ref = $results_ref->{$sql} if $results_ref and keys %{$results_ref} and $results_ref->{$sql};

   if ( $array_ref and ref($array_ref) and ref($array_ref) =~ /LIST|ARRAY/ and @{$array_ref} )
   {
      # parse query results to check if db has has
      for my $row ( @{$array_ref} )
      {
        my %rowhash = %{$row};

        $oracleName = $rowhash{db_name} if $rowhash{db_name};

      }
    }
  }
  elsif ( $oracleType =~ /listener|oracle_listener/ )
  {
     # swap the position of port and name ( port is passed first, but here port is 2nd arg)
     my $temp;
     $temp = $oracleName2 if $oracleName2;
     undef $oracleName2 if $oracleName2;
     $oracleName2 = $oracleName if $oracleName;
     undef $oracleName if $oracleName;
     $oracleName = $temp if $temp;
  }


  if ( $oracleType =~ /listener|oracle_listener/ )
  {
    ( $rsxml{elem_attribs}{RESOURCE_NAME} , $rsxml{elem_attribs}{IS_SCAN} ) = 
        has::Common::hasGetResourceNameForOracleApp($oracleHome,$oracleType,$oracleName,$oracleName2);
  }
  else
  {
   $rsxml{elem_attribs}{RESOURCE_NAME} = 
      has::Common::hasGetResourceNameForOracleApp($oracleHome,$oracleType,$oracleName,$oracleName2);
  }

  $rsxml{elem_attribs}{RESOURCE_NAME} = 'UNKNOWN' unless $rsxml{elem_attribs}{RESOURCE_NAME};

  has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}


# name : has_build_metric_from_env
# desc : build a one row em_result result set from env
#
# arg  :
#  metric name
#  oracleHome of the resource
#
# return:
#
sub has_build_metric_from_env($;$)
{

  my ($metric_name,$oracleHome) = @_;

  my %rshash;
  my %rsxml;
  my $oracleType;
  my $oracleName;
  my $crsHome;

  warn "ERROR:has_metrics.pl:has_build_metric_from_env:metric $metric_name is not supported \n"
   and return unless $has::Common::has_metric_config{basic_fields}{$metric_name};

  if ( $metric_name =~ /listener_cmdb_metric|scan_listener_cmdb_metric/ )
  {
    $oracleType = 'listener';
    $oracleName = $ENV{LISTENER_NAME} if $ENV{LISTENER_NAME};
  }

  for my $ord ( keys %{$has::Common::has_metric_config{basic_fields}{$metric_name}} )
  {
    my $fld = $has::Common::has_metric_config{basic_fields}{$metric_name}{$ord};

    if ( defined $ENV{"$fld"} )
    {
      $rsxml{elem_attribs}{$fld} = $ENV{"$fld"};
      next;
    }
 
    if ( $fld =~ /^(HOST_NAME|MACHINE)$/ )
    {
      if ( $fld =~ /^HOST_NAME/ )
      {
        $rsxml{elem_attribs}{$fld} = $ENV{EM_HOST_NAME} if $ENV{EM_HOST_NAME};
      }
      elsif ( $fld =~ /^MACHINE/ )
      {
        $rsxml{elem_attribs}{$fld} = $ENV{EM_MACHINE_NAME} if $ENV{EM_MACHINE_NAME};
      }

      if ( not $rsxml{elem_attribs}{$fld} )
      {
        $rsxml{elem_attribs}{$fld} = has::Common::hasGetLocalHostName();
      }

    }
    elsif ( $fld =~ /^ORACLE_HOME$/ )
    {
      $rsxml{elem_attribs}{$fld} = $oracleHome if defined $oracleHome;
    }
    elsif ( $fld =~ /^RESOURCE_NAME$/ )
    {
      $rsxml{elem_attribs}{$fld} = has::Common::hasGetResourceNameForOracleApp($oracleHome,$oracleType,$oracleName);
      $rsxml{elem_attribs}{$fld} = 'UNKNOWN' unless $rsxml{elem_attribs}{$fld};
    }
    elsif ( $fld =~ /^CLUSTER_NAME$/ )
    {
      $rsxml{elem_attribs}{$fld} = has::Common::hasGetClusterName($crsHome);
    }
    elsif ( $fld =~ /^SCAN_NAME$/ )
    {
      $rsxml{elem_attribs}{$fld} = has::Common::hasListenerGetScanName($oracleHome);
    }

    warn "WARN:has_metrics.pl:has_build_metric_from_env:Failed to instrument field $fld for metric $metric_name \n"
     unless defined $rsxml{elem_attribs}{$fld};

  }

  has::Common::has_em_results(\%rsxml,\%rshash,$metric_name);

  for my $row ( @{$rshash{$metric_name}} )
  {
    print "$row\n";
  }

  return 1;
 
}


# name : has_event_resource_instance_metric
# desc : instrument the metrics for resource instances for generating
#          alerts if resource instance is down
#
# arg  :
#  metric_name
#  args to the os command, # OacleHome or db
#  args to the os command, # resource type to be filtered
#  args to the os command, # type of filter (YES,NO)
#
# return:
#
sub has_event_resource_instance_metric($;@)
{
  my ( $metric_name,$crsHome,@args) = @_; 

  unshift @args,'TYPE';

  return has_instrument_metrics($metric_name,$crsHome,\@args);
}


# name : hasGetDBHasManaged
# desc : return FALSE is the database is not managed by the 
#         single instance HAS
#
# arg  :
#  metric_name
#  args to the os command, # OacleHome or db
#
# return:
#
sub hasGetDBHasManaged($;$)
{

  my ($metric_name,$oh,$type,$sid) = @_;

  my %stdinArgs;
  my $password ;
  my $username ;
  my $dbOcrType;
  my $crsHome;
  my $address;
  my $role;
  my $dbOracleHome;

  # Get the username,password and target database base address from
  # fetchlet environment
  %stdinArgs = has::Common::get_stdinvars();
   
  $password = $stdinArgs{EM_TARGET_PASSWORD} if $stdinArgs{EM_TARGET_PASSWORD};
  $username = $stdinArgs{EM_TARGET_USERNAME} if $stdinArgs{EM_TARGET_USERNAME};

  $address = $ENV{EM_TARGET_ADDRESS} if $ENV{EM_TARGET_ADDRESS};
  $role = $ENV{EM_TARGET_ROLE} if $ENV{EM_TARGET_ROLE};
  $dbOracleHome = $ENV{EM_TARGET_ORACLE_HOME} if $ENV{EM_TARGET_ORACLE_HOME};
  $dbOracleHome = $oh if $oh and not $dbOracleHome;

  ($dbOcrType,$crsHome) = has::Common::hasIsDBManagedByHas($username,$password,$address,$role,$dbOracleHome,$sid); 
  
  $dbOcrType = 'N/A' unless $dbOcrType;
  $crsHome = 'N/A' unless $crsHome;

  print "em_result=$dbOcrType|$crsHome\n";

  return 1;
 
}

# name : hasGetLsnrHasManaged
# desc : return FALSE is the lsnr is not managed by the 
#         single instance HAS
#
# arg  :
#  metric_name
#  args to the os command, # OacleHome or db
#
# return:
#
sub hasGetLsnrHasManaged($;$$)
{
  my ($metric_name,$oh,$type,$lsnrPort,$name) = @_;

  warn("DEBUG:has_metrics:hasGetLsnrHasManaged: lsnrport : $lsnrPort");
  my ($lsnrOcrType,$crsHome) = has::Common::hasIsLsnrManagedByHas($oh,$lsnrPort,$name); 
  
  $lsnrOcrType = 'N/A' unless $lsnrOcrType;
  $crsHome = 'N/A' unless $crsHome;

  warn("DEBUG:has_metrics:hasGetLsnrHasManaged: result: $lsnrOcrType");
  print "em_result=$lsnrOcrType|$crsHome\n";

  return 1;
}


# name : hasGetHasStatusMetric
# desc : get status of crs in a siha case
#
# arg  :
#  metric_name
#  args to the os command, crsHome 
#
# return:
#
sub hasGetHasStatusMetric(;$$)
{

  my ($metric_name,$crsHome) = @_;

  my $statusref;
  my $status = 2; # warning=2, 0=down, 1=up
  my $cmdoutput = '';
  my $nonodelist;


  $statusref = has::Common::hasGetClusterStatus($nonodelist,$crsHome,'ha');
 
  warn "WARN:has_metrics.pl:hasGetHasStatusMetric Failed to get Status for Single Instance HA" unless $statusref and keys %{$statusref}; 
 
  if ( $statusref and defined $statusref->{crsIsUp} )
  {
     $status = $statusref->{crsIsUp};
  }
  
  if ( $statusref and defined $statusref->{o} )
  {
     $cmdoutput = $statusref->{o};
  }

  print "em_result=$status|$cmdoutput\n";

  return 1;
 
}



# name : hasGetCRSStatusMetric
# desc : get status of crs for a cluster or local node case
#
# arg  :
#  metric_name
#  args to the os command, crsHome 
#  node or node list seperated by ,
#  crs version category
#
# return:
#
sub hasGetCRSStatusMetric(;$$$$)
{

  my ($metric_name,$crsHome,$crsVersionCategory,$nodelist) = @_;

  my $statusref;

  # put in the crsversion category check here 
  if ( $crsVersionCategory and $crsVersionCategory =~ /10gR1|pre10g|DefaultRange/i )
  {
    warn("WARN:has_metrics:hasGetCRSStatusMetric: Cluster status check metric not supported for this cluster");
    print "em_result=1|0|\n";
    return 1;
  }

  if ( not $nodelist )
  {
    # to ensure that node list is first populated by pre 11g
    my $confref = has::Common::hasGetClusterNodeListPre11g($crsHome);

    $nodelist = has::Common::hasGetNodeList($crsHome);
  }


  # for the list of nodes execute cluvfy
  if ( $nodelist )
  {
  
    $statusref = has::Common::hasGetClusterStatus($nodelist,$crsHome);
  
    #if crs is down on all nodes and it might be an  
    # user equivalence an issue check for local node only to get crs status
    if  ( $statusref and ref($statusref) =~ /HASH/ and keys %{$statusref} 
           and defined $statusref->{crsIsUp} and $statusref->{crsIsUp} == 1 )
    {
       #do nothing
       warn("DEBUG:has_metrics:hasGetCRSStatusMetric: Cluster status check successful for $nodelist Cluster is Up");
    }
    else
    {
       my %saveres = %$statusref;
       my $nonodelist;
  
       $statusref = has::Common::hasGetClusterStatus($nonodelist,$crsHome);
       
       # if cluvfy is successful for the local host
       #  modify the fail count and mark clusterware up 
       if ( $statusref->{crsIsUp} == 1 )
       {
          $saveres{failedCount} = $saveres{failedCount} - 1 if $saveres{failedCount};
          $saveres{successCount} = $saveres{successCount} + 1;
          $saveres{crsIsUp} = $statusref->{crsIsUp};
       }
  
       $statusref = \%saveres;
    }
  }
  else
  {
    # if olsnodes returns null then return an warning and 
    #mark cluster status as down
    warn("WARN:has_metrics.pl:hasGetCRSStatusMetric: Failed to get the list of nodes for cluster");
  
    $statusref->{crsIsUp} = 2;
    $statusref->{failedCount} = 0;
    $statusref->{o} = "has_metrics.pl::hasGetCRSStatusMetric Failed to get cluster node list";
  }
  
  $statusref->{o} =~ s/\n/ /g if $statusref and $statusref->{o};
  
  print "em_result=$statusref->{crsIsUp}|$statusref->{failedCount}|$statusref->{o}\n";

  return 1;
 
}


# name : has_instrument_metrics
# desc : fn to instrument metrics for the metric_name specified and cache them 
#        to file
#
# arg  :
#  metric_name
#  crshome
#  args to the os command
#
# return:
#
sub has_instrument_metrics($;@)
{
  my ($metric_name,$crsHome,$args) = @_;
  my $parent_metric_name;

  # check if this is a 11gr2 version cluster
  # the order of search directories for olsnode/lsnodes
  my %olsnodeorder;
  my $nCrsHome;

  $olsnodeorder{0}{path}=$crsHome if $crsHome;

  if ( $crsHome )
  {
    undef $crsHome unless has::Common::hasIsReadable($crsHome);

    if ( $crsHome )
    {
      undef $crsHome unless has::Common::hasCheckForEmcrsp($crsHome);
    }
  }


  if ( not $crsHome )
  {

    $olsnodeorder{1}{path}='CRS_HOME';
    $olsnodeorder{2}{path}='EM_CRS_HOME';
    $olsnodeorder{3}{path}='ORACLE_HOME';
    $olsnodeorder{4}{path}='EMDROOT';

    for my $order ( sort  {$a <=> $b} keys %olsnodeorder )
    {
      my $path;
  
      next unless $olsnodeorder{$order} and $olsnodeorder{$order}{path};
  
      $path = $olsnodeorder{$order}{path};
      $path = $ENV{$olsnodeorder{$order}{path}} if $ENV{$olsnodeorder{$order}{path}};
  
      next unless $path;
      next if $path =~ /^(#CRS_HOME#|CRS_HOME|EM_CRS_HOME|ORACLE_HOME|EMDROOT)$/;
  
      next unless has::Common::hasCheckForEmcrsp($path);
     
      $nCrsHome = $path and last;
  
    }
  
    if ( not $nCrsHome )
    {
      if ( not has::Common::hasCheckForEmcrsp() )
      {
        warn "WARN:has_metrics.pl:This is a pre 11gR2 cluster, binary emcrsp is not found";
        return;
      }
    }
  
    $crsHome = $nCrsHome if $nCrsHome;

  }

  # get the agent state dir to cache the metrics
  $has::Common::has_metric_config{cache_dir} = has::Common::has_get_cache_dir() or
   warn "WARN:Failed to get a cache dir for caching files"
   and return;

  # clean up the cache generated during the last run of this metric
  #  cache is cleaned only for the cache generating parent metric which is 
  #   supposed to be schedule first
  has_rm_cache($metric_name) or
   warn "WARN:Failed to cleanup existing cache files for $metric_name\n"
    and return;


  # if this is an dependent cached metric, cached during the run of this parent
  #  metric, check if cache file exists, else continue instrumenting.
  my $flnm = 
   catfile($has::Common::has_metric_config{cache_dir},$has::Common::has_metric_config{cache_file}{$metric_name});

  return 1
   if $metric_name =~ /^$has::Common::has_metric_config{dependent_cached_metrics}$/i and has::Common::hasIsReadable($flnm);

  # if this is a dependent metric then switch to execute the parent metric
  #  else parent metric is same as dependent metric
  warn "WARN:Failed, parent metric is not defined for dependent metric $metric_name\n"
   and return
    if $metric_name =~ /^$has::Common::has_metric_config{dependent_cached_metrics}$/i
     and not defined $has::Common::has_metric_config{parent_metric}{$metric_name};

  if  ( $metric_name =~ /^$has::Common::has_metric_config{dependent_cached_metrics}$/i )
  {
    $parent_metric_name = $has::Common::has_metric_config{parent_metric}{$metric_name};
  }
  else
  {
    $parent_metric_name = $metric_name;
  }

  warn "WARN:Failed to get the parent metric to execute"
   and return 
    unless $parent_metric_name;

  # instrument the metrics and dependent metrics, cache them to file and return
  has_entities($parent_metric_name,$crsHome,$args) or
   warn "ERROR:Failed to execute read the cached metrics for $metric_name\n"
    and return;

  # read the appropriate cached file and print it to stdout
  return has_rd_cache($metric_name); 

}



# name : hasGetLocalCRSStatusMetric
# desc : get status of crs for local node 
#
# arg  :
#  metric_name
#  args to the os command, crsHome 
#  node or node list seperated by ,
#  crs version category
#
# return:
#
sub hasGetLocalCRSStatusMetric(;$$$)
{
   my ($metric_name,$crsHome,$crsVersionCategory,$localNodeName) = @_;
   
   $localNodeName = has::Common::hasGetNodeName($crsHome) unless $localNodeName;
   $localNodeName = '' unless $localNodeName;

   return hasGetCRSStatusMetric($metric_name,$crsHome,$crsVersionCategory,$localNodeName);
}

#------------------------------------------------------------------------------
# FUNCTION :    has_main
#
# DESC
# start the function
#
# ARGUMENTS
# metric name
# args to function
#------------------------------------------------------------------------------
sub has_main($;\@)
{
  my ($metric_name,$argref) = @_;

  chomp $metric_name if $metric_name;

  $metric_name =~ s/^\s+|\s+$//g if $metric_name;

  die "WARN:A function name is required to be passed to has_main\n" unless $metric_name;

  die "WARN:Function name $metric_name is not declared in has_metrics.pl\n"
   unless ( $has::Common::has_metric_config{function}{$metric_name}
     or $has::Common::has_metric_config{$metric_name}{multi_function} );

  #initalization
  has_init() or die "WARN:Failed to initialize in has_init for $metric_name\n";

  my @args;
  @args = @{$argref} if $argref;
 
  if ( $has::Common::has_metric_config{$metric_name}{multi_function} )
  {
    for my $mn ( @{$has::Common::has_metric_config{$metric_name}{multi_function}} )
    {
      &{$has::Common::has_metric_config{function}{$mn}}($mn,@args) or
       die "ERROR:Failed to execute the instrumentation function for $mn,$metric_name\n";
    }
  }
  else
  {
   # invoke the function
 
    &{$has::Common::has_metric_config{function}{$metric_name}}($metric_name,@args) or
     die "ERROR:Failed to execute the instrumentation function for $metric_name\n";
  }

  # print error/warnings as em_warnings to stdout
  # cmdb collections croak with warnings, filed bug# for it
  if ( not defined $ENV{EM_HAS_NO_PRINT_WARNINGS} )
  {
    has::Common::has_printerrors()  or die "ERROR:Failed to print processing errors/warnings for $metric_name\n";
  }

  # clean up
  has_cleanup();

  return 1;
}


#-----------------------------------------------------------
#  Begin Execut#ion Here
#-----------------------------------------------------------
# Read the metric to be instrumented, the default is data for 
# storage_report_data metric
my $has_metric_name = $ARGV[0] if $ARGV[0];

# remove leading or trailing blanks
$has_metric_name =~ s/\s//g if $has_metric_name;

#$has_metric_name = 'resources' unless $has_metric_name;


# pick the other args
my @args;
if ( @ARGV and @ARGV > 1 )
{
  for my $arraynum ( 1..@ARGV)
  {
    push @args, $ARGV[$arraynum];
  }
}

#-----------------------------------------------------------
#  this block is for debug in dev env it will be delted
#-----------------------------------------------------------
if ( $DEV_DEBUG )
{
  my $filedebug = '/tmp/hasdbg.txt';
  my $tm = localtime;
  open(FHD,'>>',$filedebug) or die "Failed to open debug file $filedebug\n";
  print FHD "Invoked the has metrics at $tm for metric $has_metric_name";
  for my $x ( @args )
  {
     print FHD "$x , " if $x;
  }

  print FHD "Invoked with env\n";
  for my $x ( sort keys %ENV )
  {
     print FHD "$x=" if $x;
     print FHD "$ENV{$x}" if $x and $ENV{$x};
     print FHD "\n";
  }
  print FHD "\n";
}
#-----------------------------------------------------------

#-----------------------------------------------------------
# Regression and capture prepartion
#-----------------------------------------------------------
my $has_run_mode = $ENV{HAS_TEST_MODE} if $ENV{HAS_TEST_MODE};

if ( $has_run_mode and $has_run_mode =~ /CAPTURE|REGRESSION/i )
{
  @has::Common::iface = @ARGV;

  # read the contents of the dat file if regression or capture
  has::Common::readRegressionDatFile();
 
  my $envref = has::Common::hasGetEnv();

  if ( $has_run_mode and $has_run_mode =~ /REGRESSION/i and $envref and ref($envref) and keys %{$envref} )
  {

    for my $k ( keys %{$envref} )
    {
      $ENV{$k}=$envref->{$k};
    }

    # restore HAS_TEST_MODE back to regression if running in regression mode
    $ENV{HAS_TEST_MODE}='REGRESSION';

  }


}

#-----------------------------------------------------------

has_main($has_metric_name,@args) if $has_metric_name;

#-----------------------------------------------------------
if ( $DEV_DEBUG )
{
  close(FHD);
}
#-----------------------------------------------------------

#graceful exit with exit status success
POSIX:_exit(0);

END
{
  $ENV{ORACLE_HOME} = $oldOH if $oldOH;
}
