#
#  $Id: HasCluster.pm /st_emdbsa_11.2/23 2011/07/20 17:31:43 ajdsouza Exp $ 
#
#  $Id: HasCluster.pm /st_emdbsa_11.2/23 2011/07/20 17:31:43 ajdsouza Exp $ 
#
#
# NAME  
#	 HasCluster.pm
#
# DESC 
#	 Common cluster subroutines 
#
#
# FUNCTIONS
# AUTOLOADER
#
# NOTES
#
#
# MODIFIED	(MM/DD/YY)
# ajdsouza       11/01/10 - bug fix 10118817
# ajdsouza       06/21/11 - XbranchMerge
#                           ajdsouza_blr_backport_10392806_11.1.0.1.0 from
#                           st_emdbgc_10.2.0.1.0
# ajdsouza       03/30/10 - set SRVM_TRACE=FALSE is nto defined as TRUE for CVU commands
# ajdsouza       03/19/10 - Bug#9496903 fix scscan fix, use resource name
# ajdsouza       03/11/10 - Bug fix#9466874 - do not invoke hasGetDomainName as var hostDomainSub is not undefed
# ajdsouza       03/04/10 - scanName will have gns/locahhost domain appended to it
# ajdsouza       02/16/10 - lowered any WARN scan/eons/voting disk/has/resource message to DEBUG 
# ajdsouza       02/16/10 - do not discover cluster if crsd is down on a node
# ajdsouza       02/16/10 - bug# 9255116 
# ajdsouza       02/16/10 - pre11gR2 cluster errors should be debug 
# ajdsouza       02/16/10 - emcrsp command during discovery should ignore failure as 
#                            stack could be down
# ajdsouza       02/04/10 - bug9350140 
# ajdsouza       01/13/10 - bug9171474 
# ajdsouza       11/14/09 - change target type to oracle_has from has and back to has
# ajdsouza       11/14/09 - append node name if cluster name is crs
# ajdsouza       11/14/09 - Bug fix#9117595
# ajdsouza       10/30/09 - Fix solaris libskgxn issues in getVendorClusterware
# ajdsouza       10/15/09 - Bug fix#9009918
# ajdsouza       10/12/09 - invoke scanapi with dynamic flag
# ajdsouza       02/02/10 - bug9212119  
# ajdsouza       01/13/10 - bug9171474 
# ajdsouza       10/07/09 - Bug fix#8988446
# ajdsouza       09/01/09 - invoke funrtion in has::Common for discovering cluster and has
#                             targets
# ajdsouza       12/07/08 - Bug fix#7689615
# kramarat       06/09/08 - Siha support for Listener
# ajdsouza       05/13/08 - 
# vgoli          03/24/08 - use JDBC/OCI wrappers
# ajdsouza       01/23/07 - Created 
#
#
package has::HasCluster;
use Exporter;
use strict;
use warnings;
use locale;
use File::Spec::Functions;
use File::Path;
use Data::Dumper;
#this module is not available in emagent 10.2.0.4
my $hostNameSub = 'hostOSD::getHostName';
my $hostDomainSub = 'hostOSD::getDomainName';
eval "use hostOSD; 1 " or undef $hostNameSub and undef $hostDomainSub and warn "DEBUG:hostOSD perl module is not available";
use Sys::Hostname;
use DBI;
use XML::Parser;
use FindBin;
use Config;
use Net::Domain qw(hostdomain);
use has::Common;
use has::sCommon;
require "has/SQL.pl";
require "emd_common.pl";
require "semd_common.pl";
$Data::Dumper::Indent = 2;
$Data::Dumper::Deepcopy = 1;
$Data::Dumper::Purity = 1;
$Data::Dumper::Sortkeys = 1;
our @ISA = qw(Exporter);
our @EXPORT = qw( );
#------------------------------------------------------------------------------
# global package variables 
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
# package level variables
#-----------------------------------------------------------------------------
# variables for xml related parsing
my $has_xref;
my $has_fref;
#-----------------------------------------------------------------------------
# sub routines
#-----------------------------------------------------------------------------
sub hasGetFunctionString($;@);
sub readRegressionDatFile();
sub get_fn_results($);
sub regression_test($$;@);
sub hasInit(;$);
sub hasInitSignalsStderr();
sub hasCleanup(;$);
sub hasRestoreStderr();
sub runsystemcommand( $;$$ );
sub print_warn_message(@);
sub print_error_message(@);
sub exit_fail(@);
sub hasGetXmlErrorStack();
sub get_error_stack();
sub has_handle_error(@);
sub has_printerrors;
sub save_systemcmdoutput();
sub has_start_handler;
sub has_end_handler;
sub has_char_handler;
sub parse_xml($);
sub traverse_xml($$$@);
sub delete_element($);
sub mark_depth($);
sub append_element( $$ );
sub make_element ($;$$);
sub print_xml($$);
sub dump_xml($$);
# Traverse the tree depth first based on the start and tag passed
# and the specified order
sub fntrdf( $$$$\&$;$$$ );
# Traverse the tree breadth first based on the start and tag passed
# and the specified order
sub fntrbf( $$$$\&$; );
sub has_get_cache_dir ();
sub hasDoesFileExist($;);
sub hasIsReadable($;);
sub hasIsWriteable($;);
sub hasIsDir($;);
sub hasReturnFileContents($;$);
sub hasWriteToFile($$$;);
sub hasSetCRSEnv(;$);
sub hasRestoreCRSEnv();
sub hasGetEnv();
sub hasGetNodeCRSHome(;$$);
sub hasGetCRSHome(;$);
sub hasGetEnvCRSHome();
sub hasGetOSLocalHostName();
sub hasGetLocalHostName();
sub hasGetDomainName(;$);
sub hasGetLocalHostDomain(;$);
sub hasGetDomainNameNSLookup($;);
sub hasCheckForEmcrsp($);
sub hasGetEntityInformation($;$$);
sub hasDiscoverCluster(;$);
sub hasCheckForHasClusterInventory(;$);
sub hasGetHasName(;$$);
sub hasIsVendorClusterware($);
sub hasSearchClusterName(;$);
sub appendNodeNameToClusterName($$;);
sub hasDoesCommandExist($$;);
sub hasCemutloClusterName($);
sub hasCemutlsClusterName($);
sub hasGetClusterConfigPre11g(;$$);
sub hasGetHasNodeName(;$$);
sub hasGetHasReleaseVersion(;$$);
sub hasGetClusterNodeListPre11g(;$);
sub hasGetClusterGetNodeNamePre11g(;$);
sub hasGetClusterConfigEmcrsp(;$$);
sub hasGetClusterStatus( $;$$ );
sub getClusterCacheRef();
sub clusterConfigCopyVals($$;);
sub hasGetClusterConfig(;$$);
sub hasEMClusterHasTargetDiscovery($;$);
sub hasGetClusterName(;$$);
sub hasGetNodeList(;$);
sub hasGetNodeStatus(;$$);
sub hasGetNodeName(;$);
sub hasGetClusterVersion(;$$);
sub hasGetClusterVendor(;$$);
sub hasGetClusterActiveVersion(;$$);
sub hasGetIsVendorCluster(;$$);
sub computeOcrType($);
 sub hasEvalOcrType($;$);
sub hasGetOcrType(;$$);
sub hasGetClusterNodes(;$);
sub hasGetGnsInformation(;$);
sub hasGetScanInformation(;$);
sub hasClusterHealthCheck($;$);
sub hasGetScanVIPInformation(;$);
sub hasGetScanName(;$);
sub hasGetScanIP(;$);
sub hasGetScanPort(;$);
sub hasGetEONSInformation(;$);
sub hasGetEONSPort(;$);
sub hasGetVIPInformation(;$);
sub hasGetVotingDiskInformation(;$);
sub hasGetOCRInformation(;$);
sub hasGetDBResourceInformation($);
sub hasGetLsnrResourceInformation($);
sub hasIsDBManagedByHas($$$$;$$$);
sub hasIsLsnrManagedByHas($$;$);
sub hasCheckAndReturnEmcrspResults($);
sub hasadm_resource_types_mark_base_types($);
sub hasGetResourceNameForOracleApp($$;$$);
sub hasListenerGetScanName($;$);
sub hasIsListenerScan($;$);
sub has_dependencies($$$);
sub has_em_results ($$$;$);
sub has_gen_key_value($$;);
sub has_entity_fn($$$;@);
sub has_gpnp_server_pool_fn($$$;@);
sub has_resource_crs_servers_fn($$$;@);
sub fnnpprnd ( $$$$$ );
sub fnnpprdp ( $$$$$ );
sub fnnpprnb ( $$$ );
sub hasGetResourceDependencyString($;);
sub hasGetResourceDependents($;);
#-------------------------------------------------------------------------------
# FUNCTION : hasGetFunctionString
#
# DESC 
#  return the string for function name + arguments
#
# ARGUMENTS
# sub name
# reference to the sub to be executed
# reference to the list of arguments to the sub
#
#-------------------------------------------------------------------------------
sub hasGetFunctionString($;@)
{
#  my ($sub,$array_ref) = @_;
  my ($sub,@args) = @_;
  #my @args;
  #@args = @$array_ref if $array_ref and ref($array_ref) and ref($array_ref) =~ /LIST/;
  #@args = $array_ref if $array_ref and ref($array_ref) and ref($array_ref) =~ /SCALAR/;
  warn "WARN:has::Common::hasGetFunctionString No function name passed for regression" and return unless $sub;
  #-----------------------------------------------------------------------------
  # Start Regression or capture
  #-----------------------------------------------------------------------------
  # create a mangled name for capture and regression test mode
  my $function_name = "fn_";
  # use the interface name to build the function string
  # helps during multiple shell invocations if they need different tests
  map
  {
    my $element_value = $_;
    $element_value =~ s/\s+//g if $element_value;
    undef $element_value if $element_value and $element_value =~ /^\s*$/;
    if ( not $element_value )
    {
      $function_name =  "$function_name\_null";
    }
    else
    {
      $function_name = "$function_name\_$element_value";
    }
  } @has::Common::iface if @has::Common::iface;
  $function_name = "$function_name\_$sub";
  # mangle the arguments into a function_name to be saved as a file
  # for non scalar refs, put the ref type and not the address
  # the function counter will take care of opening the right file
  # durign regression
  map 
  { 
    my $element_value = $_;
    $element_value =~ s/\s+//g if $element_value;
    undef $element_value if $element_value and $element_value =~ /^\s*$/;
    
    if ( not $element_value )
    {
       $function_name =  "$function_name\_null";
    }
    else
    {
      my $ref_type = ref($element_value) if $element_value; 
      
      if ( not $ref_type )
      { 
        $function_name =  "$function_name\_$element_value";
      }
      elsif (  $ref_type =~ /SCALAR/i )
      {
        $function_name = "$function_name\_$$element_value";
      }
      else
      {
        my $dmpstrg;
        $dmpstrg = Dumper($ref_type);
        $dmpstrg='' unless $dmpstrg;
        $function_name = "$function_name\_$dmpstrg";
      }
    }
  } @args if @args;
 
  # Remove these special characters from the file name
  $function_name =~ s/\n/_nl_/g;
  $function_name =~ s/\@/_a_/g;
  $function_name =~ s/\&/_am_/g;
  $function_name =~ s/\*/_as_/g;
  $function_name =~ s/\\/_b_/g;
  $function_name =~ s/\(|\)|{|}|\[|\]/_br_/g;
  $function_name =~ s/\^/_c_/g;
  $function_name =~ s/\$/_d_/g;
  $function_name =~ s/\./_do_/g;
  $function_name =~ s/\!/_e_/g;
  $function_name =~ s/\`/_es_/g;
  $function_name =~ s/\=/_eq_/g;
  $function_name =~ s/\\|\//_fs_/g;
  $function_name =~ s/\#/_h_/g;
  $function_name =~ s/\-/_hf_/g;
  $function_name =~ s/\n/_nl_/g;
  $function_name =~ s/\+/_p_/g;
  $function_name =~ s/\|/_pp_/g;
  $function_name =~ s/\s+/_s_/g;
  $function_name =~ s/\~/_t_/g;
  $function_name =~ s/\,/_cm_/g;
  $function_name =~ s/\:/_sc_/g;
  warn "WARN:has::Common::hasGetFunctionString Failed to generate a function signature for sub $sub \n" 
   and return unless $function_name;
  # Appened a count to the function name 
  # The same function may be invoked multiple times with
  # different results
  $has::Common::regression_fn_count{$function_name}=0 
    unless $has::Common::regression_fn_count{$function_name};
  $has::Common::regression_fn_count{$function_name}++;
  $function_name = "$function_name\_cnt\_$has::Common::regression_fn_count{$function_name}";
  return $function_name;
}
#-------------------------------------------------------------------------------
# FUNCTION :  readRegressionDatFile()
#
# DESC     : 
# read the contents of te regression/capture dat file and cache values to
# hash variable %has::Common::has_test_res_ref
#
# ARGS     :
#
#-------------------------------------------------------------------------------
sub readRegressionDatFile()
{
     
   if ( $has::Common::has_test_res_ref )
   {
     return $has::Common::has_test_res_ref;
   }
   my $has_result_fpath;
   $has_result_fpath = $ENV{HAS_TEST_FILE_PATH} if $ENV{HAS_TEST_FILE_PATH};
   $has_result_fpath = File::Spec->catfile(File::Spec->tmpdir(), $has::Common::has_test_res_filen) unless $has_result_fpath;
   stat($has_result_fpath);
   warn "WARN:has::Common::readRegressionDatFile File $has_result_fpath for persisting command results is not accessible\n"
     and return unless -e $has_result_fpath and -r $has_result_fpath;
   $has::Common::has_test_res_ref = do "$has_result_fpath";
   warn "WARN:has::Common::readRegressionDatFile Failed to read the captured test results from file $has_result_fpath\n"
    and return 
     unless 
     (
       $has::Common::has_test_res_ref
        and ref($has::Common::has_test_res_ref) =~ /HASH/i
         and keys %{$has::Common::has_test_res_ref}
     );
   return $has::Common::has_test_res_ref;
}
#-------------------------------------------------------------------------------
# FUNCTION :  get_fn_results($)
#
# DESC     : 
# return the captured results for a function call
#
# ARGS     :
#  $  - mangled interface name
#
#-------------------------------------------------------------------------------
sub get_fn_results($)
{
     
  my ($interface_name) = @_;
  #$interface_name =~ s/\s+//;
  warn "WARN:has::Common::get_fn_results Interface name expected as arg in get_fn_results\n" and return unless $interface_name;
  #$interface_name  =~ s/\./_dt_/g;
   
  my $intfcachref =  has::Common::readRegressionDatFile();
  warn "WARN:has::Common::get_fn_results Failed to find the interface name $interface_name \n"
   and return unless $intfcachref and ref($intfcachref) and defined $intfcachref->{$interface_name};
  my $fnresults = $intfcachref->{$interface_name};
  my $VAR1;
  eval $fnresults 
   or warn "WARN:has::Common::get_fn_results Failed to eval results for interface $interface_name\n" and return;
  return $VAR1;
}
#-------------------------------------------------------------------------------
# FUNCTION : regression_test
#
# DESC 
# Perform a capture for regression test, or perform a regression_test
# Specified by enironment variables
#
# ARGUMENTS
# sub name
# reference to the sub to be executed
# reference to the list of arguments to the sub
#
#-------------------------------------------------------------------------------
sub regression_test($$;@)
{
  #my ($sub,$sub_ref,$array_ref) = @_;
  my ($sub,$sub_ref,@args) = @_;
  warn "WARN::has::Common::regression_test Subroutine name for regression is null" and return unless $sub;
  #my @args = @$array_ref;
  #-----------------------------------------------------------------------------
  # fall thru below this line only for test mode
  # either REGRESSION or CAPTURE
  #-----------------------------------------------------------------------------
  # The test mode validation
  warn "WARN:has::Common:: regression_test Unknown run mode  can be REGRESSION or CAPTURE\n"
   and return unless $ENV{HAS_TEST_MODE} and $ENV{HAS_TEST_MODE} =~ /REGRESSION|CAPTURE/i;
  #-----------------------------------------------------------------------------
  # Start Regression or capture
  #-----------------------------------------------------------------------------
  # create a mangled name for capture and regression test mode
  my $function_name;
  $function_name = has::Common::hasGetFunctionString($sub,@args);
  #$function_name = has::Common::hasGetFunctionString($sub,$array_ref);
  warn "WARN::has::Common::regression_test Function name for regression is null" and return unless $function_name;
  #------------------------------------------------------------
  # If test regression mode read from stored file and return 
  # results
  #------------------------------------------------------------
  if ( $ENV{HAS_TEST_MODE} =~ /REGRESSION/i )
  {
    
    # no results expected, execute the function and return
    return &$sub_ref(@args) if not defined wantarray;
    my $result_ref = has::Common::get_fn_results($function_name);
    return unless $result_ref;
 
    # results expected is a list
    if ( wantarray )
    {
      return @$result_ref; 
    }
    else
    {
    # result expected is a scalar or 
    # a reference to a list
     my $ref_type = ref($result_ref);
     # if the result read is a reference to a scalar, 
     # return scalar value
     return $$result_ref if $ref_type =~ /SCALAR/i;
     # if result read is a ref to a list , return reference
     return $result_ref;
    }
  }
  #------------------------------------------------------
  # fall thru below this line only If test capture mode
  #------------------------------------------------------
  # no results expected, so nothing to capture, 
  # execute the subroutine and return
  #return &$sub_ref(@args) if not defined wantarray;
  #------------------------------------------------------
  # fall thru below this line only If return value is 
  # expected
  #------------------------------------------------------
  # If an list result is expected 
  if ( wantarray )
  {
   my @results_array =  &$sub_ref(@args);
   # Store the results   
   #return 1 if $has::Common::has_test_res_ref->{$function_name};
   return @results_array if $has::Common::has_test_res_ref->{$function_name};
   my $dumped_results;
   if ( \@results_array )
   {
     $dumped_results = Dumper(\@results_array) or warn "WARN::has::Common::regression_test Failed to save results for function $function_name\n" and return;
   }
 
   $has::Common::has_test_res_ref->{$function_name} = $dumped_results if $dumped_results;
   $has::Common::has_test_res_ref->{$function_name} = undef unless $dumped_results;
   # return the results
   return @results_array;
  
  }
  else
  # an scalar result is expected
  # scalar could be an reference to a list
  {
   my $results_ref = &$sub_ref(@args);
   my $store_results_ref = $results_ref;
   # if result is not a ref
   # take a pointer to the scalar
   # to store results
   my $ref_type = ref($results_ref);
   $store_results_ref = \$results_ref unless $ref_type;
   # Store the results   
   #return 1 if $has::Common::has_test_res_ref->{$function_name};
   if ( not defined $has::Common::has_test_res_ref->{$function_name} )
   {
     my $dumped_results;
     if ( $store_results_ref )
     {
      $dumped_results = Dumper($store_results_ref) 
        or warn "WARN:has::Common::regression_test Failed to save results for function $function_name\n" and return;
     }
   
     $has::Common::has_test_res_ref->{$function_name} = $dumped_results if $dumped_results;
     $has::Common::has_test_res_ref->{$function_name}  = undef unless $dumped_results;
   }
   # return the results
   return $results_ref if $results_ref;
   return;
  }
}
#------------------------------------------------------------------------------
# FUNCTION :    hasInit
#
# DESC
#  do initialization before using library
#   should be called explicityly by calling function
#
# ARGUMENTS
# crsHome
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub hasInit(;$)
{
  my ( $crsHome ) = @_;
  has::Common::hasCleanup($crsHome);
  $Data::Dumper::Indent = 2;
  $Data::Dumper::Deepcopy = 1;
  my $xml_error_ref = has::Common::make_element('errors');
  undef  %has::Common::has_xml_errors;
  %has::Common::has_xml_errors = %{$xml_error_ref};
  return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasInitSignalsStderr
#
# DESC
# initialize 
# Perform the initialization steps
# Prepare the directory for logging
#                 
# ARGUMENTS
# 
#-----------------------------------------------------------------------------
sub hasInitSignalsStderr() 
{
  
 #install signal handlers for warn and die
 $SIG{'__DIE__'} = sub {  has::Common::has_handle_error( @_ );
                         has_exitfail() };
 $SIG{'__WARN__'} = sub { has::Common::has_handle_error( @_)};
 # initialize the library
 has::Common::hasInit();
 # Save the STDERR before redirecting it to logfile
 open(OLDERR,">&STDERR");
 # If there is an error opening a log file, redirect stderr to null
 my $devnull = File::Spec->devnull();
 open(STDERR,">$devnull");
 # read hostname and the ENV variables for target context 
 $has::Common::has_em_targettype= $ENV{EM_TARGET_TYPE} if $ENV{EM_TARGET_TYPE};
 $has::Common::has_em_targetname= $ENV{EM_TARGET_NAME} if $ENV{EM_TARGET_NAME};
 $has::Common::has_em_targetguid= $ENV{EM_TARGET_GUID} if $ENV{EM_TARGET_GUID};
 return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasCleanup
#
# DESC
#  do cleanup before leaving library
#   should be called explicityly by calling function
#
# ARGUMENTS
# crsHome
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub hasCleanup(;$)
{
  my ( $crsHome ) = @_;
  %has::Common::hasClusterConfig = ();
  @has::Common::oldOH = ();
  @has::Common::oldCRSHome = ();
  @has::Common::oldEMCRSHome = ();
  # cleanup errr stack from previous run
  %has::Common::has_error_stack = ();
  %has::Common::has_xml_errors = ();
  # hash variables to keep track of base resource type, attrrib base type
  # resource instance - resource name
  # internal resource types
  %has::Common::hasadm_restype_base = ();
  %has::Common::hasadm_restype_attrib = ();
  %has::Common::hasadm_resource_type_list = ();
  %has::Common::hasadm_resource_list = ();
  %has::Common::hasadm_resourceinst_list = ();
  %has::Common::hasadm_list_internalresource_types = ();
  return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasRestoreStderr
#
# DESC
# clean up, print results,print errors , successful exit
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
sub hasRestoreStderr()
{
 
  # Restore STDERR
  close(STDERR);
  # Restore back the stderr fd
  open(STDERR,">&OLDERR");
  close(OLDERR);
 # cleanup the library
 has::Common::hasCleanup();
  return 1;
}
#------------------------------------------------------------------------------
# FUNCTION : runsystemcommand($;$$)
#
# DESC 
# Run a system command , retry n times if it times out
#
# ARGUMENTS
# command to be executed ( e.g. crsctl start crs )
# variable args to the command ( eg nodename)
#  a hash ref with
#  {timeout} in seconds, default 120
#  {tries} no of tries , default 2. 
#  {timeout_return} flag 1 to indicate if function should return 
#   in case of timeout, default is to die
#  {exit_failure_list}
#  {exit_success_list}
#   The lists to indicate failure and sucess exit status if they are other
#   than 0=Success all other exit status are =Fail
#   You can define one of the list of both the lists
#    e.g. for 'crsctl start crs' 
#    {exit_failure_list}=[(1,3,5,6)]
#    {exit_success_list}=[(0,2)]
#
# RETURNS:
#  result set either an array or a arg
#  the $? for the os command as {command_return_status} of the hash ref arg
#  {command_error_message} returns $! $@
#
#------------------------------------------------------------------------------
sub runsystemcommand( $;$$ )
{
  my ($fullcmd,$args,$argref) = @_;
  # fix for bug 6261302
  # go over path and get the full path to the variable
  # Not reqd to do this in regression or capture mode when
  # ENV{HAS_TEST_MODE} = capture or regression
  return has::sCommon::runsystemcommand($fullcmd,$args,$argref) 
   if $ENV{HAS_TEST_MODE} and $ENV{HAS_TEST_MODE} =~ /capture|regression/i;
  my $cmd;
  my $cmdargs;
  #split any args from command
  ($cmd,$cmdargs) = ( $fullcmd =~ /^([^\s]+) (.*)/ );
  $cmd = $fullcmd and undef $cmdargs unless $cmd;
  $cmd =~ s/^\s+//g if $cmd;
  # $cmd is asolute path , invoke runsystemcommand
  return has::sCommon::runsystemcommand($fullcmd,$args,$argref)
    if has::Common::hasIsReadable($cmd);
  # command does not have path figure out path from env variable
  my @paths;
  my $fullpath;
  @paths = File::Spec->path();
  # on windows get the appended path
  if ( $^O  =~ /windows|mswin/i )
  {
   if ( @paths and $paths[0] )
   {    
     # Bug fix#9117595
     #  with perl 5.10 if command is prepended then first in path array is '.'
     my $pathtosplit = $paths[0];
     $pathtosplit =~ s/^\s+|\s+$//;
     if ( ( not $pathtosplit or $pathtosplit !~ /:(?!(\\|\/))/ ) and @paths > 1 and $paths[1] )
     {
       $pathtosplit = $paths[1];
       $pathtosplit =~ s/^\s+|\s+$//;
     }
     my @setpaths = split/:(?!(\\|\/))/,$pathtosplit if $pathtosplit and $pathtosplit =~ /:(?!(\\|\/))/;
     if ( @setpaths )
     {    
        my @newsetpath;
        for my $setpath ( @setpaths )
        {
         next unless $setpath;
         push @newsetpath,$setpath;
        }
        @setpaths = @newsetpath;
     }    
     if ( @setpaths )
     {
       #shift @paths;
       unshift @paths,@setpaths;
     }
   }
  }
  for my $path ( @paths )
  {
   
    for my $extn ( ( '', '.exe', '.bat' ) )
    {
     my $tcmd = "$cmd$extn";
     $fullpath = catfile($path,$tcmd);
     last if has::Common::hasIsReadable($fullpath);
     
     undef $fullpath;
     next;
    
    }
    last if $fullpath;
  }
  $fullpath = "$fullpath $cmdargs" if $fullpath and $cmdargs;
  $fullpath = $fullcmd unless $fullpath;
  return has::sCommon::runsystemcommand($fullpath,$args,$argref);
}
#------------------------------------------------------------------------------
# FUNCTION :    print_warn_message
#
# DESC
# print warning messages
#
# ARGUMENTS
#  message
#------------------------------------------------------------------------------
sub print_warn_message(@)
{ 
   my ( $message ) = @_; 
   chomp $message; 
   $message =~ s/\n/ /g if $message;
   $message =~ s/^\s+|\s+$//
    if $message;
   $message = '' unless $message;
   # log the message to the log file
   EMD_PERL_WARN("WARN:Warning message from script $message");
   # send the warning message to emagent
   print "em_warning=Warning message from script $message\n";
   return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    print_error_message
#
# DESC
# print error messages
#
# ARGUMENTS
#  message
#------------------------------------------------------------------------------
sub print_error_message(@)
{ 
   my ( $message ) = @_; 
   chomp $message; 
   $message =~ s/\n/ /g if $message;
   $message =~ s/^\s+|\s+$// if $message;
   $message = '' unless $message;
   # log the message to the log file
   EMD_PERL_ERROR("ERROR:Error message from script $message");
   # send the warning message to emagent
   print "em_error=Error message from script $message\n";
   return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    exit_fail
#
# DESC
# clean up, print errors before failure exit
#
# ARGUMENTS
#  message
#------------------------------------------------------------------------------
sub exit_fail(@)
{ 
   my ( $message ) = @_; 
   print_error_message($message);
   exit 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetXmlErrorStack
#
# DESC
#  returns an ref to the has xml error stack
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
sub hasGetXmlErrorStack()
{ 
  return \%has::Common::has_xml_errors;
}
#------------------------------------------------------------------------------
# FUNCTION :    get_error_stack
#
# DESC
#  returns an ref to the has error stack
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
sub get_error_stack()
{ 
  return \%has::Common::has_error_stack;
}
#------------------------------------------------------------------------------
# FUNCTION :    has_handle_error
#
# DESC
#  log the message to trace and for error and warn keep it in the stack to be
#   printed at the end
#
# ARGUMENTS
#  message
#------------------------------------------------------------------------------
#----------------------------------------------------------------------------
# SIGNAL handler for die and warn to Log error and warning messages
#----------------------------------------------------------------------------
# die will exit the program with a warn message
sub has_handle_error(@)
{ 
   my ( $message ) = @_; 
   my $mtype;
   chomp $message if $message; 
   $message =~ s/^\s+|\s+$// if $message;
   return unless $message;
   # log only errors to the em respository, ignore debug and trace messages
   ( $mtype, $message ) = ( $message =~ /^(ERROR|WARN|DEBUG|INFO)\s*:\s*(.+)/i ) if $message =~ /^(ERROR|WARN|DEBUG|INFO)\s*:/i;
   $message =~ s/^\s+|\s+$// if $message;
   $mtype =~ s/^\s+|\s+$// if $mtype;
   # c executable has :: between error and message , so remove the extra :
   #$message =~ s/^:+// if $message;
   # log the message to the log file
   if ( $mtype and $mtype =~ /WARN/i and $message )
   {
     EMD_PERL_WARN("WARN:has::Common:: $message");
   }
   elsif ( $mtype and $mtype =~ /DEBUG/i and $message )
   {
     EMD_PERL_DEBUG("DEBUG:has::Common:: $message");
   }
   elsif ( $mtype and $mtype =~ /INFO/i and $message )
   {
     EMD_PERL_INFO("INFO:has::Common:: $message");
   }
   elsif ( $mtype and $mtype =~ /ERROR/i and $message )
   {
     EMD_PERL_ERROR("ERROR:has::Common:: $message");
   }
   elsif ( $message )
   {
     EMD_PERL_ERROR("ERROR:has::Common:: $message");
   }
   $mtype = 'ERROR' unless $mtype;
   return 1 unless $mtype and $mtype =~ /ERROR|WARN/i and $message;
   # Log a message only once
   return 1 if $has::Common::has_error_stack{$mtype}{$message};
   $has::Common::has_error_stack{$mtype}{$message}=1;
   # maintain the serial order of the messages
   my $no_of_messages = keys %{$has::Common::has_error_stack{$mtype}};
   $has::Common::has_error_stack{$mtype}{$message}=$no_of_messages;
   # also store the error as xml
   my $errref   = has::Common::make_element('error');
   my $etyperef = has::Common::make_element('type',$mtype);
   my $emsgref  = has::Common::make_element('message',$message);
   has::Common::append_element($errref,$etyperef);
   has::Common::append_element($errref,$emsgref);
   has::Common::append_element(\%has::Common::has_xml_errors,$errref);
   return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    has_printerrors
#
# DESC
# print the errors with seperators
#
# ARGUMENTS
#  exit_type optional values fail|exit_fail|success
#
#------------------------------------------------------------------------------
sub has_printerrors
{
  my ( $exit_type ) = @_;
  my $em_err_type;
  $em_err_type = 'em_error' if $exit_type and $exit_type =~ /fail/i;
  $em_err_type = 'em_warning' unless $em_err_type;
  my $errstackref;
  $errstackref = has::Common::get_error_stack();
  return 1 unless $errstackref and ref($errstackref);
  for my $mtype ( qw ( ERROR WARN ) )
  {
    next unless $errstackref->{$mtype};
    # sort by the message serial num asc
    for my $msg ( sort 
       { $errstackref->{$mtype}{$a} <=> $errstackref->{$mtype}{$b}} 
        keys%{$errstackref->{$mtype}} )
    {
      next unless $msg;
      chomp $msg;
      $msg =~ s/^\s+|\s+$//g;
      next unless $msg;
      print "$em_err_type=$mtype|$msg\n";
    }
  }
  return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    save_systemcmdoutput
#
# DESC
#  save the comd output from os commandinvoked thru runsystemcommand
#  to an os file - can be used for regression tests
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
sub save_systemcmdoutput()
{
  return 1 unless $ENV{HAS_TEST_MODE} and $ENV{HAS_TEST_MODE} =~ /CAPTURE/i;
  $has::Common::has_test_res_ref->{test_description} = $ENV{HAS_TEST_MODE_DESC} if $ENV{HAS_TEST_MODE_DESC};
  my $thewholestrg = Dumper($has::Common::has_test_res_ref)
   or warn "WARN::has::Common::save_systemcmdoutput Failed to save the results for test $has::Common::has_test_res_ref\n" and return;
  my $test_file = catfile(File::Spec->tmpdir(),$has::Common::has_test_res_filen);
  stat($test_file);
  warn "WARN::has::Common::save_systemcmdoutput File $test_file for captured regression test data is not accessible\n" and return if -e $test_file and not -w $test_file;
  open(TFH,">$test_file") or warn "WARN::has::Common::save_systemcmdoutput Failed to open file $test_file for capturing test results \n" and return;
  print TFH $thewholestrg;
  close(TFH);
  return 1;
}
# name : has_start_handler
# desc : handler to be invoked by perl parser when starting an element
#
# arg :
#  to be passed by the perl parser
sub has_start_handler
{
  my ($pr,$el,%attrs) = @_;
  $pr->{cdata_buffer} = '';
  my %ehash = (element=>$el);
  $has_fref = \%ehash unless  $has_fref;
  for my $name ( keys %attrs )
  {
    $ehash{attrs}{$name}=$attrs{$name};
  }
  $ehash{parent} = $has_xref if $has_xref;
  $ehash{depth} = $ehash{parent}->{depth}+1 if $ehash{parent};
  $ehash{depth} = 0 unless $ehash{depth};
  push @{$has_xref->{child_elements}{$el}},\%ehash if $has_xref;
  push @{$has_xref->{children}},\%ehash if $has_xref;
  $has_xref = \%ehash;
}
# name : has_end_handler
# desc : handler function for perl parser when element closes
#
# arg : 
#  passed by perl parser
sub has_end_handler
{
  my ($pr,$el) = @_;
  $has_xref = $has_xref->{parent};
}
# name : has_char_handler
# desc : handler function for perl parser for char
#
# arg 
#  passed by parser
#
sub has_char_handler
{
  my ($pr,$tag) = @_;
  $has_xref->{name}=$tag unless $has_xref->{name};
  $has_xref->{name} =~ s/^\s|\s+$//g;
  chomp $has_xref->{name};
#  delete $has_xref->{name} unless $has_xref->{name};
}
# name : has_parse_xml
# desc :  parse a xml string to a perl variable
#
# arg  : 
#  xml string to be parsed
#
# return:
#  hash of parsed perl variable
#
sub parse_xml($)
{
 my ( $result ) = @_;
 warn "WARN:has::Common::parse_xml No XML content to parse" and return unless $result;
 my $p = new XML::Parser(ErrorContext => 2,
                        ProtocolEncoding => 'UTF-8',
                        );
 $p->setHandlers(Start => \&has_start_handler,
	        End => \&has_end_handler,
                Char  => \&has_char_handler);
 undef $has_fref;
 undef $has_xref;
 # save the signal handler defined for die
 my $diesh = $SIG{__DIE__} if $SIG{__DIE__};
 # remove any signal handler defined for die
 $SIG{__DIE__}='';
 eval{  $p->parse($result) };
 # restore back the original die signal handler
 $SIG{__DIE__} = $diesh if $diesh;
 die  "ERROR:has::Common::parse_xml $@ Failed to Parse $result\n" if $@ and $result;
 die  "ERROR:has::Common::parse_xml $@ Failed to Parse \n" if $@;
 return %$has_fref;
}
# name : traverse_xml
# desc : traverse the xml tree, execute specificed function for each element
#
# args :
#  ref to hash of root of xml
#  ref to error handlig function
#  ref to traverse function
#  ref to list of args to function
#
#sub traverse_xml(\%\&\&@)
sub traverse_xml($$$@)
{
  my ( $xmlref,$fnerrhndl,$fnref,@args) = @_;
  my @stack;
  # to print the array depth first
  push @stack, $xmlref if $xmlref;
  while ( my $xref = pop @stack )
  {
   next unless $xref;
   # keep error messages in the error stack
   #
   #  
   #   error|warn
   #   error_message
   #  
   #
   if ( $xref->{element} and $xref->{element} =~ /^error$/i  and $xref->{children} )
   {
     my $mtype;
     my $message;
     for my $ec ( @{$xref->{children}} )
     {
       $mtype = $ec->{name} if $ec->{element} =~ /^type$/i and $ec->{name};
       $message = $ec->{name} if $ec->{element} =~ /^message$/i and $ec->{name};
     }
     &{$fnerrhndl}("$mtype:$message") if $mtype and $message and $fnerrhndl;
   }
   # keep error messages in the error stack for opcode != 0
   #
   # 0
   # Succcess
   #
   elsif ( $xref->{element} and $xref->{element} =~ /^opcode$/i  and $xref->{children} )
   {
     my $code;
     my $mtype;
     my $message;
     my $emessage;
     for my $ec ( @{$xref->{children}} )
     {
       $code = $ec->{name} if $ec->{element} =~ /^code$/i and $ec->{name};
       $message = $ec->{name} if $ec->{element} =~ /^message$/i and $ec->{name};
     }
     # if code != 0 indicates a failure for that element
     if ( $code )
     {
        $mtype = 'WARN';  #a failure of a step is a warning as other steps might have succeeded
        my $entity_name;
        my $entity_type;
        my $parent_ref = $xref->{parent} if $xref->{parent};
 
        if ( $parent_ref and $parent_ref->{attrs} )
        {
          $entity_name = $parent_ref->{attrs}{entity_name} if $parent_ref->{attrs}{entity_name};
          $entity_type = $parent_ref->{attrs}{entity_type} if $parent_ref->{attrs}{entity_type};
          $emessage = "for $entity_type $entity_name" if $entity_type and $entity_name;
          $emessage = "for $entity_type" if $entity_type and not $emessage;
          $emessage = "for $entity_name" if $entity_name and not $emessage;
        }
        $message = "$message $emessage" if $message and $emessage;
        $message = "Failed $emessage" if $emessage and not $message;
        $message = "Failed" unless $message;
        &{$fnerrhndl}("$mtype:$message") if $mtype and $message and $fnerrhndl;
     }
   }
   # keep error messages in the error stack for return code != 0
   #
   # 0
   # Succcess
   #
   elsif ( $xref->{element} and $xref->{element} =~ /^returncode$/i  and $xref->{children} )
   {
     my $code;
     my $mtype;
     my $message;
     for my $ec ( @{$xref->{children}} )
    {
       $code = $ec->{name} if $ec->{element} =~ /^code$/i and $ec->{name};
       $message = $ec->{name} if $ec->{element} =~ /^mesg$/i and $ec->{name};
     }
     # if code != 0 indicates a failure for that element
     if ( $code )
     {
        $mtype = 'ERROR';  #a failure overall is an ERROR
        $message = "Failed" unless $message;
        
        &{$fnerrhndl}("$mtype:$message") if $mtype and $message and $fnerrhndl;
     }
   }
   # execute function for each element
   &{$fnref}($xref,@args) or die "ERROR:has::Common::traverse_xml Failed to execute function\n";
   next if $xref->{ignore};
   push @stack, reverse @{$xref->{children}} if $xref->{children};
  }
  return 1;
}
# name : delete_element
# desc :  delete a element , remove it from the xml tree
#
# arg :
#  ref to the hash of the element to be removed
sub delete_element($)
{
  my ($elref) = @_;
  # mark it to be ignored
  $elref->{ignore}=1;
  # delete its children
  delete $elref->{children} if  $elref->{children};
  delete $elref->{child_elements} if  $elref->{child_elements};
  # delete it from the parent elements
  if ( $elref->{parent} and 
        $elref->{parent}->{child_elements} and
         $elref->{parent}->{child_elements}{$elref->{element}} )
  {
    my @child_elements = 
      @{$elref->{parent}->{child_elements}{$elref->{element}}};
    for my $i ( 0..@child_elements )
    {
      splice 
       @{$elref->{parent}->{child_elements}{$elref->{element}}},
        $i,1  if $child_elements[$i] == $elref;
    }
  }
  if ( $elref->{parent} and 
        $elref->{parent}->{children} )
  {
    my @child_elements = @{$elref->{parent}->{children}};
    for my $i ( 0..@child_elements )
    {
      splice @{$elref->{parent}->{children}},$i,1
       if $child_elements[$i] == $elref;
    }
  }
  %{$elref}=();
  undef %{$elref};
  return 1;
}
# name : mark_depth
# desc : mark depth of each element in xml var
#
# arg
#  ref to hash of parent var
#
sub mark_depth($)
{
  my ( $elref ) = @_;
  die "ERROR: Parent node has no depth defined for ".Dumper($elref) if $elref->{parent} and $elref->{parent}->{depth} !~ /\d+/;
  $elref->{depth} = $elref->{parent}->{depth} + 1 if $elref->{parent};
  $elref->{depth} = 0 unless $elref->{depth};
  return 1;
}
# name : append_element
# desc append element
#
# args
#  ref to parent
#  ref to child
# return:
#  ref to parent
sub append_element( $$ )
{
  my ( $pref, $cref ) = @_;
  push @{$pref->{children}},$cref;
  push @{$pref->{child_elements}{$cref->{element}}},$cref;
  $cref->{parent} = $pref;
  return $pref;
}
# name : make_element
# make a new element
#
# args:
#  element
#  value
#  ref to attributes array
# return:
#  ref to hash of the new element
sub make_element ($;$$)
{
  my ( $elem, $name, $attrs_ref ) = @_;
  my %element;
  $element{element} = $elem;
  $element{name}    = $name if $name;
  $element{name}    = undef unless $element{name};
  return \%element unless $attrs_ref and ref($attrs_ref) =~ /HASH/i and keys %{$attrs_ref};
  for my $attr ( keys %{$attrs_ref} )
  {
    $element{attrs}{$attr} = $attrs_ref->{$attr} if $attr;
  }
  return \%element;
}
# name : print_xml
# desc : print formatted xml from xml variable
#
# args :
#  xml doc type header
#  ref to hash of root element
#
sub print_xml($$)
{
  my ( $has_xml_doctype,$xmlref ) = @_;
  my $xmlstrg = has::Common::dump_xml($has_xml_doctype,$xmlref);
  print "$xmlstrg";
  return 1;
}
# name : dump_xml
# desc : dump formatted xml from xml variable to string
#
# args :
#  xml doc type header
#  ref to hash of root element
#
# returns:
#  xml dumped to a string
#
sub dump_xml($$)
{
  my ( $has_xml_doctype,$xmlref ) = @_;
  my @stack;
  my %to_close_at;
  my $xmlstring = '';
  # to close the element
  sub close_elements( \%\@$$ )
  {
   my ($tca_ref,$s_ref,$xref,$xmlstring) = @_;
   # before printing the current element,close any elements that 
   # are to be closed
   for my $dpth ( sort {$b <=> $a} keys %{$tca_ref} )
   {
     last unless $dpth >= $xref->{depth};
     for my $stk( sort {$b <=> $a} keys %{${$tca_ref}{$dpth}} )
     {
       last unless $stk >= @{$s_ref};
       my $temp = sprintf("%*s${$tca_ref}{$dpth}{$stk}>\n",$dpth,' ');
       $xmlstring= "$xmlstring$temp"  if $temp;
       delete ${$tca_ref}{$dpth}{$stk};
     }
     delete ${$tca_ref}{$dpth};
   }
  
   return $xmlstring;
  }
  # make sure the depth is marked for each element as it is required for indenting
  has::Common::traverse_xml($xmlref,\&has::Common::has_handle_error,\&has::Common::mark_depth);
  $xmlstring = "$has_xml_doctype\n";
  # to print the array depth first
  push @stack, $xmlref;
  while ( my $xref = pop @stack )
  {
   next unless $xref;
   $xref->{depth} = 0 unless defined $xref->{depth};
   $xmlstring=close_elements(%to_close_at,@stack,$xref,$xmlstring);
   my $temp = '';
   $temp = sprintf("%*s<",$xref->{depth},' ');
   $temp = "$temp$xref->{element}" if $xref->{element};
   $temp = $temp.'UNKNOWN' unless $xref->{element};
   # open element
   $xmlstring = "$xmlstring$temp"  if $temp;
   # print attributes
   for my $attr ( keys %{$xref->{attrs}} )
   {
    $xmlstring = 
     "$xmlstring $attr=\'$xref->{attrs}{$attr}\'";
   }
    $xmlstring = "$xmlstring>";
   $xmlstring = "$xmlstring$xref->{name}" if defined $xref->{name};
   if ( $xref->{children} )
   {
    $xmlstring = 
     "$xmlstring\n";
    $to_close_at{$xref->{depth}}{@stack}=$xref->{element};
    push @stack, reverse @{$xref->{children}};
   }
   else
   {
    $xmlstring = "$xmlstring$xref->{element}>\n" if $xref->{element};
    $xmlstring = "$xmlstring\n" unless $xref->{element};
   }
  }
  $xmlstring=close_elements(%to_close_at,@stack,$xmlref,$xmlstring);
  return $xmlstring;
}
#------------------------------------------------------------------------------
# FUNCTION :    fntrdf
#
# DESC
# Traverse the tree based on the start and tag passed
# and the specified order
#
# ARGUMENTS
#  order - preorder,postorder
#  node
#  traverse tag
#  key index
#  print fn or any other fn to act on each node
#  ref to the print string 
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub fntrdf( $$$$\&$;$$$ )
{
  my ($order,$node, $tvtag, $iekv, $fnptr,$strg_ref,$stack_ref,$has_clpstk,$has_indnthsh) = @_;
  # This is a closed loop, the stack keeps track of the node 
  # pointers already traversed
  if ( $has_clpstk and $has_clpstk->{$tvtag}{$node} )
  {
    warn "WARN:has::Common::fntrdf:Node $node->{resource} $node->{entity_type} is in a closed loop when traversing $tvtag\n ";
    return 1;
  }
  # mark the node as traversed if previously not marked so
  # this will help identify nodes which are in completely closed
  # loops
  $node->{traversed}=1 unless $node->{traversed};
  # the depth of the tree at any point 
  # is the size of the stack
  my $depth = keys %{$stack_ref};
  $depth +=1;
  $stack_ref->{$depth}=$node;
  # keep an index of the node pointer , for closed loop check
  $has_clpstk->{$tvtag}{$node}=1;
  my @kvs;
  my $node_count = 0;
  # traverse each child node specified by the traverse_tag
  if ( defined $node->{$tvtag} 
        and $node->{$tvtag}
         and ref($node->{$tvtag}) =~ /HASH/i
          and keys %{$node->{$tvtag}}
  )
  {
    # List of keys from this node to traverse in the direction
    # specified by traverse tag
    @kvs = sort { $iekv->{$a}->{type} cmp $iekv->{$b}->{type} } keys %{$node->{$tvtag}};
    # count of nodes to travserse from this node
    $node_count = @kvs;
  }
  $has_indnthsh->{$depth}=$node_count;
  # preorder processing
  if ( $order =~ /preorder/i)
  {
    if ( $fnptr )
    {
      # Execute the function to execute before traversing down the tree
      if ( not $fnptr->($node, $tvtag, $stack_ref, $strg_ref,$has_indnthsh) )
      {
        delete $stack_ref->{$depth};
        delete $has_clpstk->{$tvtag}{$node};
        return;
      }
    }
  }
  # traverse each child node specified by the traverse_tag
  if ( @kvs )
  {
    for my $i ( 1..$node_count )
    {
      my $kv = $kvs[$i-1];
      next unless $kv;
      warn "WARN:has::Common::fntrdf:Failed to find the storage entity $kv in the indexed list\n"
       and return
        unless $iekv->{$kv};
      my $next_node = $iekv->{$kv};
      next unless $next_node;
      bless $next_node;
      # keeps track of the pending children at any 
      # depth in the layout
      $has_indnthsh->{$depth} = $node_count-$i;
      if 
      ( 
        not 
         fntrdf
         (
          $order,
          $next_node,
          $tvtag,
          $iekv,
          &$fnptr,
          $strg_ref,
          $stack_ref,
          $has_clpstk,
          $has_indnthsh
         )
      )
      {
        # unwind the stack and the closed loop index
        delete $stack_ref->{$depth};
        delete $has_clpstk->{$tvtag}{$node};
        return;
      }
    }
 }
  # postorder processing
  if ( $order =~ /postorder/i)
  {
   if ( $fnptr )
   {
      # Execute the function to execute before traversing down the tree
     if ( not  $fnptr->($node, $tvtag, $stack_ref,$strg_ref,$has_indnthsh ) )
     {
        delete $stack_ref->{$depth};
        delete $has_clpstk->{$tvtag}{$node};
        return;
     }
   }
  }
  # unwind the stack and the closed loop index
  delete $stack_ref->{$depth};
  delete $has_clpstk->{$tvtag}{$node};
  return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    fntrbf
#
# DESC
# Traverse the tree based on the start and tag passed
# and the specified order
#
# ARGUMENTS
#  order - preorder,postorder
#  node
#  traverse tag
#  key index
#  print fn or any other fn to act on each node
#  ref to the result hash
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub fntrbf( $$$$\&$; )
{
  my ( $order, $node, $tvtag, $iekv, $fnptr, $res_ref ) = @_;
  my %trvstak;
  my @queue;
  # initialize the queue
  push @queue, $node;
  # while there are nodes in the queue
  while ( ( my $nnode = shift @queue)  )
  {
    # If the node has been processed skip it
    next 
     if $trvstak{$tvtag}{$nnode};
    # keep track of the processed nodes
    $trvstak{$tvtag}{$nnode}=1;
    # preorder processing - process node before pushing the children to the queue
    if ( $order =~ /preorder/i)
    {
     if ( $fnptr )
     {
       # invoke the function on the nnode
       if ( not $fnptr->($nnode, $tvtag, $res_ref) )
       {
         warn "Failed executing the function in fntrbf for node \n";
         return;
       }
     }
    }
    # if the nnode has children push them to the queue
    if
    (
      $nnode
        and ref($nnode)
        and ref($nnode) =~ /HASH/i
        and $nnode->{$tvtag}
         and ref($nnode->{$tvtag}) =~ /HASH/i
          and keys %{$nnode->{$tvtag}}
    )
    {
      # push each child node to the queue
      for my $ckv ( sort { $iekv->{$a}->{type} cmp $iekv->{$b}->{type} } keys %{$nnode->{$tvtag}} )
      {
        next unless $ckv;
        warn "Failed to find the $tvtag key_value $ckv in iekv for node \n"
         and return 
          unless $iekv->{$ckv}; 
        my $cnd = $iekv->{$ckv};
        warn "The $tvtag node key_value $ckv is not a hash in iekv for node \n"
         and return 
          unless 
          (
           $cnd
            and ref($cnd)
            and ref($cnd) =~ /HASH/i
            and keys %{$cnd}
          );
       push @queue,$cnd;
     }
   }
   # postorder processing - process node after pushing the children to the queue
   if ( $order =~ /postorder/i)
   {
    if ( $fnptr )
    {
      # invoke the function on the nnode
      if ( not $fnptr->($nnode, $tvtag, $res_ref ) )
      {
        warn "Failed executing the function in fntrbf for node \n";
        return;
      }
    }
   }
 }
 return 1;
}
#---------------------------------------------------------------------------------
# FUNCTION : has_get_cache_dir
#
# DESC 
# Get an directory to cache data on the host target
#
# ARGUMENTS
#
#---------------------------------------------------------------------------------
sub has_get_cache_dir ()
{
  my $cache_dir;
  return $has::Common::has_metric_config{cache_dir} if $has::Common::has_metric_config{cache_dir};
  $cache_dir = $ENV{EMSTATE} if $ENV{EMSTATE};
  # if the em state dir doesnt work use temp dir
  $cache_dir =  File::Spec->tmpdir() unless has::Common::hasIsWriteable($cache_dir) and has::Common::hasIsDir($cache_dir);
  # if temp doest work use current work dir
  $cache_dir = Cwd::abs_path()  unless has::Common::hasIsWriteable($cache_dir) and has::Common::hasIsDir($cache_dir);
  # terminate if it still doesnt work
  warn "WARN:Unable to get write access to cache directory $cache_dir\n"
   and return unless has::Common::hasIsWriteable($cache_dir) and has::Common::hasIsDir($cache_dir);
  # build the path to the em state directory
  $cache_dir =  catfile($cache_dir,'sysman');
  $cache_dir =  catfile($cache_dir,'emd');
  $cache_dir =  catfile($cache_dir,'state');
  $cache_dir =  catfile($cache_dir,'has');
  my $hashostName = has::Common::hasGetLocalHostName();
  for my $cdir (     ($hashostName,
                      $has::Common::has_em_targettype, 
                      $has::Common::has_em_targetname,
                      $has::Common::has_em_targetguid ) )
  {
   $cdir =~ s/\s+//g;
   #for portability use only 8 digits 
   $cdir = substr($cdir,0,8);
   next unless $cdir;
   $cache_dir = catfile($cache_dir,$cdir);
  }
  mkpath($cache_dir,0,0777) unless has::Common::hasIsReadable($cache_dir);
  warn "WARN:Failed to get a cache dir $cache_dir to cache metric files"
   and return unless has::Common::hasIsWriteable($cache_dir) and has::Common::hasIsDir($cache_dir);
  return $cache_dir;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasDoesFileExist
#
# DESC
# true if path exists false otherwise
#
# ARGUMENTS
# path
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub hasDoesFileExist($;)
{
  my ( $path ) = @_;
  return unless $path;
  stat $path;
  return 1 if -e $path;
  return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasIsReadable
#
# DESC
# true if path is readable false otherwise
#
# ARGUMENTS
# path
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub hasIsReadable($;)
{
  my ( $path ) = @_;
  return unless $path;
  stat $path;
  return 1 if -e $path and -r $path;
  return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasIsWriteable
#
# DESC
# true if path is writeable false otherwise
#
# ARGUMENTS
# path
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub hasIsWriteable($;)
{
  my ( $path ) = @_;
  return unless $path;
  stat $path;
  return 1 if -e $path and -r $path and -w $path;
  return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasIsDir
#
# DESC
# true if dir false otherwise
#
# ARGUMENTS
# path
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub hasIsDir($;)
{
  my ( $path ) = @_;
  return unless $path;
  stat $path;
  return 1 if -e $path and -r $path and -d $path;
  return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasReturnFileContents
#
# DESC
# print file contents to stdout
#
# ARGUMENTS
# dir
# file name
#
# RETURNS
#  return contents dumped to string
#------------------------------------------------------------------------------
sub hasReturnFileContents($;$)
{
  my ($dir,$file) = @_;
  my $pstring;
  my $flnm;
  $flnm = catfile($dir,$file) if $dir and $file;
  $flnm = $dir if $dir and not $flnm;
  warn "WARN:has::Common::hasReturnFileContent file  $flnm does not exist\n" and return
   unless has::Common::hasDoesFileExist($flnm);
  warn "WARN:has::Common::hasReturnFileContent file  $flnm not readable\n" and return
   unless has::Common::hasIsReadable($flnm);
  # Open the file for reading , if it fails dont return an error,
  # log an error and return gracefully, so metric is blank but with errror
  open(CSHFH,"$flnm") or warn "WARN:has::Common::hasReturnFileContents Failed to open the cached file $flnm \n" and return;
  my @cols = ;
  close(CSHFH) or warn "WARN:has::Common::hasReturnFileContents Failed to close the file $flnm \n";
  for my $row ( @cols )
  {
    chomp $row if $row;
    $row =~ s/^\s+|\s+$//g if $row;
    $pstring =  "$pstring$row\n" and next if $pstring and $row;
    $pstring =  "$row\n" and next if $row;
  }
  return $pstring if $pstring;
  return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasWriteToFile
#
# DESC
# write contents of array to file
#
# ARGUMENTS
# dir
# file name
# array with contents
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub hasWriteToFile($$$;)
{
  my ($dir,$file,$resref) = @_;
  # open the cache file for the metric and save the results there
  my $flnm = catfile($dir,$file);
  open(CSHFH,'>',$flnm) or close(CSHFH) 
   and warn "WARN:has::Common::hasWriteToFile Failed to open the file $flnm \n" and return;
  # read and save each line to file
  for my $row ( @{$resref} )
  {
    chomp $row if $row;
    $row =~ s/^\s+|\s+$//g if $row;
    print CSHFH "$row\n" if $row;
  }
  close(CSHFH) or warn "WARN:has::Common::hasWriteToFile Failed to close the file $flnm \n";
  return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasSetCRSEnv
#
# DESC
# Set the cluster environment for the cli/api
#
# ARGUMENTS
#  CRS Home if available
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub hasSetCRSEnv(;$)
{
  my ($crsHome) =@_;
  my  $pathsep = ':';
  my $os = has::Common::get_osType();
  $pathsep = ';' if $os and $os =~ /^WIN$/;
  push @has::Common::oldOH,$ENV{ORACLE_HOME} if $ENV{ORACLE_HOME};
  push @has::Common::oldCRSHome,$ENV{CRS_HOME} if $ENV{CRS_HOME};
  push @has::Common::oldEMCRSHome,$ENV{EM_CRS_HOME} if $ENV{EM_CRS_HOME};
  push @has::Common::oldOH,'' unless $ENV{ORACLE_HOME};
  push @has::Common::oldCRSHome,'' unless $ENV{CRS_HOME};
  push @has::Common::oldEMCRSHome,'' unless $ENV{EM_CRS_HOME};
  if ( $crsHome )
  {
    undef $crsHome unless has::Common::hasIsReadable($crsHome);
  }
  if ( not $crsHome )
  {
    $crsHome = has::Common::hasGetCRSHome();
  }
  if ( not $crsHome )
  {
    $crsHome = has::Common::hasGetEnvCRSHome();
  }
  $ENV{CRS_HOME} = $crsHome if $crsHome;
  $ENV{ORACLE_HOME} = $crsHome if $crsHome;
  $ENV{PATH} = '' unless $ENV{PATH};
  my %pathOrder = (1=>'EMDROOT', 2=>'CRS_HOME');
  for my $order ( sort {$a <=> $b} keys %pathOrder )
  {
    my $path = $pathOrder{$order};
    next unless $ENV{$path};
    my $binPath = catfile($ENV{$path},'bin');
    $ENV{PATH} = "$binPath$pathsep$ENV{PATH}" if $binPath;
  }
  # this is temporary to take care of emcrsp in has/bin not linked to oracle/bin
  if ( $ENV{CRS_HOME} and $ENV{ADE_VIEW_ROOT} and $ENV{NDE_PRODUCT} and not defined $ENV{HAS_TEST_MODE} )
  {
    my $hasbinpath = catfile($ENV{CRS_HOME},'..');
    $hasbinpath = catfile($hasbinpath,'bin');
     $ENV{PATH} = "$hasbinpath$pathsep$ENV{PATH}";
  }
  return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasRestoreCRSEnv
#
# DESC
# restore back the env 
#
# ARGUMENTS
#
#  RETURNS
#
#------------------------------------------------------------------------------
sub hasRestoreCRSEnv()
{
  my $oldOracleHome;
  my $oldCHome;
  my $oldEMCHome;
  $oldOracleHome = pop @has::Common::oldOH if @has::Common::oldOH;
  if ( $oldOracleHome )
  {
    $ENV{ORACLE_HOME} = $oldOracleHome;
  }
  else
  {
    delete $ENV{ORACLE_HOME} if $ENV{ORACLE_HOME};
  }
  $oldCHome = pop @has::Common::oldCRSHome if @has::Common::oldCRSHome;
  if ( $oldCHome )
  {
    $ENV{CRS_HOME} = $oldCHome;
  }
  else
  {
    delete $ENV{CRS_HOME} if keys %ENV and $ENV{CRS_HOME};
  }
  $oldEMCHome = pop @has::Common::oldEMCRSHome if @has::Common::oldEMCRSHome;
  if ( $oldEMCHome )
  {
    $ENV{EM_CRS_HOME} = $oldEMCHome;
  }
  else
  {
    delete $ENV{EM_CRS_HOME} if $ENV{EM_CRS_HOME};
  }
  return 1;
}
     
#------------------------------------------------------------------------------
# FUNCTION :    hasGetEnv
#
# DESC
# restore back the env before regression tests
# this will make sure env is stored when capturign for regression
# and is used durig regression
#
# ARGUMENTS
#
#  RETURNS
#  the ref to has of env variables 
#
#------------------------------------------------------------------------------
sub hasGetEnv()
{
  my %env;
  for my $k ( keys %ENV )
  {
    $env{$k} = $ENV{$k};
  }
  return \%env;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetNodeCRSHome
#
# DESC
# return the CRSHome for the node 
#
# ARGUMENTS
# nodename
# crsHome if known
#
# RETURN
# CRSHome
#
#------------------------------------------------------------------------------
sub hasGetNodeCRSHome(;$$)
{
# do not call clusterconfig or clusternodes here as those functions call this sub
# just look up the different data structures to pick the right value
  my ( $nodename,$crsHome ) = @_;
   
  my $confref = has::Common::getClusterCacheRef();
  $nodename =~ s/\s+//g if $nodename;
  if ( $nodename )
  {
   undef $nodename if $nodename =~ /^$/ or $nodename eq '';
  }
  $confref = has::Common::hasGetClusterConfig($crsHome);
  # get crs home for each node, assuming they are same on all nodes as we have no way
  # to get crsHome from other nodes unless css api provides it
  if ( $nodename and $confref and ref($confref) and keys %{$confref} )
  {
    my $ndCrsHome;
    for my $ch ( keys %{$confref} )
    {
      next unless $confref->{$ch} 
       and $confref->{$ch}{node}
       and $confref->{$ch}{node}{details}
       and $confref->{$ch}{node}{details}{$nodename}
       and $confref->{$ch}{node}{details}{$nodename}{crs_home};
      $ndCrsHome = $ch;
    }
    return $ndCrsHome if $ndCrsHome;
  }
  return has::Common::hasGetCRSHome($crsHome);
     
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetCRSHome
#
# DESC
# return the CRSHome for the local node
#
# ARGUMENTS
# crsHome if known
#
# RETURN
# CRSHome
#
#------------------------------------------------------------------------------
sub hasGetCRSHome(;$)
{
# do not call clusterconfig or clusternodes here as those functions call this sub
# just look up the different data structures to pick the right value
  my ( $crsHome ) = @_;
   
  my $disCRSHome;
  # get the discovery home
  my $confref = has::Common::hasDiscoverCluster();
  # is a crsHome is passed and it matched the discovered crsHome viola, that is the crsHome
  # the order of search directories for olsnode/lsnodes
  my %olsnodeorder;
  if ( $crsHome )
  {
    $olsnodeorder{0}{path}=$crsHome;
  }
  $olsnodeorder{10}{path}='CRS_HOME';
  $olsnodeorder{11}{path}='EM_CRS_HOME';
  $olsnodeorder{12}{path}='ORACLE_HOME';
  # counter for discovered firs in olsnodeorder
  my $i = 1;
  # pick the crsHome which matches the env variables
  # sort in desc IDX order as newer crs homes have higher IDX number in installs where theure might have been a upgrade
  my %crshomeidx;
  for my $ch ( keys %{$confref} )
  {
    if ( $confref->{$ch} and $confref->{$ch}{discover} and $confref->{$ch}{discover}{IDX} )
    {
     $crshomeidx{$ch} = $confref->{$ch}{discover}{IDX} and next ;
    }
    $crshomeidx{$ch} = -1;
  }
  for my $ch ( sort { $crshomeidx{$b} <=> $crshomeidx{$a} } keys %crshomeidx )
  {
    $disCRSHome = $ch;
    #if this is not discovered then skip
    next unless $confref->{$ch}{discover};
  
    for my $order ( sort  {$a <=> $b} keys %olsnodeorder )
    {
      my $path;
  
      next unless $olsnodeorder{$order} and $olsnodeorder{$order}{path};
  
      if ( $olsnodeorder{$order}{path} =~ /^(CRS_HOME|EM_CRS_HOME|ORACLE_HOME|EMDROOT)$/ )
      { 
         $path = $ENV{$olsnodeorder{$order}{path}} if $ENV{$olsnodeorder{$order}{path}};
      }
      else
      {
          $path = $olsnodeorder{$order}{path};
      }
  
      next unless $path;
      next if $path =~ /^(#CRS_HOME#|CRS_HOME|EM_CRS_HOME|ORACLE_HOME|EMDROOT)$/;
     
      # discovered crsHome from inventory is different from crsHome
      if ( $disCRSHome eq $path )
      {
        return $disCRSHome;
      }
      else
      {
        if ( $olsnodeorder{$order}{path} =~ /^(CRS_HOME|EM_CRS_HOME|ORACLE_HOME|EMDROOT)$/ )
        {
          warn "DEBUG:has::Common::hasGetCRSHome:Discovered cluster home $disCRSHome is different from $olsnodeorder{$order}{path} $path";
        }
        else
        {
          warn "DEBUG:has::Common::hasGetCRSHome:Discovered cluster home $disCRSHome is different from $path";
        }
      }
  
    }
    # no env matches the disc home, return passed crs home, discovered home  or env as crs home
    #  in that order
    $olsnodeorder{$i}{path}= $disCRSHome if $disCRSHome;
    $i++; 
  }
   # the discoovered dir does nto match any of the other dirs
  for my $order ( sort  {$a <=> $b} keys %olsnodeorder )
  {
    my $path;
    next unless $olsnodeorder{$order} and $olsnodeorder{$order}{path};
    if ( $olsnodeorder{$order}{path} =~ /^(CRS_HOME|EM_CRS_HOME|ORACLE_HOME|EMDROOT)$/ )
    { 
       $path = $ENV{$olsnodeorder{$order}{path}} if $ENV{$olsnodeorder{$order}{path}};
    }
    else
    {
        $path = $olsnodeorder{$order}{path};
    }
    next unless $path;
    next if $path =~ /^(#CRS_HOME#|CRS_HOME|EM_CRS_HOME|ORACLE_HOME|EMDROOT)$/;
    next unless has::Common::hasIsReadable($path);
    warn "DEBUG:has::Common::hasGetCRSHome: Using $path as cluster home" and return $path;
  }
  # no disc home and no env for crs home
  warn "WARN:has::Common::hasGetCRSHome Failed to get cluster home dir for " and return;
}
     
#------------------------------------------------------------------------------
# FUNCTION :    hasGetEnvCRSHome
#
# DESC
# return the CRSHome from the env 
#
# ARGUMENTS
#
# RETURN
# CRSHome
#
#------------------------------------------------------------------------------
sub hasGetEnvCRSHome()
{
  my %olsnodeorder;
  $olsnodeorder{1}{path}='CRS_HOME';
  $olsnodeorder{2}{path}='EM_CRS_HOME';
  $olsnodeorder{3}{path}='ORACLE_HOME';
  for my $order ( sort {$a <=> $b} keys %olsnodeorder )
  {
    my $path;
    next unless $olsnodeorder{$order} and $olsnodeorder{$order}{path};
    if ( $olsnodeorder{$order}{path} =~ /CRS_HOME|EM_CRS_HOME|ORACLE_HOME|EMDROOT/ )
    { 
       $path = $ENV{$olsnodeorder{$order}{path}} if $ENV{$olsnodeorder{$order}{path}};
    }
    else
    {
        $path = $olsnodeorder{$order}{path};
    }
    next unless $path;
    next if $path =~ /#CRS_HOME#/;
    next unless has::Common::hasIsReadable($path);
    return $path;
  }
  # no disc home and no env for crs home
  warn "WARN:has::Common::hasGetEnvCRSHome Failed to get cluster home from env " and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetLocalHostName
#
# DESC
# return the Host Name 
#
# ARGUMENTS
#
#------------------------------------------------------------------------------
sub hasGetOSLocalHostName()
{
  # return host name is not cluster
  my $hostName;
  #this module is not available in emagent 10.2.0.4
  #my $subOSDHost = "hostOSD::getHostName";
  #my $subref = \&$subOSDHost;
  if ( $hostNameSub )
  {
    my $fnref = \&$hostNameSub;
    $hostName =  &$fnref;
  }
  $hostName = hostname unless $hostName;
  return $hostName;
}
sub hasGetLocalHostName()
{
  
  my $confref = has::Common::getClusterCacheRef();
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{pre11gr2_config}{host_name} 
       if $confref->{$ch}{pre11gr2_config} 
      and $confref->{$ch}{pre11gr2_config}{host_name};
      return $confref->{$ch}{host_name} 
       if $confref->{$ch}{host_name};
    }
  }
  # return host name is not cluster
  my $hostName =  has::Common::hasGetOSLocalHostName();
  if ( $hostName and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      $confref->{$ch}{pre11gr2_config}{host_name} = $hostName;
      $confref->{$ch}{host_name} = $hostName;
    }
  }
  return $hostName if $hostName;
  warn "ERROR:has::Common::hasGetLocalHostName:Failed to get the host name for the cluster node" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetDomainName
#
# DESC
# return the Host Domain 
#
# ARGUMENTS
#  host name
#
#------------------------------------------------------------------------------
sub hasGetDomainName(;$)
{
  # return host Domain 
  my ( $hostName ) = @_;
  my $domain;
  $hostName = hostname unless $hostName;
  $hostName = hasGetLocalHostName() unless $hostName;
  if ( $hostDomainSub )
  {
    my $fnref = \&$hostDomainSub;
    $domain =  &$fnref($hostName);
  }
  return $domain if $domain;
  warn "DEBUG:has::Common::hasGetDomainName:Failed to get the Domain name for the node" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetLocalHostDomain
#
# DESC
# return the local Host Domain 
#
# ARGUMENTS
#  local host name or null
#
#------------------------------------------------------------------------------
sub hasGetLocalHostDomain(;$)
{
  # return host Domain 
  my ( $hostName ) = @_;
  my $hostdomain = Net::Domain::hostdomain();
  $hostdomain =~ s/^\s+|\s+$// if $hostdomain;
  $hostdomain =~ s/^\.// if $hostdomain and $hostdomain =~ /^\./;
  return $hostdomain if $hostdomain;
#  $hostdomain = has::Common::hasGetDomainName($hostName);
  
#  return $hostdomain if $hostdomain;
  warn "DEBUG:has::Common::hasGetLocalHostDomain:Failed to get the Domain name for the local Host" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetDomainNameNSLookup
#
# DESC
# return the Domain for a IP address or name
#
# ARGUMENTS
#  name or ip address
#
#------------------------------------------------------------------------------
sub hasGetDomainNameNSLookup($;)
{
  # return Domain 
  my ( $address ) = @_;
  # This function is a NOOP right now
  return;
  warn "DEBUG:has::Common::hasGetDomainNameNSLookup:Failed to get the Domain name for the address $address" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasCheckForEmcrsp
#
# DESC
#  checks if emcrsp is present
#
# ARGUMENTS
# crsHome if known
#
# RETRUNS
#  1 for true
#------------------------------------------------------------------------------
sub hasCheckForEmcrsp($)
{
  my ( $crsHome ) = @_;
  # no cluster home is passed then 
  # get the cluster home from the env 
  #  and if not available or no emcrsp then discover one
  if ( not $crsHome )
  {
    # get the cluster home by discovery
    $crsHome = has::Common::hasGetCRSHome();
    if ( not $crsHome )
    {
      $crsHome = has::Common::hasGetEnvCRSHome();
    }
  }
  warn "DEBUG:has::Common::hasCheckForEmcrsp: cluster home directory passed is null" and return unless $crsHome;
  # check if emcrsp exists in the crs bin dir
  my $emcrsppath;
  my %dirlist;
  # this is only in dev env
  my $hasbin = catdir('has','bin');
  %dirlist = ( 1=>$crsHome );
  for my $order ( sort {$a<=>$b} keys %dirlist )
  {
    my $dir = $dirlist{$order};
    next unless $dir;
    for my $bin ( ( 'bin',$hasbin ) )
    {
      my $binpath = catdir($dir,$bin);
      for my $exe ( ('emcrsp','emcrsp.exe','emcrsp.bat') )
      {
        $emcrsppath = catfile($binpath,$exe);
        if ( has::Common::hasIsReadable($emcrsppath) )
        {
          last;
        }
        else
        {
         undef $emcrsppath;
        } 
      }
       last if $emcrsppath;
    }
    last if $emcrsppath;
  }
  # use emcrsp to get node name if it exists
  warn "DEBUG:has::Common::hasCheckForEmcrsp emcrsp is not present in cluster home $crsHome" 
   and return unless $emcrsppath;
  return 1;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasDiscoverCluster
#
# DESC
# return a hash of
#    {coma_seperated =>
#     nodearray => )
#  of nodes in a cluster
#
# ARGUMENTS
#
# RETURNS
# hash result list
#------------------------------------------------------------------------------
sub hasDiscoverCluster(;$)
{
#
#   
#      
#      
#   
#
  sub has_inventory_fn(\%\%)
  {
    my ( $elref, $rsref ) = @_;
    return 1 unless $elref->{element} and $elref->{element} =~ /^(HOME)$/i;
    return 1
     unless $elref->{attrs} and $elref->{attrs}{CRS} and $elref->{attrs}{CRS} =~ /^true$/i;
    return 1
     if $elref->{attrs} and $elref->{attrs}{remote} and $elref->{attrs}{remote} =~ /^true$/i;
    return 1
     if $elref->{attrs} and $elref->{attrs}{REMOTE} and $elref->{attrs}{REMOTE} =~ /^true$/i;
    warn "WARN:has::Common::hasDiscoverCluster LOC is NULL for CRS in Oracle Inventory file"
    and return 1 unless $elref->{attrs} and $elref->{attrs}{LOC};
    # save oracle homes, add nodes later on
    my $ch = $elref->{attrs}{LOC};
    $rsref->{$ch}{discover}{crs_home}=$ch;
    my $idx = $elref->{attrs}{IDX} if exists $elref->{attrs}{IDX};
    $idx = 0 unless defined $idx;
    $idx++ if $idx =~ /\d+/;
    $rsref->{$ch}{discover}{IDX}=$idx;
    $rsref->{$ch}{discover}{IDX}=-1 unless $rsref->{$ch}{discover}{IDX};
    $rsref->{$ch}{discover}{TYPE}=$elref->{attrs}{TYPE} if $elref->{attrs}{TYPE};
    $rsref->{$ch}{discover}{TYPE}=0 unless $rsref->{$ch}{discover}{TYPE};
    # get NODE NAME from children
    return 1 unless  $elref->{children} and @{$elref->{children}};
    for my $nodeListRef ( @{$elref->{children}} )
    {
      next unless $nodeListRef and $nodeListRef->{element} and $nodeListRef->{element} =~ /^NODE_LIST$/i;
      next unless $nodeListRef->{children} and @{$nodeListRef->{children}};
      for my $noderef ( @{$nodeListRef->{children}} )
      {
         next unless $noderef and $noderef->{element} and $noderef->{element} =~ /^NODE$/i;
         warn "WARN:has::Common::hasDiscoverCluster NODE name is null in Oracle Inventory file"
          and next unless $noderef->{attrs} and $noderef->{attrs}{NAME};
         $rsref->{$ch}{discover}{nodes}{$noderef->{attrs}{NAME}}=1;
      }
      last;
    }
    return 1;
  }
  my ( $dynProp ) = @_;
  my $invPath;
  my $invXMLContent;
  my %xmlVar;
  my $confref = has::Common::getClusterCacheRef();
  # if already discovered returned the cached values
  if ( $confref and ref($confref) and keys %{$confref} )
  {
     for my $ch ( keys %{$confref} )
     {
        return $confref if $confref->{$ch}{discover};
     }
  }
  #bugfix 7692044
  my $os = has::Common::get_osType();
  my $scriptsDir = $ENV{EM_SCRIPTS_DIR} if $ENV{EM_SCRIPTS_DIR};
  my $perlBin = $ENV{EM_PERLBIN_DIR} if $ENV{EM_PERLBIN_DIR};
  if ( $os and $os =~ /^WIN$/ )
  {
   #bugfix 8988446
   my $perlexe;
   if ( $perlBin )
   {
     $perlexe = catfile($perlBin,'perl');
   }
   if ( $scriptsDir )
   {
     $scriptsDir =  catfile($scriptsDir,'has');
   }
   if ( not $scriptsDir )
   {
       $scriptsDir = $FindBin::Bin if $FindBin::Bin;
   }
   if ( not $perlexe )
   {
       $perlexe = $Config{perlpath} if $Config{perlpath};
   }
   if ( $scriptsDir and $perlexe )
   { 
     my $has_metrics_script =  catfile($scriptsDir,'has_getinvdir.pl');
     $invPath = `$perlexe $has_metrics_script`;
     chomp $invPath if $invPath;
   }
      
  }
  else
  {
    $invPath= has::Common::getInventoryXmlPath();
  }
  warn "WARN:has::Common:hasDiscoverCluster Failed to find Oracle Inventory File " and return unless $invPath;
  warn "WARN:has::Common:hasDiscoverCluster Failed to read Oracle Inventory File $invPath" 
   and return unless has::Common::hasIsReadable($invPath);
  $invXMLContent = has::Common::hasReturnFileContents($invPath);
  warn "WARN:has::Common:hasDiscoverCluster Oracle Inventory file $invPath has no content\n" 
   and return unless $invXMLContent;
  %xmlVar =  has::Common::parse_xml($invXMLContent);
  has::Common::traverse_xml(\%xmlVar,\&has::Common::has_handle_error,\&has_inventory_fn,$confref) 
    or warn "WARN:Failed to traverse the Oracle Inventory xml $invXMLContent" and return;
  # if this is for dynamic prop, we need only the crshome not the other stuff
  return $confref if $dynProp;
  #get clustername for each cluster
  #all nodes with the same clusterHome are bunched under the same crsHome
  my @nodearray;
  my $nodelist;
  my $clsName;
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
  
      warn "DEBUG:has::Common:hasDiscoverCluster cluster home $ch is not in Inventory file $invXMLContent"
       unless $confref->{$ch}{discover}
       and keys %{$confref->{$ch}{discover}};
  
      # get the emcrsp config information, it tells us if this is a has/cluster target
      has::Common::hasGetClusterConfigEmcrsp($ch);
      if ( $confref->{$ch} and $confref->{$ch}{ocr_type} and $confref->{$ch}{ocr_type} =~ /^has$/ )
      {
       # since this is a has home from discovery get siha name if ocr_type is has
       $clsName =  has::Common::hasGetHasName($ch);
      }
      else
      {
       # since this is a cluster home from discovery use cemutlo  get cluster name if ocr_type is not has
       $clsName =  has::Common::hasCemutloClusterName($ch);
      }
  
      $confref->{$ch}{discover}{cluster_name}=$clsName if $clsName;
      $confref->{$ch}{discover}{crs_home}=$ch;
  
      # build the node list for this crsHome
      if ( $confref->{$ch}{discover}{nodes} )
      {
        for my $node ( keys %{$confref->{$ch}{discover}{nodes}} ) 
        {
          next unless $node;
    
          push @nodearray,$node;
          $nodelist = "$nodelist,$node" if $nodelist;
          $nodelist = $node unless $nodelist;
    
        }
      }
  
      # copy the cluster name,home ,nodelist, nodearray dir back to the root hash
      $confref->{$ch}{discover}{nodelist} = $nodelist if $nodelist;
  
      $confref->{$ch}{discover}{nodearray} = \@nodearray if @nodearray;
  
    }
  }
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      has::Common::clusterConfigCopyVals($confref->{$ch}{discover}, $confref->{$ch})
       if $confref->{$ch}{discover};
    }
  }
  return $confref;
}
#bug fix 9350140
# added function to check id has/cluster inventory exists
#------------------------------------------------------------------------------
# FUNCTION :    hasCheckForHasClusterInventory
#
# DESC
# return a hash of
#    {type =>
#     oracle_home => )
#
# ARGUMENTS
#
# RETURNS
# result list
#  type = has|cluster
#  oracle_home
#------------------------------------------------------------------------------
sub hasCheckForHasClusterInventory(;$)
{
#
#   
#      
#      
#   
#
  sub has_checkinventory_fn(\%\%)
  {
    my ( $elref, $rsref ) = @_;
    return 1 unless $elref->{element} and $elref->{element} =~ /^(HOME)$/i;
    return 1
     unless $elref->{attrs} and $elref->{attrs}{CRS} and $elref->{attrs}{CRS} =~ /^true$/i;
    return 1
     if $elref->{attrs} and $elref->{attrs}{remote} and $elref->{attrs}{remote} =~ /^true$/i;
    return 1
     if $elref->{attrs} and $elref->{attrs}{REMOTE} and $elref->{attrs}{REMOTE} =~ /^true$/i;
    warn "WARN:has::Common::hasCheckForHasClusterInventory LOC is NULL for CRS in Oracle Inventory file"
    and return 1 unless $elref->{attrs} and $elref->{attrs}{LOC};
    # save oracle homes, add nodes later on
    my $ch = $elref->{attrs}{LOC};
    $rsref->{$ch}{discover}{crs_home}=$ch;
    my $idx = $elref->{attrs}{IDX} if exists $elref->{attrs}{IDX};
    $idx = 0 unless defined $idx;
    $idx++ if $idx =~ /\d+/;
    $rsref->{$ch}{discover}{IDX}=$idx;
    $rsref->{$ch}{discover}{IDX}=-1 unless $rsref->{$ch}{discover}{IDX};
    $rsref->{$ch}{discover}{TYPE}=$elref->{attrs}{TYPE} if $elref->{attrs}{TYPE};
    $rsref->{$ch}{discover}{TYPE}=0 unless $rsref->{$ch}{discover}{TYPE};
    # get NODE NAME from children
    return 1 unless  $elref->{children} and @{$elref->{children}};
    for my $nodeListRef ( @{$elref->{children}} )
    {
      next unless $nodeListRef and $nodeListRef->{element} and $nodeListRef->{element} =~ /^NODE_LIST$/i;
      next unless $nodeListRef->{children} and @{$nodeListRef->{children}};
      for my $noderef ( @{$nodeListRef->{children}} )
      {
         next unless $noderef and $noderef->{element} and $noderef->{element} =~ /^NODE$/i;
         warn "DEBUG:has::Common::hasCheckForHasClusterInventory NODE name is null in Oracle Inventory file"
          and next unless $noderef->{attrs} and $noderef->{attrs}{NAME};
         $rsref->{$ch}{discover}{nodes}{$noderef->{attrs}{NAME}}=1;
      }
      last;
    }
    return 1;
  }
  my ( $dynProp ) = @_;
  my $invPath;
  my $invXMLContent;
  my %xmlVar;
  my $hasclcheckref;
  my $confref = has::Common::getClusterCacheRef();
  # if already discovered returned the cached values
  if ( $confref and ref($confref) and keys %{$confref} )
  {
     for my $ch ( keys %{$confref} )
     {
        return $confref if $confref->{$ch}{discover};
     }
  }
  #bugfix 7692044
  my $os = has::Common::get_osType();
  my $scriptsDir = $ENV{EM_SCRIPTS_DIR} if $ENV{EM_SCRIPTS_DIR};
  my $perlBin = $ENV{EM_PERLBIN_DIR} if $ENV{EM_PERLBIN_DIR};
  if ( $os and $os =~ /^WIN$/ )
  {
   #bugfix 8988446
   my $perlexe;
   if ( $perlBin )
   {
     $perlexe = catfile($perlBin,'perl');
   }
   if ( $scriptsDir )
   {
     $scriptsDir =  catfile($scriptsDir,'has');
   }
   if ( not $scriptsDir )
   {
       $scriptsDir = $FindBin::Bin if $FindBin::Bin;
   }
   if ( not $perlexe )
   {
       $perlexe = $Config{perlpath} if $Config{perlpath};
   }
   if ( $scriptsDir and $perlexe )
   { 
     my $has_metrics_script =  catfile($scriptsDir,'has_getinvdir.pl');
     $invPath = `$perlexe $has_metrics_script`;
     chomp $invPath if $invPath;
   }
      
  }
  else
  {
    $invPath= has::Common::getInventoryXmlPath();
  }
  warn "DEBUG:has::Common:hasCheckForHasClusterInventory Failed to find Oracle Inventory File " and return unless $invPath;
  warn "DEBUG:has::Common:hasCheckForHasClusterInventory Failed to read Oracle Inventory File $invPath" 
   and return unless has::Common::hasIsReadable($invPath);
  $invXMLContent = has::Common::hasReturnFileContents($invPath);
  warn "DEBUG:has::Common:hasCheckForHasClusterInventory Oracle Inventory file $invPath has no content\n" 
   and return unless $invXMLContent;
  %xmlVar =  has::Common::parse_xml($invXMLContent);
  has::Common::traverse_xml(\%xmlVar,\&has::Common::has_handle_error,\&has_checkinventory_fn,$confref) 
    or warn "DEBUG:has::Common::hasCheckForHasClusterInventory Failed to traverse the Oracle Inventory xml $invXMLContent" and return;
  # if this is for dynamic prop, we need only the crshome not the other stuff
  #return $confref if $dynProp;
  #get clustername for each cluster
  #all nodes with the same clusterHome are bunched under the same crsHome
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
  
      warn "DEBUG:has::Common:hasCheckForHasClusterInventory cluster home $ch is not in Inventory file $invXMLContent"
       unless $confref->{$ch}{discover} and keys %{$confref->{$ch}{discover}};
  
      # get the emcrsp config information, it tells us if this is a has/cluster target
      has::Common::hasGetClusterConfigEmcrsp($ch);
      $hasclcheckref->{oracle_home}=$ch;
      if ( $confref->{$ch} and $confref->{$ch}{ocr_type} and $confref->{$ch}{ocr_type} =~ /^has$/ )
      {
        $hasclcheckref->{type}='has';
      }
      else
      {
        $hasclcheckref->{type}='cluster';
      }
 
    }
  }
  return $hasclcheckref;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetEntityInformation
#
# DESC
# return the required entity information for the cluster 
#
# ARGUMENTS
#  ref to the metadata to be used to extract data
#  crsHome if known
#  fn pointer or null if you want to use the default fn
#
#  meta should have
#   ->{cmd}
#   ->{element_name}
#   ->{attrs}{[attr_name]}
#   ->{name_value}{[name]}
# RETURN
#  ref to the hash to be filled in with values
#------------------------------------------------------------------------------
sub hasGetEntityInformation($;$$)
{
  # name : hasGetEntityInformation_fn
  # desc : it filters our the required information
  #
  # arg  :
  #  ref to xml element to be filtered
  #  ref to the result hass entities  array
  #  ref to the metadata hash to be used to extract data
  # 
  sub hasGetEntityInformation_fn($$$)
  {
    my ( $elref, $rsref, $mtref ) = @_;
    # return unless meta data is provided
    return 1 unless $mtref and $mtref->{element_name};
    return 1 unless $elref->{element} and $elref->{element} =~ /^$mtref->{element_name}$/i;
    # get the name for this entity
    my $thisval;
    if ( $elref->{attrs} )
    {
      for my $hdval ( keys %{$elref->{attrs}} )
      {
        $thisval =  $elref->{attrs}{$hdval} if $hdval =~ /^entity_name$/;
      }
    }
   $thisval =~ s/^\s+|\s+$//g;
    warn "DEBUG:has::Common:: failed to get entity_name from $elref->{element}" and return 1 unless $thisval;
    # if there are attrs to be read read them
    if ( $mtref->{$mtref->{element_name}}{attrs} )
    {
       for my $hdval ( keys %{$elref->{attrs}} )
       {
         $hdval =~ s/^\s+|\s+$//g;
         next unless $hdval;
         next unless $mtref->{$mtref->{element_name}}{attrs}{$hdval};
 
         $rsref->{$thisval}{$hdval} = $elref->{attrs}{$hdval} if $elref->{attrs}{$hdval};
         $rsref->{$thisval}{$hdval} =~ s/^\s+|\s+$//g if $rsref->{$thisval}{$hdval};
        }
    }
    # return unless there are name value paris to be read from attributes
    return 1 unless $mtref->{$mtref->{element_name}}{name_value};
    # read all name=value pairs from the attributes of the node
    for my $elref1 (   @{$elref->{children}} )
    {
      next unless $elref1->{element} =~ /^attributes$/;
      for my $elref2 (   @{$elref1->{children}} )
      {
        next unless $elref2->{element} =~ /^attribute$/;
        my $name;
        my $value;
        for my $elref3 (   @{$elref2->{children}} )
        {
          next unless $elref3->{element} =~ /^(name|value)$/;
          $name = $elref3->{name} if $elref3->{element} =~ /^name$/;
          $value = $elref3->{name} if $elref3->{element} =~ /^value$/;
          $name =~ s/^\s+|\s+$//g if $name;
          $value =~ s/^\s+|\s+$//g if $value;
        }
        next unless $name;
        # if this name is not required skip it
        next unless $mtref->{$mtref->{element_name}}{name_value}{$name};
        $value = '' unless $value;
        $rsref->{$thisval}{$name}=$value;
      }
    }
    return 1;
  }
  my ( $metaref,$crsHome,$fn_ptr ) = @_;
  $fn_ptr = \&hasGetEntityInformation_fn unless $fn_ptr;
  my $reslistref;
  return 1 unless $metaref and $metaref->{cmd};
  # no cluster home is passed then 
  # get the cluster home from the env 
  #  and if not available or no emcrsp then discover one
  if ( not $crsHome )
  {
    $crsHome = has::Common::hasGetCRSHome();
    # get the cluster home by discovery
    if ( not $crsHome or not has::Common::hasCheckForEmcrsp($crsHome) )
    {
      $crsHome = has::Common::hasGetEnvCRSHome();
    }
  }
  return $reslistref 
   and warn "DEBUG:has::Common::hasGetEntityInformation:Cluster home is not passed or not set in env" 
   unless $crsHome;
  return $reslistref 
   and warn "DEBUG:has::Common::hasGetEntityInformation:binary emcrsp is not present in $crsHome" 
   unless has::Common::hasCheckForEmcrsp($crsHome);
  has::Common::hasSetCRSEnv($crsHome);
  my $cmdresults = has::Common::runsystemcommand($metaref->{cmd});
  has::Common::hasRestoreCRSEnv();
  $cmdresults = has::Common::hasCheckAndReturnEmcrspResults($cmdresults);
  warn "DEBUG:has::Common::hasGetEntityInformation Failed to get results from command - $metaref->{cmd}" 
   and return $reslistref unless $cmdresults;
  my  %xmlvar =  has::Common::parse_xml($cmdresults);
  my %resulthash;
  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,$fn_ptr,\%resulthash,$metaref);
  return \%resulthash;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasCemutlsClusterName
#
# DESC
#  get the clustername from cemutls for non oracle clusters
#
# ARGUMENTS
#   crsHome for vendor or EMDROOR
# RETURN
#  cluster name
#------------------------------------------------------------------------------
sub hasCemutlsClusterName($)
{
  my ($dir) = @_;
 
  # bug 4667678
  # TBD is solaris 64 bit run the 64 bit cemutls
  my $cemutls;
  my %cemutls_command_args = (exit_failure_list => [()]);
      
  if (has::Common::get_osType() eq "SOL")
  { 
   # bug fix 9255116
   if ( has::Common::hasDoesFileExist('/opt/ORCLcluster/lib/libskgxn2.so') )
   { 
    # bug fix 8459125
    my $o = has::Common::runsystemcommand("file /opt/ORCLcluster/lib/libskgxn2.so 2>&1",'',\%cemutls_command_args);
    if ($o and $o =~ /64-bit/)
    {
       my $cemutls64path;
       $cemutls64path = catfile($dir,'bin') if $dir;
       $cemutls64path = catfile($cemutls64path,'cemutls64') if $cemutls64path;
       stat $cemutls64path if $cemutls64path;
       $cemutls = "cemutls64" if $cemutls64path and -e $cemutls64path;
    }
   }
  }
  $cemutls = "cemutls" unless $cemutls;
  %cemutls_command_args = (exit_failure_list => [()]);
  has::Common::hasSetCRSEnv($dir);
  my $vo = has::Common::runsystemcommand("$cemutls -w 2>&1",'',\%cemutls_command_args);
  has::Common::hasRestoreCRSEnv();
  
  warn "DEBUG:has::Common::hasCemutlsClusterName Error getting cluster name from $cemutls" 
    and return
    if $cemutls_command_args{command_return_status};
  chomp($vo) if $vo;
  $vo =~ s/\n/ /g if $vo;
  $vo =~ s/^\s+|\s+$// if $vo;
  #return has::Common::appendNodeNameToClusterName($vo,$dir) if $vo and $dir;
  return $vo if $vo;
  warn "DEBUG:has::Common::hasCemutlsClusterName Failed to get cluster name from $cemutls" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    appendNodeNameToClusterName
#
# DESC
#  suffix nodeName from nodeList if clusterName is crs
#
# ARGUMENTS
#   custerName,crsHome 
# RETURN
#  cluster name
#------------------------------------------------------------------------------
sub appendNodeNameToClusterName($$;)
{
  my ( $clusterName,$crsHome) = @_;
  warn "WARN:has::Common::appendNodeNameToClusterName cluster name passed is NULL" and return $clusterName unless $clusterName;
  warn "WARN:has::Common::appendNodeNameToClusterName crsHome passed is NULL" and return $clusterName unless $crsHome;
  return $clusterName unless $clusterName =~ /^crs$/;
  my $confref = has::Common::hasGetClusterNodeListPre11g($crsHome);
  warn "DEBUG:has::Common::appendNodeNameToClusterName Failed to get nodearray from hasGetClusterNodeListPre11g for $crsHome" 
   and return $clusterName unless ( $confref and ref($confref) and ref($confref) =~ /HASH/ );
  my $nodeListRef;
  if ( $confref->{$crsHome} and $confref->{$crsHome}{node} and $confref->{$crsHome}{node}{nodearray} )
  {
    $nodeListRef = $confref->{$crsHome}{node}{nodearray};
  }
  elsif ( $confref->{$crsHome} and $confref->{$crsHome}{discover} and $confref->{$crsHome}{discover}{nodearray} )
  {
    $nodeListRef = $confref->{$crsHome}{discover}{nodearray};
  }
  else
  {
    for my $ch ( keys %{$confref} )
    {
      next unless $confref->{$ch} and $confref->{$ch}{node} and $confref->{$ch}{node}{nodearray};
      $nodeListRef = $confref->{$ch}{node}{nodearray};
    }
  }
   
  if ($nodeListRef and ref($nodeListRef) and ref($nodeListRef) =~ /ARRAY/i )
  {
    for my $nodeName ( sort {$a cmp $b} @$nodeListRef )
    {
        next unless $nodeName;
        $nodeName =~ s/^\s+|\s+$//;
        next unless $nodeName;
        $clusterName = "crs_$nodeName";
        last;
    }
  }
  
  return $clusterName;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasDoesCommandExist
#
# DESC
#  check if a command exists
#
# ARGUMENTS
#   crsHome
# RETURN
#  cluster name
#------------------------------------------------------------------------------
sub hasDoesCommandExist($$;)
{
   my ( $cmd,$homedir) = @_;
   my $fullcmd;
    for my $ext ( ( 'exe', 'bat' , undef) )
    {
      my $ecmd;
      $ecmd = "$cmd\.$ext" if $ext;
      $ecmd = $cmd unless $ecmd;
      $fullcmd = catfile($homedir,'bin',$ecmd);
      stat $fullcmd;
      last if has::Common::hasIsReadable($fullcmd);
      undef $fullcmd;
    }
    return 1 if $fullcmd;
    return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasCemutloClusterName
#
# DESC
#  get the clustername from cemutlo for oracle clusters
#
# ARGUMENTS
#   crsHome 
# RETURN
#  cluster name
#------------------------------------------------------------------------------
sub hasCemutloClusterName($)
{
  my ( $crsHome) =  @_;
  my $clusterName;
  my $ocrtype = has::Common::hasGetOcrType($crsHome);
  if ( $ocrtype and $ocrtype =~ /^has$/ )
  {
   $clusterName = has::Common::hasGetHasName($crsHome);
  }
  else
  {
   if ( $crsHome )
   {
    if ( not has::Common::hasDoesCommandExist('cemutlo',$crsHome) )
    {
      warn "DEBUG:has::Common::hasCemutloClusterName Binary cemutlo does not exist" and return;
    }
   }
    my %cemutlo_command_args = (exit_failure_list => [()]);
  
    has::Common::hasSetCRSEnv($crsHome);
    $clusterName = has::Common::runsystemcommand('cemutlo -n 2>&1','',\%cemutlo_command_args);
    has::Common::hasRestoreCRSEnv();
    
    warn "DEBUG:has::Common::hasCemutloClusterName Error getting cluster name from cemutlo" 
     and return
      if $cemutlo_command_args{command_return_status};
  
    chomp($clusterName) if $clusterName;
    $clusterName =~ s/\n/ /g if $clusterName;
    $clusterName =~ s/^\s+|\s+$// if $clusterName;
    if ( $clusterName )
    {
      if ( $clusterName =~ /[^a-zA-Z0-9\-_]+/ )
      {
        # crs may be shutdown, to avoid descrepency between discovery on different nodes, do not 
        # return a clustername
        #$clusterName = 'crs';
        undef $clusterName;
      }
    }
  }
  #return has::Common::appendNodeNameToClusterName($clusterName,$crsHome) if $clusterName and $crsHome;
  return $clusterName if $clusterName;
  warn "DEBUG:has::Common::hasCemutloClusterName Failed to get cluster name from cemutlo" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasSearchClusterName
#
# DESC
#  get the clustername for the cluster directory passed
#  if not look up inventory and get the cluster name from the 
#    cluster installs therte
#  if not use EMDROOT to get clustername using cemutls
#
# ARGUMENTS
#   crsHome for vendor or EMDROOR
# RETURN
#  cluster name 
#  list of clustername, clusterhome,isVendor used to get the cluster name
# isVendor is 0 for oracle ( cemutlo )
#             1 for vendor ( cemutls )
#------------------------------------------------------------------------------
sub hasSearchClusterName(;$)
{
   my ( $crsHome ) = @_;
   my $clsName;
   my $clsdiscref;
   if ( $crsHome )
   {
      $clsName = has::Common::hasCemutloClusterName($crsHome); 
   }
   if ( $clsName)
   {
     return wantarray? ($clsName,$crsHome,0):$clsName;
   }
 
   $clsdiscref = has::Common::hasDiscoverCluster();
   if ( $clsdiscref and keys %{$clsdiscref} )
   {
      for my $ch ( keys %{$clsdiscref} )
      {
        if ( not $clsdiscref->{$ch}{cluster_name} )
        {
           $clsName = has::Common::hasCemutloClusterName($ch);
           $clsdiscref->{$ch}{cluster_name} = $clsName if $clsName;
        }
        else
        {
          $clsName = $clsdiscref->{$ch}{cluster_name};
        }
        
        if ( $clsName )
        {
           $crsHome = $ch;
           last;
        }
      }
   }
   if ( $clsName)
   {
     return wantarray? ($clsName,$crsHome,0):$clsName;
   }
   if ( $ENV{EMDROOT} ) 
   {
     $clsName = has::Common::hasCemutlsClusterName($ENV{EMDROOT});
     undef $crsHome;
   }
   elsif ( $crsHome )
   {
     $clsName = has::Common::hasCemutlsClusterName($crsHome);
   }
   else
   {
     warn "WARN:has::Common::hasSearchClusterName: EMDROOT is not passed to get the cluster name" 
   }
   if ($clsName)
   {
     return wantarray? ($clsName,$crsHome,1):$clsName;
   }
   warn "WARN:has::Common::hasSearchClusterName:Failed to get clustername \n" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetHasName
#
# DESC
#  generate a name for has ( has_ )
#
# ARGUMENTS
#   crsHome for vendor or EMDROOR
# RETURN
#  cluster name 
#------------------------------------------------------------------------------
sub hasGetHasName(;$$)
{
   my ( $crsHome,$dynProp ) = @_;
   my $clsName;
  
   my $hostName = has::Common::hasGetLocalHostName(); 
   $clsName = "has_$hostName" if $hostName;
   $clsName = "has_" unless $clsName;
   my $confref = has::Common::getClusterCacheRef();
   $confref->{$crsHome}{cluster_name}=$clsName if $crsHome;
   return wantarray? ($clsName,$crsHome,0):$clsName;
 
}
#------------------------------------------------------------------------------
# FUNCTION :    hasIsVendorClusterware
#
# DESC
#  get the clustername from cemutls for non oracle clusters
#
# ARGUMENTS
#   crsHome for vendor or EMDROOR
# RETURN
#  cluster name
#------------------------------------------------------------------------------
sub hasIsVendorClusterware($)
{
  my ( $dir) = @_;
 
  # bug 4667678
  # TBD is solaris 64 bit run the 64 bit cemutls
  my $cemutls;
  my %cemutls_command_args = (exit_failure_list => [()]);
  
  if (has::Common::get_osType() eq "SOL")
  {
   # bug fix 9255116
   if ( has::Common::hasDoesFileExist('/opt/ORCLcluster/lib/libskgxn2.so') )
   {
    # bug fix 8459125
    my $o = has::Common::runsystemcommand("file /opt/ORCLcluster/lib/libskgxn2.so 2>&1",'',\%cemutls_command_args);
    if ($o and $o =~ /64-bit/)
    {
       my $cemutls64path;
       $cemutls64path = catfile($dir,'bin') if $dir;
       $cemutls64path = catfile($cemutls64path,'cemutls64') if $cemutls64path;
       stat $cemutls64path if $cemutls64path;
       $cemutls = "cemutls64" if $cemutls64path and -e $cemutls64path;
    }
   }
  }
  $cemutls = "cemutls" unless $cemutls;
  %cemutls_command_args = (exit_failure_list => [()]);
  has::Common::hasSetCRSEnv($dir);
  my $vo = has::Common::runsystemcommand("$cemutls -w 2>&1",'',\%cemutls_command_args);
  has::Common::hasRestoreCRSEnv();
  
  chomp($vo) if $vo;
  $vo =~ s/\n/ /g if $vo;
  $vo =~ s/^\s+|\s+$// if $vo;
  # if cemutls fails then this is not a vendor cluster
  if ( $cemutls_command_args{command_return_status} )
  {
    return wantarray?($vo,$cemutls_command_args{command_return_status}):0;
  }
  # cemutls is successful do this is a vendor cluster
  return wantarray?($vo,$cemutls_command_args{command_return_status}) :1;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterConfigPre11g
#
# DESC
# return the config information for the cluster emcrsp
#
# ARGUMENTS
#  crsHome if known
#  dynamicProperty - not null valud if function is called for dynamic prop 
#   computation
# RETURN
#  ref to the hash to be filled in with values
#------------------------------------------------------------------------------
sub hasGetClusterConfigPre11g(;$$)
{
  my ( $crsHome,$dynProp ) = @_;
  undef $crsHome if $crsHome and not has::Common::hasIsReadable($crsHome);
  # no cluster home is passed then 
  # get the cluster home from the env 
  #  and if not available or no emcrsp then discover one
  if ( not $crsHome )
  {
    # get the cluster home by discovery
    $crsHome = has::Common::hasGetCRSHome();
    if ( not $crsHome )
    {
      $crsHome = has::Common::hasGetEnvCRSHome();
    }
  }
  warn "WARN:has::Common::hasGetClusterConfigPre11g:Cluster home is not passed or not set in env" unless $crsHome;
  my $isVendorCW = 1;
  my $version;
  my $patch;
  my $errMsg;
  my $vendor;
  my $clsName;
  my $o;
  my $vo;
  my $CRSVersion;
  my $cmd_status;
  
  #There is a CRS home in this cluster
  my $nCrsHome;
  # is this a has or cluster
  my $ocrtype = has::Common::hasGetOcrType($crsHome,$dynProp);
  # I added this check as this will result in timeout of dynamic property comps
  # if crs home is known then do nothins
  # we should add a flag to indicate if this is called from dynamic prop or a metric
  # TBD
  if ( $dynProp )
  {
    ($clsName,$nCrsHome,$isVendorCW) = has::Common::hasSearchClusterName($crsHome);
  
    if ( $nCrsHome and $crsHome and $crsHome ne $nCrsHome )
    {
      warn "WARN:has::Common::hasGetClusterConfigPre11g the cluster home $nCrsHome is different from $crsHome";
      $crsHome=$nCrsHome;
    }
    elsif ( $nCrsHome and not $crsHome )
    {
      warn "WARN:has::Common::hasGetClusterConfigPre11g the cluster home is $nCrsHome ";
      $crsHome=$nCrsHome;
    }
    elsif ( $clsName and not $nCrsHome )
    {
      warn "WARN:has::Common::hasGetClusterConfigPre11g: No home found for cluster $clsName ,different from $crsHome";
    }
  }
 
  if ( $crsHome and $crsHome !~ /#CRS_HOME#/ )
  {
  
      ( $vo,$cmd_status) = has::Common::hasIsVendorClusterware($crsHome);
      # No vendor clusterware in use if cemutls fails
      if ( $cmd_status )
      {
        $isVendorCW = 0;
      }
      else
      { 
        $isVendorCW = 1;
      }
      # use cemutlo to get info on oracle clusterware
      my %command_args = (exit_failure_list => [()]);
     
      if ( not $ocrtype or $ocrtype !~ /^has$/ )
      {
       has::Common::hasSetCRSEnv($crsHome);
       $o = has::Common::runsystemcommand('cemutlo -w 2>&1','',\%command_args);
       has::Common::hasRestoreCRSEnv();
  
       if ( $command_args{command_return_status} )
       {
        warn("ERROR:has:Common::hasGetClusterConfigPre11g:Failed executing command cemutlo -w");
       }
       else
       {
         chomp($o) if $o;
         $o =~ s/\n/ /g if $o;
         $o =~ s/^\s+|\s+$// if $o;
    
         warn("ERROR:has:Common::hasGetClusterConfigPre11g:Failed to get any results from cemutlo -w") unless $o;
    
         warn("ERROR:has:Common::hasGetClusterConfigPre11g:Unknown format of the output from cemutlo -w : $o")
          if $o and $o !~ /(.*)\:(.*)\:(.*)/;
    
         # use the output from cemutls if output from cemutlo does not match
         $o = $vo if $vo and not $o and $o  !~ /(.*)\:(.*)\:(.*)/;
    
         ($version,$patch,$vendor) = ($o =~ /(.*)\:(.*)\:(.*)/) if $o and $o =~ /(.*)\:(.*)\:(.*)/;
    
         $version = "$version.$patch" if $version and $patch;
       }
      }
  
      #Active CRS Version
      %command_args = (exit_failure_list => [()]); 
      has::Common::hasSetCRSEnv($crsHome);
      if ( $ocrtype and $ocrtype =~ /^has$/ )
      {
       $o = has::Common::runsystemcommand('crsctl query has releaseversion 2>&1','',\%command_args);
      }
      else
      {
       $o = has::Common::runsystemcommand('crsctl query crs activeversion 2>&1','',\%command_args);
      }
      has::Common::hasRestoreCRSEnv();
  
      warn("ERROR:has:Common::hasGetClusterConfigPre11g:Failed executing command crsctl query crs activeversion") 
      if $command_args{command_return_status};
  
      chomp($o) if $o;
      $o =~ s/\n/ /g if $o;
      $o =~ s/^\s+|\s+$// if $o;
  
      warn("ERROR:has:Common::hasGetClusterConfigPre11g:Failed to get any results from crsctl query crs activeversion")
       unless $o;
  
      if  (  $command_args{command_return_status} )
      {
         $CRSVersion = '';
      }
      elsif ( $o )
      {
         warn("ERROR:has:Common::hasGetClusterConfigPre11g:Unknown format of the output from crsctl query crs activeversion : $o")
           if $o and $o !~ /\[([\d|\.]+)\]/;
  
         ($CRSVersion) = ( $o =~ /\[([\d|\.]+)\]/ ) if $o and $o =~ /\[([\d|\.]+)\]/;
      }
  
      # 10.1 crsctl doesn't return the version. 
      $CRSVersion = '10.1' if not $CRSVersion and $version and $version =~ /^1.1$/;
  
  }
  else
  {
      #No CRS is known to the cluster
      $crsHome = $ENV{EMDROOT} if $ENV{EMDROOT};
  
      ($o,$cmd_status) = has::Common::hasIsVendorClusterware($crsHome);
      # cemutls -w failed, so this is a oracle cluster 
      if ( $cmd_status )
      {
        warn("WARN:has:Common::hasGetClusterConfigPre11g:Failed executing command cemutls -w as oracle cluster");
      }
      elsif ( $o )
      {
  
        warn("ERROR:has:Common::hasGetClusterConfigPre11g: Unknown format of the output from cemutls -w : $o")
         unless $o =~ /(.*)\:(.*)\:(.*)/;
  
       ($version,$patch,$vendor) = ($o =~ /(.*)\:(.*)\:(.*)/) if $o =~ /(.*)\:(.*)\:(.*)/;
  
      }
      else
      {
        warn("ERROR:has:Common::hasGetClusterConfigPre11g: Vendor cluster, but failed to get results from cemutls -w : "); 
      }
  
      $version = "$version.$patch" if $version and $patch;
  
  }
  $crsHome = '#CRS_HOME#' unless $crsHome;
   my $confref = has::Common::getClusterCacheRef();
  $confref->{$crsHome}{pre11gr2_config}{cluster_name}= $clsName if $clsName;
  $confref->{$crsHome}{pre11gr2_config}{crs_version} = $CRSVersion if $CRSVersion;
  $confref->{$crsHome}{pre11gr2_config}{vendor} = $vendor if defined $vendor;
  $confref->{$crsHome}{pre11gr2_config}{version} = $version if defined $version;
  $confref->{$crsHome}{pre11gr2_config}{isvendorcw} = $isVendorCW if defined $isVendorCW;
  has::Common::clusterConfigCopyVals($confref->{$crsHome}{pre11gr2_config}, $confref->{$crsHome})
   if $crsHome and $confref->{$crsHome}{pre11gr2_config};
  return $confref;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetHasNodeName
#
# DESC
# return the node name for the local node
#
# ARGUMENTS
# crsHome if known
#------------------------------------------------------------------------------
sub hasGetHasNodeName(;$$)
{
  my ( $crsHome ,$dynProp ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{nodename} 
   if $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{nodename};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{nodename} if $confref->{$ch}{nodename};
    }
  }
  $confref = has::Common::hasGetClusterConfigEmcrsp($crsHome);
  return $confref->{$crsHome}{nodename} if $confref and $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{nodename};
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{nodename} if $confref->{$ch}{nodename};
    }
  }
  warn "WARN:has::Common::hasGetHasNodeName Failed to get the node name for the has node, trying to get hostname";
  my $hostName =  has::Common::hasGetLocalHostName();
  if ( $hostName )
  {
     $hostName  =~ s/\..*//;
     $hostName = lc $hostName if $hostName;
  }
  warn "WARN:has::Common::hasGetHasNodeName Failed to get the node name for the has node" and return unless $hostName;
  return $hostName;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetHasReleaseVersion
#
# DESC
#  return the release version for has
#
# ARGUMENTS
#  crsHome if known
#
# RETURN
#  ref to the hash to be filled in with values
#------------------------------------------------------------------------------
sub hasGetHasReleaseVersion(;$$)
{
  my ( $crsHome, $dynProp ) = @_;
  undef $crsHome if $crsHome and not has::Common::hasIsReadable($crsHome);
  # no cluster home is passed then 
  # get the cluster home from the env 
  #  and if not available or no emcrsp then discover one
  if ( not $crsHome )
  {
    # get the cluster home by discovery
    $crsHome = has::Common::hasGetCRSHome();
    if ( not $crsHome )
    {
      $crsHome = has::Common::hasGetEnvCRSHome();
    }
  }
  warn "WARN:has::Common::hasGetHasReleaseVersion:Cluster home is not passed or not set in env" unless $crsHome;
  my $o;
  my $CRSVersion;
  my %command_args;
  
  if ( $crsHome and $crsHome !~ /#CRS_HOME#/ )
  {
  
      #Active CRS Version
      %command_args = (exit_failure_list => [()]); 
      has::Common::hasSetCRSEnv($crsHome);
      $o = has::Common::runsystemcommand('crsctl query has releaseversion 2>&1','',\%command_args);
      has::Common::hasRestoreCRSEnv();
  
      warn("ERROR:has:Common::hasGetHasReleaseVersion:Failed executing command crsctl query has releaseversion") 
      if $command_args{command_return_status};
  
      chomp($o) if $o;
      $o =~ s/\n/ /g if $o;
      $o =~ s/^\s+|\s+$// if $o;
  
      warn("ERROR:has:Common::hasGetHasReleaseVersion:Failed to get any results from crsctl query has releaseversion")
       unless $o;
  
      if  (  $command_args{command_return_status} )
      {
         $CRSVersion = '';
      }
      elsif ( $o )
      {
         warn("ERROR:has:Common::hasGetHasReleaseVersion:Unknown format of the output from crsctl query has releaseversion : $o")
           if $o and $o !~ /\[([\d|\.]+)\]/;
  
         ($CRSVersion) = ( $o =~ /\[([\d|\.]+)\]/ ) if $o and $o =~ /\[([\d|\.]+)\]/;
      }
  
  
      if ( not $CRSVersion )
      {
      
          #Active CRS Version
          %command_args = (exit_failure_list => [()]); 
          has::Common::hasSetCRSEnv($crsHome);
          $o = has::Common::runsystemcommand('crsctl query has softwareversion 2>&1','',\%command_args);
          has::Common::hasRestoreCRSEnv();
      
          warn("ERROR:has:Common::hasGetHasReleaseVersion:Failed executing command crsctl query has softwareversion") 
          if $command_args{command_return_status};
      
          chomp($o) if $o;
          $o =~ s/\n/ /g if $o;
          $o =~ s/^\s+|\s+$// if $o;
      
          warn("ERROR:has:Common::hasGetHasReleaseVersion:Failed to get any results from crsctl query has softwareversion")
           unless $o;
      
          if  (  $command_args{command_return_status} )
          {
             $CRSVersion = '';
          }
          elsif ( $o )
          {
             warn("ERROR:has:Common::hasGetHasReleaseVersion:Unknown format of the output from crsctl query has softwareversion : $o")
               if $o and $o !~ /\[([\d|\.]+)\]/;
      
             ($CRSVersion) = ( $o =~ /\[([\d|\.]+)\]/ ) if $o and $o =~ /\[([\d|\.]+)\]/;
          }
      
      }
  }
  $crsHome = '#CRS_HOME#' unless $crsHome;
   my $confref = has::Common::getClusterCacheRef();
  $confref->{$crsHome}{pre11gr2_config}{crs_version} = $CRSVersion if $CRSVersion;
  $confref->{$crsHome}{pre11gr2_config}{version} = $CRSVersion if defined $CRSVersion;
  has::Common::clusterConfigCopyVals($confref->{$crsHome}{pre11gr2_config}, $confref->{$crsHome})
   if $crsHome and $confref->{$crsHome}{pre11gr2_config};
  $CRSVersion = '' unless $CRSVersion;
  return $CRSVersion if $CRSVersion;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterGetNodeNamePre11g
#
# DESC
# return the node name for the local node using olsnode/lsodes
#
# ARGUMENTS
# crsHome if known
#
# RETURN
# ref to hash
#
#------------------------------------------------------------------------------
sub hasGetClusterGetNodeNamePre11g(;$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $nodename;
  $crsHome =~ s/^\s+|\s+$//g if $crsHome;
  my $confref = has::Common::getClusterCacheRef();
  my $ocrType = has::Common::hasGetOcrType($crsHome,$dynProp);
  if ( $ocrType and $ocrType =~ /^has$/ )
  {
    return $confref;
  }
  # the order of search directories for olsnode/lsnodes
  my %olsnodeorder;
  my $ncrsHome;
  if ( $crsHome )
  {
    $olsnodeorder{0}{path}=$crsHome;
    $olsnodeorder{0}{cmd}='olsnodes';
  }
  my $discCrsHome = has::Common::hasGetCRSHome();
  if ( $discCrsHome )
  {
    $olsnodeorder{1}{path}=$discCrsHome;
    $olsnodeorder{1}{cmd}='olsnodes';
  }
  $olsnodeorder{2}{path}='CRS_HOME';
  $olsnodeorder{2}{cmd}='olsnodes';
  $olsnodeorder{3}{path}='EM_CRS_HOME';
  $olsnodeorder{3}{cmd}='olsnodes';
  $olsnodeorder{4}{path}='ORACLE_HOME';
  $olsnodeorder{4}{cmd}='olsnodes';
  $olsnodeorder{5}{path}='EMDROOT';
  $olsnodeorder{5}{cmd}='lsnodes';
  for my $order ( sort  {$a <=> $b} keys %olsnodeorder )
  {
    my $path;
    my $cmd;
    next unless $olsnodeorder{$order} and $olsnodeorder{$order}{path} and $olsnodeorder{$order}{cmd};
    $path = $olsnodeorder{$order}{path};
    $path = $ENV{$olsnodeorder{$order}{path}} if $ENV{$olsnodeorder{$order}{path}};
    next unless $path;
    next if $path =~ /#CRS_HOME#|EMDROOT|CRS_HOME|EM_CRS_HOME|ORACLE_HOME/;
    next unless has::Common::hasIsReadable($path);
    $cmd = $olsnodeorder{$order}{cmd};
    #get the list of nodes using olsnodes
    my %command_args = (exit_failure_list => [()]);
    my $fullcmd;
    for my $ext ( ( 'exe', 'bat' , undef) )
    {
      my $ecmd;
      $ecmd = "$cmd\.$ext" if $ext;
      $ecmd = $cmd unless $ecmd;
      $fullcmd = catfile($path,'bin',$ecmd);
      stat $fullcmd;
      last if has::Common::hasIsReadable($fullcmd);
      undef $fullcmd;
   
    }
    next unless $fullcmd;
    # bug 4667678
    has::Common::hasSetCRSEnv($path);
    #Bug fix#9117595
    $nodename = has::Common::runsystemcommand($fullcmd.' -l 2>&1','',\%command_args);
    has::Common::hasRestoreCRSEnv();
    warn("DEBUG:has::Common::hasGetClusterGetNodeNamePre11g:Failed executing command **$fullcmd") 
     and next if $command_args{command_return_status};
   
    # remove new line
    chomp $nodename if $nodename;
   if ( $nodename )
   {
    $ncrsHome = $path unless $path =~ /EMDROOT/;
    warn("DEBUG:has::Common::hasGetClusterGetNodeNamePre11g:CRS home is: $ncrsHome") and last if $ncrsHome;
    warn("DEBUG:has::Common::hasGetClusterGetNodeNamePre11g:No CRS home is: using $fullcmd to get node name, possibly vendor clusterware") and last unless $ncrsHome;
   }
  }
  $confref = has::Common::getClusterCacheRef();
  warn("WARN:has::Common::hasGetClusterGetNodeNamePre11g:Failed to get nodename from cluster binaries olsnodes/lsnodes") 
   unless $nodename;
  return $confref unless $nodename;
  $crsHome = $ncrsHome if $ncrsHome and not $crsHome;
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      $confref->{$ch}{pre11gr2_config}{nodename} = $nodename unless $confref->{$ch}{pre11gr2_config}{nodename};
    }
   
    if ( $crsHome )
    {
     $confref->{$crsHome}{pre11gr2_config}{nodename} = $nodename if $nodename;
     $confref->{$crsHome}{pre11gr2_config}{crs_home} = $crsHome unless $confref->{$crsHome}{pre11gr2_config}{crs_home};
    }
  }
  else
  {
   $crsHome = '#CRS_HOME#' unless $crsHome;
   $confref->{$crsHome}{pre11gr2_config}{nodename} = $nodename;
   $confref->{$crsHome}{pre11gr2_config}{crs_home} = $crsHome unless $confref->{$crsHome}{pre11gr2_config}{crs_home};
  }
  has::Common::clusterConfigCopyVals($confref->{$crsHome}{pre11gr2_config}, $confref->{$crsHome})
   if $crsHome and $confref->{$crsHome}{pre11gr2_config};
  return $confref;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterNodeListPre11g
#
# DESC
# return a hash of
#    {nodelist =>
#     nodearray => )
#     crs_home =>
#  of nodes in a cluster
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# hash result list
#------------------------------------------------------------------------------
sub hasGetClusterNodeListPre11g(;$)
{
  my ($crsHome ) = @_; # CRSHome
  my $nodes;
  my @nodes;
  my $ncrsHome;
  my $confref = has::Common::getClusterCacheRef();
  if (  $crsHome 
        and $confref and ref($confref) and keys %{$confref}
        and $confref->{$crsHome}
        and $confref->{$crsHome}{node} 
        and $confref->{$crsHome}{node}{nodearray} )
  {
    return $confref;
  }
  $crsHome =~ s/^\s+|\s+$//g if $crsHome;
  undef $crsHome if $crsHome and $crsHome =~ /^\s*$/;
  # the order of search directories for olsnode/lsnodes
  my %olsnodeorder;
  if ( $crsHome )
  {
    $olsnodeorder{0}{path}=$crsHome;
    $olsnodeorder{0}{cmd}='olsnodes';
  }
  my $discCrsHome = has::Common::hasGetCRSHome();
  if ( $discCrsHome )
  {
    $olsnodeorder{1}{path}=$discCrsHome;
    $olsnodeorder{1}{cmd}='olsnodes';
  }
  $olsnodeorder{2}{path}='CRS_HOME';
  $olsnodeorder{2}{cmd}='olsnodes';
  $olsnodeorder{3}{path}='EM_CRS_HOME';
  $olsnodeorder{3}{cmd}='olsnodes';
  $olsnodeorder{4}{path}='ORACLE_HOME';
  $olsnodeorder{4}{cmd}='olsnodes';
  $olsnodeorder{5}{path}='EMDROOT';
  $olsnodeorder{5}{cmd}='lsnodes';
  for my $order ( sort  {$a <=> $b} keys %olsnodeorder )
  {
    my $path;
    my $cmd;
    undef $ncrsHome if $ncrsHome;
    undef $nodes if $nodes;
    next unless $olsnodeorder{$order} and $olsnodeorder{$order}{path} and $olsnodeorder{$order}{cmd};
    $path = $olsnodeorder{$order}{path};
    $path = $ENV{$olsnodeorder{$order}{path}} if $ENV{$olsnodeorder{$order}{path}};
    next unless $path;
    next if $path =~ /#CRS_HOME#|EM_CRS_HOME|CRS_HOME|ORACLE_HOME|EMDROOT/;
    next unless has::Common::hasIsReadable($path);
    $cmd = $olsnodeorder{$order}{cmd};
    #get the list of nodes using olsnodes
    my %command_args = (exit_failure_list => [()]);
    my $fullcmd;
    for my $ext ( ( 'exe', 'bat' , undef) )
    {
      my $ecmd;
      $ecmd = "$cmd\.$ext" if $ext;
      $ecmd = $cmd unless $ecmd;
      $fullcmd = catfile($path,'bin',$ecmd);
      stat $fullcmd;
      last if has::Common::hasIsReadable($fullcmd);
      undef $fullcmd;
   
    }
    next unless $fullcmd;
    has::Common::hasSetCRSEnv($path);
    $nodes = has::Common::runsystemcommand($fullcmd.' 2>&1','',\%command_args);
    has::Common::hasRestoreCRSEnv();
    warn("DEBUG:has::Common::hasGetClusterNodeListPre11g:Failed executing command **$fullcmd") 
     and next if $command_args{command_return_status};
    if ( $nodes )
    {
      # bug 4667678
      $ncrsHome = $path unless $path =~ /EMDROOT/;
      warn("DEBUG:has::Common::hasGetClusterNodeListPre11g:Getting node list from $cmd at $path") and last;
    } 
  }
  warn("DEBUG:has::Common::hasGetClusterNodeListPre11g:Failed to get nodelist from cluster binaries olsnodes and cluvfy") 
    unless $nodes;
  # try to get the information from discovery if olsnodes fails if cluster is down
  if ( 
        $crsHome
        and $confref and ref($confref) and keys %{$confref}
        and $confref->{$crsHome}
        and $confref->{$crsHome}{discover}
        and $confref->{$crsHome}{discover}{nodelist}
  )
  {
    $nodes = $confref->{$crsHome}{discover}{nodelist};
  }
  warn("WARN:has::Common::hasGetClusterNodeListPre11g:Failed to get nodelist from cluster binaries olsnodes and discovery") 
    and return unless $nodes;
  chomp($nodes) if $nodes;
  $nodes=~ s/^\s+|\s+$//g if $nodes;
  $nodes =~ s/\n/,/g if $nodes;
  $nodes =~ s/,\s+,/,,/g if $nodes;
  $nodes =~ s/,+/,/g if $nodes;
  $nodes=~ s/^,|,$//g if $nodes;
  return $confref unless $nodes;
  @nodes = split/,/,$nodes if $nodes;
  $crsHome = $ncrsHome if $ncrsHome and not $crsHome;
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      $confref->{$ch}{node}{nodelist} = $nodes unless $confref->{$ch}{node}{nodelist};
      $confref->{$ch}{node}{nodearray} = \@nodes unless $confref->{$ch}{node}{nodearray};
    }
   
    if ( $crsHome )
    {
     $confref->{$crsHome}{node}{nodelist} = $nodes;
     $confref->{$crsHome}{node}{nodearray} = \@nodes;
     $confref->{$crsHome}{node}{crs_home} = $crsHome unless $confref->{$crsHome}{node}{crs_home};
    }
  }
  else
  {
   $crsHome = '#CRS_HOME#' unless $crsHome;
   
   $confref->{$crsHome}{node}{nodelist} = $nodes if $nodes;
   $confref->{$crsHome}{node}{nodearray} = \@nodes if @nodes;
   $confref->{$crsHome}{node}{crs_home} = $crsHome unless $confref->{$crsHome}{node}{crs_home};
  }
  has::Common::clusterConfigCopyVals($confref->{$crsHome}{node}, $confref->{$crsHome})
   if $crsHome and $confref->{$crsHome}{node};
  return $confref;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterConfigEmcrsp
#
# DESC
# return the config information for the cluster using emcrsp for 11gR2 clusters
#   and higher
#
# ARGUMENTS
#  crsHome if known
#  force recompute if already computed
# RETURN
#  ref to the hash to be filled in with values
#------------------------------------------------------------------------------
sub hasGetClusterConfigEmcrsp(;$$)
{
  # name : hasGetClusterConfigEmcrsp_fn
  # desc : it filters our the nodename
  #
  # arg  :
  #  ref to xml element to be filtered
  #``ref to the result hass entities  nodearray
  # 
  sub hasGetClusterConfigEmcrsp_fn(\%\%)
  {
    my ( $elref, $rsref ) = @_;
    return 1 unless $elref->{element} and $elref->{element} =~ /^(nodename|header|entity)$/i;
    #$rsref->{nodename}=$elref->{name} and return 1 if  $elref->{element} =~ /^nodename$/;
    if  ( $elref->{element} =~ /^nodename$/  )
    {
      $rsref->{node}{nodename}=$elref->{name} if $elref->{name}; 
      return 1;
    }
    # if the header is passed , read the cluster_name and crs_version 
    if  ( $elref->{element} =~ /^header$/i )
    {
      if ( $elref->{attrs} )
      {
        # read all attribs from
        # 
        for my $hdval ( keys %{$elref->{attrs}} )
        {
            $rsref->{$hdval} = $elref->{attrs}{$hdval};
            $rsref->{node}{crs_version} = $elref->{attrs}{$hdval} and next if $hdval =~ /^crs_version$/i;
            $rsref->{node}{cluster_name} = $elref->{attrs}{$hdval} and next if $hdval =~ /^cluster_name$/i;
            $rsref->{node}{ocr_configured} = $elref->{attrs}{$hdval} and next if $hdval =~ /^ocr_configured$/i;
            $rsref->{node}{ocr_location} = $elref->{attrs}{$hdval} and next if $hdval =~ /^ocr_location$/i;
        }
      }
    }
    # if node entity
    if  ( $elref->{element} =~ /^entity$/i )
    {
      my $thisnode;
      my %nodehash;
      # return unless entitu is of type entity
      return 1 unless $elref->{attrs} and $elref->{attrs}{entity_type} 
       and $elref->{attrs}{entity_type} =~ /^node$/;
      # get the node name
      if ( $elref->{attrs} )
      {
        # read all attribs from
        # 
        for my $hdval ( keys %{$elref->{attrs}} )
        {
          $thisnode =  $elref->{attrs}{$hdval} if $hdval =~ /^entity_name$/;
        }
      }
      $thisnode =~ s/^\s+|\s+$//g;
      # read all name=value pairs from the attributes of the node
      for my $elref1 (   @{$elref->{children}} )
      {
        next unless $elref1->{element} =~ /^attributes$/;
        for my $elref2 (   @{$elref1->{children}} )
        {
          next unless $elref2->{element} =~ /^attribute$/;
          my $name;
          my $value;
          for my $elref3 (   @{$elref2->{children}} )
          {
            next unless $elref3->{element} =~ /^(name|value)$/;
            $name = $elref3->{name} if $elref3->{element} =~ /^name$/;
            $value = $elref3->{name} if $elref3->{element} =~ /^value$/;
            $name =~ s/^\s+|\s+$//g if $name;
            $value =~ s/^\s+|\s+$//g if $value;
          }
          $nodehash{$name}=$value if $name;
        }
        # if you did not get the node name from entity_name try to get it from 
        #  attribute name
        $thisnode = $nodehash{NAME} if not $thisnode and $nodehash{NAME};
        next unless $thisnode;
        # save the node name information
        for my $nm ( keys %nodehash )
        {
          $rsref->{node}{details}{$thisnode}{$nm}=$nodehash{$nm};
        }
        last;
      }
    }
    return 1;
  }
  my ( $crsHome,$force ) = @_;
  # no cluster home is passed then 
  # get the cluster home from the env 
  #  and if not available or no emcrsp then discover one
  if ( not $crsHome )
  {
    $crsHome = has::Common::hasGetCRSHome();
    # get the cluster home by discovery
    if ( not $crsHome or not has::Common::hasCheckForEmcrsp($crsHome) )
    {
      $crsHome = has::Common::hasGetEnvCRSHome();
    }
  }
  my $confref = has::Common::getClusterCacheRef();
  return $confref 
   and warn "DEBUG:has::Common::hasGetClusterConfigEmcrsp:Cluster home is not passed or not set in env" 
   unless $crsHome;
  # already computed emcrsp for this crs home
  if ( not $force )
  {
   return $confref
    if $confref and ref($confref) and ref($confref) =~ /HASH/i 
     and $confref->{$crsHome} and $confref->{$crsHome}{emcrsp};
  }
  return $confref
   and warn "DEBUG:has::Common::hasGetClusterConfigEmcrsp:binary emcrsp is not present in $crsHome" 
   unless has::Common::hasCheckForEmcrsp($crsHome);
   my %cemutls_command_args = (exit_success_list => [(0,1)]);
  has::Common::hasSetCRSEnv($crsHome);
  my $cmdresults = has::Common::runsystemcommand("emcrsp","em config -e node",\%cemutls_command_args);
  has::Common::hasRestoreCRSEnv();
  $cmdresults = has::Common::hasCheckAndReturnEmcrspResults($cmdresults);
  warn "DEBUG:has::Common::hasGetClusterConfigEmcrsp Failed to get results from command - emcrsp em config -e node" 
   and return unless $cmdresults;
  my  %xmlvar =  has::Common::parse_xml($cmdresults);
  my %nodeList;
  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&hasGetClusterConfigEmcrsp_fn,\%nodeList);
  # this will override the discover information otherwise , esp if cssd is down and 
  # we do not get the nodelist from emcrsp olsnodes commands we rely on the nodelist
  # from discovery
  #$confref->{$crsHome} = \%nodeList;
  if ( keys %nodeList )
  {
     for my $nlkey ( keys %nodeList )
     {
        $confref->{$crsHome}{$nlkey}=$nodeList{$nlkey};
     }
  }
  # set marker that this crsHome has emcrsp so a 11gr2 cluster
  $confref->{$crsHome}{emcrsp}=1;
  
  # from node details get the node array and node nodelist
  if (  $confref 
        and $confref and ref($confref) and keys %{$confref}
        and $confref->{$crsHome}
        and $confref->{$crsHome}{node} 
        and $confref->{$crsHome}{node}{details} 
        and keys %{$confref->{$crsHome}{node}{details}} )
  {
     my @nodearray;
     my $nodelist;
     for my $nodename ( keys %{$confref->{$crsHome}{node}{details}} )
     {
        $nodename =~ s/^\s+|\s+$//g;
        next unless $nodename;
        push @nodearray,$nodename;
        $nodelist = "$nodelist,$nodename" and next if $nodelist;
        $nodelist = "$nodename" and next unless $nodelist;
        # make sure we have NAME and HOST_NAME here as they are keys 
        $confref->{$crsHome}{node}{details}{$nodename}{NAME} = $nodename 
         unless  $confref->{$crsHome}{node}{details}{$nodename}{NAME}; 
        $confref->{$crsHome}{node}{details}{$nodename}{HOST_NAME} = $nodename 
         unless  $confref->{$crsHome}{node}{details}{$nodename}{HOST_NAME}; 
        $confref->{$crsHome}{node}{details}{$nodename}{crs_home} = $crsHome 
         unless  $confref->{$crsHome}{node}{details}{$nodename}{crs_home}; 
    
     }
     # for node
     $confref->{$crsHome}{node}{crs_home} = $crsHome;
     $confref->{$crsHome}{node}{nodelist} = $nodelist if $nodelist;
     $confref->{$crsHome}{node}{nodearray} = \@nodearray;
  }
  # if this has emcrsp check if it is a has target
  if (  $confref 
        and $confref and ref($confref) and keys %{$confref}
        and $confref->{$crsHome}
        and $confref->{$crsHome}{node} 
        and $confref->{$crsHome}{node}{ocr_configured}
        and $confref->{$crsHome}{node}{ocr_location}
     )
  {
    has::Common::computeOcrType($confref->{$crsHome}{node}); 
  }
  has::Common::clusterConfigCopyVals($confref->{$crsHome}{node}, $confref->{$crsHome})
   if $crsHome and $confref->{$crsHome}{node};
  return $confref;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterStatus
#
# DESC
# get clusterware status
#
# ARGUMENTS
#  list of nodes
#  CRS Home if available
#  type ha|crs
#
#  RETURNS
#  returns a hash ref of results
#
#------------------------------------------------------------------------------
sub hasGetClusterStatus ( $;$$ )
{
  my ( $nodes,$crsHome,$type ) = @_;
  my @nlist;
  my @fnlist;
  my $nodearg = '';
  my $oldOH;
  my $confref = has::Common::getClusterCacheRef();
  if ( $crsHome )
  {
    $crsHome =~ s/\s+//g;
    undef $crsHome if $crsHome =~ /^$/ or $crsHome eq '';
  }
  # by default get cluster status
  $type = 'crs' unless $type and $type =~ /^(ha|crs)$/i;
  # no cluster home is passed then 
  # get the cluster home from the env 
  #  and if not available or no emcrsp then discover one
  if ( not $crsHome )
  {
    $crsHome = has::Common::hasGetCRSHome();
    # get the cluster home by discovery
    if ( not $crsHome )
    {
      $crsHome = has::Common::hasGetEnvCRSHome();
    }
  }
  return and warn "WARN:has::Common::hasGetClusterStatus:Cluster home is not passed or not set in env" 
   unless $crsHome;
  $confref->{$crsHome}{cluvfy_status}{crsIsUp} = 1;
  $confref->{$crsHome}{cluvfy_status}{failedCount} = 0;
  $confref->{$crsHome}{cluvfy_status}{successCount} = 0;
  $confref->{$crsHome}{cluvfy_status}{errCount}=0;
  @nlist = split(/,/, $nodes) if $nodes;
  # add onefor the case of cluvfy without -n nodes
  push(@nlist,'') unless @nlist;
  if ( $nodes )
  {
    $nodes =~ s/\s+//g;
    undef $nodes if $nodes =~ /^$/ or $nodes eq '';
    $nodearg = " -n $nodes" if $nodes;
  }
  warn("DEBUG:has::Common::hasGetClusterStatus:**Node List: $nodearg");
  my %command_args = (exit_failure_list => [()],timeout=>290,tries=>1);
  # execute cluvfy new format with -display_status
  # ideally this should be a version check so cluvfy is executed only once
  has::Common::hasSetCRSEnv($crsHome);
  if ( $has::Commmon::has_metric_config{ENV_VARIABLE_SRVM_TRACE} )
  {
         $ENV{SRVM_TRACE}='TRUE';
  }
  else
  {
         $ENV{SRVM_TRACE}='FALSE';
  }
  
  # trying to set CV directories as instructed by dipak.saggi due to bug in cluvfy
  # if two cluvfy are executed at the same time then can run on each others stage dir
  #delete $ENV{CV_DESTLOC} if $ENV{CV_DESTLOC};
  #delete $ENV{CV_TRACELOC} if $ENV{CV_TRACELOC};
  #my $cvdir = has::Common::has_get_cache_dir() or
  # warn "WARN:Failed to get a cache dir for setting CV_DESTLOC and CV_TESTLOC for cluvfy";
  #if ( $cvdir )
  #{
  #   $cvdir = catfile($cvdir,'clscv');
  #   my $cvdestloc = catfile($cvdir,'crsdestloc');
  #   my $cvtraceloc = catfile($cvdir,'crstraceloc');
  #   $ENV{CV_DESTLOC}=$cvdestloc if $cvdestloc;
  #   $ENV{CV_TRACELOC}=$cvtraceloc if $cvtraceloc;
  #}
  $confref->{$crsHome}{cluvfy_status}{o} = has::Common::runsystemcommand(
                "cluvfy comp $type -display_status $nodearg 2>\&1",'',\%command_args);
  #delete $ENV{CV_DESTLOC} if $ENV{CV_DESTLOC};
  #delete $ENV{CV_TRACELOC} if $ENV{CV_TRACELOC};
  has::Common::hasRestoreCRSEnv();
  delete $ENV{SRVM_TRACE} if $ENV{SRVM_TRACE};
  $confref->{$crsHome}{cluvfy_status}{o} =~ s/^\s+|\s+$// 
   if $confref->{$crsHome}{cluvfy_status}{o};
  #execute cluvfy new format without -display_status if flag option is not
  # supported
  unless ( $confref->{$crsHome}{cluvfy_status}{o} 
   and  $confref->{$crsHome}{cluvfy_status}{o} =~ /NODE_STATUS::/i )
  {
    %command_args = (exit_failure_list => [()],timeout=>290,tries=>1);
    has::Common::hasSetCRSEnv($crsHome);
    if ( $has::Commmon::has_metric_config{ENV_VARIABLE_SRVM_TRACE} )
    {
       $ENV{SRVM_TRACE}='TRUE';
    }
    else
    {
       $ENV{SRVM_TRACE}='FALSE';
    }
 
  
  
    # trying to set CV directories as instructed by dipak.saggi due to bug in cluvfy
    # if two cluvfy are executed at the same time then can run on each others stage dir
    #delete $ENV{CV_DESTLOC} if $ENV{CV_DESTLOC};
    #delete $ENV{CV_TRACELOC} if $ENV{CV_TRACELOC};
    #my $cvdir = has::Common::has_get_cache_dir() or
    # warn "WARN:Failed to get a cache dir for setting CV_DESTLOC and CV_TESTLOC for cluvfy";
    #if ( $cvdir )
    #{
    # $cvdir = catfile($cvdir,'clscv');
    # my $cvdestloc = catfile($cvdir,'crsdestloc');
    # my $cvtraceloc = catfile($cvdir,'crstraceloc');
    # $ENV{CV_DESTLOC}=$cvdestloc if $cvdestloc;
    # $ENV{CV_TRACELOC}=$cvtraceloc if $cvtraceloc;
    #}
    $confref->{$crsHome}{cluvfy_status}{o} = has::Common::runsystemcommand( "cluvfy comp $type $nodearg 2>\&1",'',\%command_args);
    #delete $ENV{CV_DESTLOC} if $ENV{CV_DESTLOC};
    #delete $ENV{CV_TRACELOC} if $ENV{CV_TRACELOC};
    has::Common::hasRestoreCRSEnv();
    delete $ENV{SRVM_TRACE} if $ENV{SRVM_TRACE};
    $confref->{$crsHome}{cluvfy_status}{o} =~ s/^\s+|\s+$// if $confref->{$crsHome}{cluvfy_status}{o};
  }
  chomp($confref->{$crsHome}{cluvfy_status}{o}) 
   if $confref->{$crsHome}{cluvfy_status}{o};
  $confref->{$crsHome}{cluvfy_status}{o} =~ s/\n/,/g 
   if $confref->{$crsHome}{cluvfy_status}{o};
  $confref->{$crsHome}{cluvfy_status}{o} =~ s/\cM//g 
   if $confref->{$crsHome}{cluvfy_status}{o};
  die("ERROR:has::Common::hasGetClusterStatus:crs_status_cluster.pl:Failed to get any results from cluvfy comp $type -n $nodearg") 
   unless $confref->{$crsHome}{cluvfy_status} 
   and $confref->{$crsHome}{cluvfy_status}{o};
  # output with -display_status flag 
  # cluvfy comp crs -display_status -n staca31
  #NODE_STATUS::Node1:SUCC
  #NODE_STATUS::Node2:EFAIL
  #NODE_STATUS::Node3:VFAIL
  #NODE_STATUS::Node4:SUCC
  #OVERALL_STATUS::EFAIL
  # if cluvfy comp crs -n node1, node2 does not return a success 
  # for all nodes the exit status is failure
  for ( (1) )
  {
    # crs is up if cluvfy did not return a failure
    unless  ( $command_args{command_return_status} )
    {
      $confref->{$crsHome}{cluvfy_status}{crsIsUp} = 1;
      $confref->{$crsHome}{cluvfy_status}{failedCount}=0;
      $confref->{$crsHome}{cluvfy_status}{successCount}=@nlist;
      last;
    }
    #------------------------------------------------------------------
    # check for failed nodes etc only if the cluvfy returned a failure
    #------------------------------------------------------------------
    #
    # for pre11g version we are trying to get the list of failed nodes
    # from the cli output using the following logic
    # cluvfy  comp crs -n 
    # a) fails on all nodes
    #   Verification of CRS integrity was unsuccessful on all the nodes.
    # b) partial failure , fails on some nodes
    #   Verification of CRS integrity was unsuccessful.
    #   checks did nto pass for the following node(s):
    #   node1,node2
    # c)all successful
    #   Verification of CRS integrity was successful
    #
    #  email sent by dipak.saggi confirming the messages
    #        "Verification of {0} was unsuccessful on all the nodes. "
    #       "Verification of {0} was unsuccessful. "
    #       "Checks did not pass for the following node(s):"
    #
    my $with_node_status_for_em = 'NODE_STATUS::|OVERALL_STATUS::';
    my $failed_on_some_nodes= 
     'Checks\s+did\s+not\s+pass\s+for\s+the\s+following\s+node\(s\):';
    my $failed_on_all_nodes =
     'unsuccessful\s+on\s+all\s+the\s+nodes';
    my $error_executing_cluvfy =
'ERROR:|CRS\s+is\s+not\s+installed\s+on\s+any\s+of\s+the\s+nodes|Verification\s+cannot\s+proceed|User\s+equivalence\s+is\s+not\s+set\s+for\s+nodes';
    # if cluvfy supports the displsy_status flag and gives NODE_STATUS
    if ( $confref->{$crsHome}{cluvfy_status}{o} =~ /$with_node_status_for_em/ )
    {
       my @inp = split /\n/,$confref->{$crsHome}{cluvfy_status}{o};
       my @rows = grep /$with_node_status_for_em/, @inp;
       my $nodeVerificationFailed;
       for my $row ( @rows )
       {
          $row =~ s/^\s+|\s+$//;
          #success
          $confref->{$crsHome}{cluvfy_status}{successCount} +=1 and next 
           if $row and $row =~ /NODE_STATUS::.+:SUCC$/;
          #if command returns overall success, then take the results of overall
          # and declare crs is up on all nodes
          if ( $row and $row =~ /OVERALL_STATUS::SUCC/ )
	  {
            $confref->{$crsHome}{cluvfy_status}{crsIsUp}=1;
            $confref->{$crsHome}{cluvfy_status}{failedCount}=0;
            $confref->{$crsHome}{cluvfy_status}{successCount} = @nlist;
            last;
	  }
          # verification failed, increment failed count
          $confref->{$crsHome}{cluvfy_status}{failedCount} +=1  and next 
           if $row =~ /NODE_STATUS::.+:VFAIL/;
          # execution failure
          next unless $row =~ /NODE_STATUS::.+:EFAIL/;
          $confref->{$crsHome}{cluvfy_status}{errCount} += 1;
          my ($node) = ( $row =~ /NODE_STATUS::(.+):EFAIL/);
          warn("WARN:has::Common::hasGetClusterStatus Failed to get node name from execution failure of cluvfy while checking Clusterware Status $row\n")
            and next unless $node;
          $nodeVerificationFailed = "$nodeVerificationFailed,$node" and next if $nodeVerificationFailed;
          $nodeVerificationFailed = "$node" unless $nodeVerificationFailed;
       }
       warn("WARN:has::Common::hasGetClusterStatus Status verification failed due to cluvfy execution failure for node(s) $nodeVerificationFailed\n")
          if $nodeVerificationFailed;
       # crs is down if all nodes failed or failed count is > nodes
       $confref->{$crsHome}{cluvfy_status}{crsIsUp} = 0 
        if scalar(@nlist) == $confref->{$crsHome}{cluvfy_status}{failedCount};
       $confref->{$crsHome}{cluvfy_status}{crsIsUp} = 0 
        unless $confref->{$crsHome}{cluvfy_status}{successCount};
    }
    elsif ($confref->{$crsHome}{cluvfy_status}{o} =~ /$failed_on_some_nodes/i)
    {
       my $fo = $confref->{$crsHome}{cluvfy_status}{o};
       $fo =~ s/\n+/,/g;
       my ($ignore,$fnodes) =
         ( $fo =~ /.*($failed_on_some_nodes)(.*)$/i);
        $fnodes =~ s/,\s+,/,/g;
        $fnodes =~ s/,+/,/g;
        $fnodes =~ s/^\s+|\s+$//;
        $fnodes =~ s/^,+|,+$//;
        warn("DEBUG:has::Common::hasGetClusterStatus:crs_status_cluster.pl:**failed node list:($fnodes)");
        @fnlist = split(/,/, $fnodes);
        $confref->{$crsHome}{cluvfy_status}{failedCount} = scalar(@fnlist);
        # if crs is up on atleast one node then crs is up
        if (  scalar(@nlist) == scalar (@fnlist) )
	{
         $confref->{$crsHome}{cluvfy_status}{crsIsUp} = 0;
         last;
        }
        $confref->{$crsHome}{cluvfy_status}{crsIsUp} = 1;
        last;
    }
    elsif ($confref->{$crsHome}{cluvfy_status}{o} =~ /$failed_on_all_nodes/i )
    {
      $confref->{$crsHome}{cluvfy_status}{crsIsUp} = 0;
      $confref->{$crsHome}{cluvfy_status}{failedCount}=scalar(@nlist);
      last;
    }
    elsif ($confref->{$crsHome}{cluvfy_status}{o} =~ /$error_executing_cluvfy/i )
    {
      $confref->{$crsHome}{cluvfy_status}{crsIsUp}=2;
      $confref->{$crsHome}{cluvfy_status}{failedCount}=scalar(@nlist);
      warn("WARN:has::Common::hasGetClusterStatus Failed to get Status for cluster, CRS verification failed for cluster $confref->{$crsHome}{cluvfy_status}{o}\n");
      last;
    }
    else
    {
      $confref->{$crsHome}{cluvfy_status}{crsIsUp} = 2;
      $confref->{$crsHome}{cluvfy_status}{failedCount}=scalar(@nlist);
      warn("WARN:has::Common::hasGetClusterStatus Failed to get Status for cluster , CRS verification failed for cluster $confref->{$crsHome}{cluvfy_status}{o}\n");
      last;
    }
  }
  $confref->{$crsHome}{crs_home} = $crsHome 
   if $crsHome and not $confref->{$crsHome}{crs_home};
  return $confref->{$crsHome}{cluvfy_status};
}
#------------------------------------------------------------------------------
# FUNCTION :    hasClusterHealthCheck
#
# DESC
# get clusterware health check status
#
# ARGUMENTS
#  CRS Home if available
#  type = crs
#
#  RETURNS
#  returns a hash ref of results
#
#------------------------------------------------------------------------------
sub hasClusterHealthCheck ( $;$ )
{
  my ( $type,$crsHome ) = @_;
  my %clshm;
  my $with_node_status_for_em = 'NODE_STATUS:|OVERALL_STATUS::';
  if ( $crsHome )
  {
    $crsHome =~ s/\s+//g;
    undef $crsHome if $crsHome =~ /^$/ or $crsHome eq '';
  }
  # by default get cluster status
  $type = 'crs' unless $type and $type =~ /^(ha|crs)$/i;
  # no cluster home is passed then 
  # get the cluster home from the env 
  #  and if not available or no emcrsp then discover one
  if ( not $crsHome )
  {
    $crsHome = has::Common::hasGetCRSHome();
    # get the cluster home by discovery
    if ( not $crsHome )
    {
      $crsHome = has::Common::hasGetEnvCRSHome();
    }
  }
  return and warn "WARN:has::Common::hasClusterHealthCheck:Cluster home is not passed or not set in env" 
   unless $crsHome;
  my %command_args = (exit_failure_list => [()],timeout=>290,tries=>1);
  my $results;
  # execute cluvfy new format with -display_status
  # ideally this should be a version check so cluvfy is executed only once
  has::Common::hasSetCRSEnv($crsHome);
  if ( $has::Commmon::has_metric_config{ENV_VARIABLE_SRVM_TRACE} )
  {
    $ENV{SRVM_TRACE}='TRUE';
  }
  else
  {
    $ENV{SRVM_TRACE}='FALSE';
  }
 
  # trying to set CV directories as instructed by dipak.saggi due to bug in cluvfy
  # if two cluvfy are executed at the same time then can run on each others stage dir
  #delete $ENV{CV_DESTLOC} if $ENV{CV_DESTLOC};
  #delete $ENV{CV_TRACELOC} if $ENV{CV_TRACELOC};
  #my $cvdir = has::Common::has_get_cache_dir() or
  # warn "WARN:Failed to get a cache dir for setting CV_DESTLOC and CV_TESTLOC for cluvfy";
  #if ( $cvdir )
  #{
  #   $cvdir = catfile($cvdir,'clshc');
  #   my $cvdestloc = catfile($cvdir,'crsdestloc');
  #   my $cvtraceloc = catfile($cvdir,'crstraceloc');
  #   $ENV{CV_DESTLOC}=$cvdestloc if $cvdestloc;
  #   $ENV{CV_TRACELOC}=$cvtraceloc if $cvtraceloc;
  #}
  $results = has::Common::runsystemcommand(
                "cluvfy comp health -_format 2>\&1",'',\%command_args);
  #delete $ENV{CV_DESTLOC} if $ENV{CV_DESTLOC};
  #delete $ENV{CV_TRACELOC} if $ENV{CV_TRACELOC};
  has::Common::hasRestoreCRSEnv();
  delete $ENV{SRVM_TRACE} if $ENV{SRVM_TRACE};
#  warn "ERROR::has::Common::hasClusterHealthCheck:Failed executing the cluster health check command cluvfy comp health \n" and return if $command_args{command_return_status};
  chomp  $results if $results;
  $results =~ s/^\s+|\s+$// if $results;
  chomp  $results if $results;
  $results =~ s/^\s+|\s+$// if $results;
  $results =~ s/\cM//g if $results;
  warn "ERROR::has::Common::hasClusterHealthCheck:Failed to get results for cluster health check from cluvfy comp health \n" 
   and return unless $results;
  my @results = split/\n/,$results;
  # if cluvfy comp crs -n node1, node2 does not return a success 
  # for all nodes the exit status is failure
  my %description;
  my $current_component;
  my $current_message;
  my $i=0;
  for my $res ( @results )
  {
    $res =~ s/^\s+|\s+$//g if $res;
    next unless $res;
    # if cluvfy supports the displsy_status flag and gives NODE_STATUS
    if ( $res =~ /$with_node_status_for_em/ )
    {
          my ( $component,$node,$status,$description) = split/::/,$res;
           warn "WARN::has::Common::hasClusterHealthCheck:Failed to read the component and status from $res" 
            and next
             unless $component or $node or $status;
          if ( $description )
          {
            $description{$component}=$description;
          }
          if ( $node =~/NODE_STATUS:/ )
          {
            my ( $node_name ) = ( $node =~ /NODE_STATUS:(.*)$/);
            warn "WARN::has::Common::hasClusterHealthCheck:Failed to read the node name from $node" 
             unless $node_name;
            $node = "$node_name" if $node_name;
          }
          elsif ( $node =~/OVERALL_STATUS/ )
          {
           $node = 'Cluster';
          }
          $current_component="$component\_$node";
          $clshm{$current_component}{elem_attribs}{GENERATED_KEY_1}="$component\_$node";
          $clshm{$current_component}{elem_attribs}{CRS_COMPONENT}="$component";
          $clshm{$current_component}{elem_attribs}{DEDUCED_COMPONENT_STATUS}="$status";
          if ( $status =~ /SUCC/ )
          {
            $clshm{$current_component}{elem_attribs}{COMPONENT_STATUS}="$status";
          }
          if ( $status =~ /SUCC/ )
          {
            $clshm{$current_component}{elem_attribs}{COMPONENT_STATUS_MESSAGE}='Successful';
          }
          elsif ( $status =~ /VFAIL/ )
          {
            $clshm{$current_component}{elem_attribs}{COMPONENT_STATUS_MESSAGE}='Failed';
          }
          elsif ( $status =~ /EFAIL/ )
          {
            $clshm{$current_component}{elem_attribs}{COMPONENT_STATUS_MESSAGE}='Execution Failed';
          }
          elsif ( $status =~ /WARN/ )
          {
            $clshm{$current_component}{elem_attribs}{COMPONENT_STATUS_MESSAGE}='Warning';
          }
          else
          {
            $clshm{$current_component}{elem_attribs}{COMPONENT_STATUS_MESSAGE}='Unknown';
          }
          $clshm{$current_component}{elem_attribs}{NODE_NAME}="$node";
          $clshm{$current_component}{elem_attribs}{COMPONENT_DESCRIPTION}=$description{$component} 
           if $description{$component};
    }
    elsif ($res =~ /^MSG_START/i)
    {
      undef $current_message;
      $i = 0;
    }
    elsif ( $res =~ /^MSG_END/i )
    {
     $clshm{$current_component}{elem_attribs}{COMPONENT_STATUS}=
      $clshm{$current_component}{elem_attribs}{DEDUCED_COMPONENT_STATUS} 
      if $clshm{$current_component}{elem_attribs}{DEDUCED_COMPONENT_STATUS};
     if ( $current_message )
     {
      warn "WARN::has::Common::hasClusterHealthCheck:Failed to get the component for error message $current_message\n" 
      unless $current_component;
     }
     else
     {
      warn "WARN::has::Common::hasClusterHealthCheck:Failed to get the component for $res\n" 
      unless $current_component;
     }
     $clshm{$current_component}{elem_attribs}{ERROR_MESSAGE}=$current_message;
     undef $current_message;
     undef $current_component;
     $i=0;
    }
    else
    {
       if ( $res =~ /^PRVF-/ )
       {
         $i++;
         $res = "($i) $res";
       }
       if ( $current_message )
       {
         $current_message = "$current_message $res";
       }
       else
       {
         $current_message = $res;
       }
    }
  }
  return \%clshm;
}
#------------------------------------------------------------------------------
# FUNCTION :    getClusterCacheRef
#
# DESC
# return the ref for config information for the cluster 
#
# ARGUMENTS
# RETURN
#  ref for config information for the cluster 
#------------------------------------------------------------------------------
sub getClusterCacheRef()
{
   return \%has::Common::hasClusterConfig;
}
#------------------------------------------------------------------------------
# FUNCTION :    clusterConfigCopyVals
#
# DESC
# copy the list of cluster properties between the two lists passed
#
# ARGUMENTS
# RETURN
#  source ref to copy from config information for the cluster 
#  destination ref to copy to config information for the cluste
#------------------------------------------------------------------------------
sub clusterConfigCopyVals($$;)
{
      
  my ( $source_ref,$dest_ref) = @_;
      
  for my $prop ( qw ( crs_home nodelist nodearray crs_version version 
                      vendor isvendorcw nodename host_name cluster_name emcrsp
                      ocr_location ocr_configured ocr_type 
                     ) )
  {
      
  	next if ( not $source_ref or not defined  $source_ref->{$prop} ) 
  	 and ( not $dest_ref or not defined $dest_ref->{$prop} );
       if ( not $dest_ref or not defined $dest_ref->{$prop} )
       { 
  	  $dest_ref->{$prop} = $source_ref->{$prop} 
          if $source_ref and $source_ref->{$prop};
       }
  
       if ( not $source_ref or not defined $source_ref->{$prop} ) 
       {
  	  $source_ref->{$prop} = $dest_ref->{$prop} 
          if $dest_ref and $dest_ref->{$prop};
       }
      
  }
      
  return 1;
      
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterConfig
#
# DESC
# return the config information for the cluster 
#
# ARGUMENTS
#  crsHome if known
# dynamicProperty - not null valud if function is called for dynamic prop 
#   computation
# RETURNS
#  ref to the hash to be filled in with values
#------------------------------------------------------------------------------
#the structure returned is
#it can hold information for multiple crs_homes which is required in dev
# environments
#
#  cluster information from cli or emcrsp
#  {}=(crs_home,cluster_name,host_name,nodelist,nodearray,nodename,host_name,
#    vendor,isvendorcw,crs_version,version,emcrsp)
#
#  local node information from cli or emcrsp
#  {}{pre11gr2_config} = {crs_home,nodename,host_name,vendor,isvendorcw,
#                              crs_version,version,nodelist,nodearray}
#
#  discovered information
#  {}{discover}{nodes}{}
#  {}{discover}{crs_home}
#  {}{discover}{cluster_name}
#  {}{discover}{nodelist}
#  {}{discover}{nodearray}
#
#  node information from cli lsnode or emcrsp
#  {}{node}{details}{}={NAME,HOST_NAME,NODE_NUM,}
#  {}{node}{crs_home}
#  {}{node}{cluster_name}
#  {}{node}{nodearray}
#  {}{node}{nodelist}
#
sub hasGetClusterConfig(;$$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $confref;
  $confref =  has::Common::getClusterCacheRef();
  $crsHome =~ s/\s+//g if $crsHome;
  undef $crsHome if $crsHome and $crsHome =~ /^\s*$/;
  # values already cached then return it back
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    if ( $crsHome )
    {
       if  ( $confref->{$crsHome} )
       {
          my $flag = 1;
      
          for my $prop ( qw ( crs_home nodelist nodearray crs_version version
                              ocr_location ocr_configured ocr_type 
                              vendor isvendorcw nodename host_name cluster_name emcrsp 
                              ) )
          {
            next if defined $confref->{$crsHome}{$prop};
            $flag = 0;
            last;
          }
      
          return $confref if $flag;
      }
    }
    # if there is no entry for the passed crs home then check for others
    #  the assumption that there is only one crsHome or one cluster per box
    else 
    {
        # values already cached then return it back
        for my $ch ( keys %{$confref} )
        {
          my $flag = 1;
          for my $prop ( qw ( crs_home nodelist nodearray crs_version version
                              ocr_location ocr_configured ocr_type 
                              vendor isvendorcw nodename host_name cluster_name emcrsp ) )
          {
            next if defined $confref->{$ch}{$prop};
            $flag = 0;
            last;
          }
          return $confref if $flag;
        }
    }
  }
  # start with discovery , populates the discover structure
  $confref = has::Common::hasDiscoverCluster();
  for my $ch ( keys %{$confref} )
  {
    next unless $confref->{$ch}{discover};
    has::Common::clusterConfigCopyVals($confref->{$ch}{discover},$confref->{$ch});
  }
  my @chomes;
  # if a crs home is passed get config only by passing that crs home
  # if no crs home is passed use discovery
  if ( $crsHome )
  {
    push @chomes,$crsHome;
  }
  else
  {
    @chomes = keys %{$confref};
  }
  if ( not @chomes )
  {
      my $crsEnvHome = has::Common::hasGetEnvCRSHome();
      push @chomes, $crsEnvHome if $crsEnvHome;
  }
  # for each cluster home get the values
  for my $ch ( @chomes )
  {
    # get he 11g node list crsctl/cemutl stuff, pupulates pre11gr2_config
    $confref = has::Common::hasGetClusterConfigPre11g($ch,$dynProp);
    # get the 11g local nodename using olsnodes -l , populates pre11gr2_config
    $confref = has::Common::hasGetClusterGetNodeNamePre11g($ch);
    has::Common::clusterConfigCopyVals($confref->{$ch}{pre11gr2_config}, $confref->{$ch})
      if $confref->{$ch}{pre11gr2_config};
    # get the 11g node list from olsnodes populates the node element
    $confref = has::Common::hasGetClusterNodeListPre11g($ch) 
     unless $confref->{$ch}{node} 
     and $confref->{$ch}{node}{nodearray} 
     and $confref->{$ch}{node}{nodelist};
    has::Common::clusterConfigCopyVals($confref->{$ch}{node},
                                       $confref->{$ch})
     if $confref->{$ch}{node};
    # get the local node name
    my $nodename = $confref->{$ch}{nodename} if $confref->{$ch}{nodename};
    # get the local host name
    $confref->{$ch}{host_name} = has::Common::hasGetLocalHostName() 
     unless $confref->{$ch}{host_name};
    my $hostname = $confref->{$ch}{host_name} 
     if $confref->{$ch}{host_name};
    # get the node list to build node details from 11gR1 olsnodes list
    if ( keys %{$confref->{$ch}} and $confref->{$ch}{nodearray} )
    {
  
      for my $node ( @{$confref->{$ch}{nodearray}} )
      {
  
        $confref->{$ch}{node}{details}{$node}{NAME}=$node;
  
        # if this is the current node then get hostname
        $confref->{$ch}{node}{details}{$node}{HOST_NAME}=$node 
         if $hostname and $nodename and $node =~ /^$nodename$/ 
         and not $confref->{$ch}{node}{details}{$node}{HOST_NAME};
  
        # TBD if this is the current node then get hostname
        # css api should provide the host name for each nodename
        $confref->{$ch}{node}{details}{$node}{HOST_NAME}=$node
         if not $confref->{$ch}{node}{details}{$node}{HOST_NAME};
  
        $confref->{$ch}{node}{details}{$node}{crs_home}=$ch
         unless $confref->{$ch}{node}{details}{$node}{crs_home};
  
      }
  
    }
  
    # get the node name
    $confref = has::Common::hasGetClusterGetNodeNamePre11g($ch) unless $confref->{$ch}{nodename};
  
    # do the 11 discovery using emcrsp
    $confref = has::Common::hasGetClusterConfigEmcrsp($ch);
  
    # synchronize the top vales from all the sub lists
    #  discover  - from discovery
    #  pre11gr2_config  - from cemutl,crsctl, olsnode
    #  node   - from emcrsp,olsnode
    has::Common::clusterConfigCopyVals($confref->{$ch}{node}, $confref->{$ch}) 
     if $confref->{$ch}{node};
    has::Common::clusterConfigCopyVals($confref->{$ch}{pre11gr2_config},$confref->{$ch})
     if $confref->{$ch}{pre11gr2_config};
    has::Common::clusterConfigCopyVals($confref->{$ch}{discover},$confref->{$ch})
     if $confref->{$ch}{discover};
  }
  return $confref;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasEMClusterHasTargetDiscovery
#
# DESC
# return the cluster name, crsHome and type(has|cluster)
#
# ARGUMENTS
# emdRoot 
# crsHome if known
#----------------------------------------------------------------------------
sub hasEMClusterHasTargetDiscovery($;$)
{
  my ($emdRoot,$crsHome)  = @_;
  my $hascluster = 'cluster';
  my $clusterName;
  undef $crsHome unless $crsHome;
  warn("ERROR:HasCluster.pm:hasEMClusterHasTargetDiscovery EMDROOT is not passed as an argument, failed discovery") 
    and return unless $emdRoot;
#-----------------------------------------------------------------------------
# Init as this function is called directly by discovery function
#-----------------------------------------------------------------------------
  has::Common::hasInitSignalsStderr();
  warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery emdRoot=$emdRoot");
  if (not $crsHome )
  {
     if (defined($ENV{CRS_HOME}))
     {
       if ($ENV{CRS_HOME} ne "#CRS_HOME#")
       {
          $crsHome = $ENV{CRS_HOME};
          warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery crsHome is picked from the environment CRS_HOME as $crsHome");
       }
     }
  }
  else
  {
    warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery crsHome passed as argument=$crsHome");
  }
  $clusterName = '';
  # a crsHome has been passed for discovery
  if ($crsHome) 
  {
    if ( has::Common::hasCheckForEmcrsp($crsHome) and has::Common::hasGetOcrType($crsHome) =~ /^has$/  )
    {
      $hascluster = 'has';
      $clusterName = has::Common::hasGetHasName($crsHome);
      if ( $clusterName )
      {
       warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery After executing  has::Common::hasGetHasName hasName=$clusterName");
      }
      else
      {
       warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery After executing  has::Common::hasGetHasName hasName=undef");
      }
    }
    else
    {
      $hascluster = 'cluster';
      $clusterName = has::Common::hasCemutloClusterName($crsHome);
      if ( $clusterName )
      {
       warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery After executing cemutlo,clusterName=$clusterName");
      }
      else
      {
       warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery After executing cemutlo,clusterName=undef");
      }
    }
  }
   # Attempt to search OUI inventory for the CRS Home if we still could not get a clsName
   if ( not $clusterName )
   {
       my $confref;
       $confref = has::Common::hasDiscoverCluster();
       if ( $confref and ref($confref) and ref($confref) =~ /HASH/ )
       {
        for my $ch ( keys %{$confref} )
        {
          next unless $confref->{$ch}{discover};
          warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery Checking $ch ...Checking $ch ...");
          if ( $confref->{$ch}{ocr_type} and $confref->{$ch}{ocr_type} =~ /has/ )
          {
            $hascluster = 'has';
            $clusterName = has::Common::hasGetHasName($ch);
            warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery After executing  has::Common::hasGetHasName hasName=$clusterName") if $clusterName;
          }
          else
          {
            $hascluster = 'cluster';
            $clusterName = has::Common::hasCemutloClusterName($ch);
            warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery After executing cemutlo,clusertName=$clusterName") if $clusterName;
          }
          $clusterName =~ s/^\s+|\s+$// if $clusterName;
          if ($clusterName)
          {
            warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery cemutlo works fine in $ch. Found ($clusterName, $ch)");
            $crsHome = $ch;
            last;
          }
        }
       }
    }
    if ( not $clusterName )
    {
        # still empty, check cemutls in case of 92 cluster
        $hascluster = 'cluster';
        $clusterName = has::Common::hasCemutlsClusterName($emdRoot);
        $clusterName =~ s/^\s+|\s+$// if $clusterName;
        warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery After executing cemutls,clusertName=$clusterName") if $clusterName;
        if ( not $clusterName )
        {
          $hascluster = 'cluster';
          # it's a 9.2 cluster, setting crsHome to emtpy.
          $crsHome = '';
        }
    }
    $clusterName = '' unless $clusterName;
    warn("DEBUG:HasCluster.pm:hasEMClusterHasTargetDiscovery Returning clusterName:$clusterName");
    # append first node name if cluster name is crs and target type is cluster to get a distinct targetName
    if ( $hascluster =~ /^cluster$/ )
    {
      $clusterName = has::Common::appendNodeNameToClusterName($clusterName,$crsHome) if $clusterName and $crsHome;
    }
    has::Common::hasRestoreStderr();
    return wantarray? ($clusterName, $crsHome, $hascluster):$clusterName;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetNodeName
#
# DESC
# return the node name for the local node
#
# ARGUMENTS
# crsHome if known
#------------------------------------------------------------------------------
sub hasGetNodeName(;$)
{
  my ( $crsHome, $dynProp ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{nodename} 
   if $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{nodename};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{nodename} if $confref->{$ch}{nodename};
    }
  }
  $confref = has::Common::hasGetClusterConfig($crsHome,$dynProp);
  return $confref->{$crsHome}{nodename} if $confref and $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{nodename};
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{nodename} if $confref->{$ch}{nodename};
    }
  }
  warn "WARN:has::Common::hasGetNodeName Failed to get the node name for the cluster node, trying to get hostname";
  my $hostName =  has::Common::hasGetLocalHostName();
  if ( $hostName )
  {
     $hostName  =~ s/\..*//;
     $hostName = lc $hostName if $hostName;
  }
  warn "WARN:has::Common::hasGetNodeName Failed to get the node name for the cluster node" and return unless $hostName;
  return $hostName;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterName
#
# DESC
# return the Cluster Name 
#
# ARGUMENTS
# crsHome if known
# dynamicProperty - not null valud if function is called for dynamic prop 
#   computation
#
#------------------------------------------------------------------------------
sub hasGetClusterName(;$$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{cluster_name} 
   if $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{cluster_name};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{cluster_name}
       if $confref->{$ch}{cluster_name};
    }
  }
  if ( $dynProp )
  {
    $confref = has::Common::hasGetClusterConfigPre11g($crsHome,$dynProp);
    return $confref->{$crsHome}{cluster_name} 
      if $crsHome
      and $confref->{$crsHome}
      and $confref->{$crsHome}{cluster_name};
    return;
  }
  $confref = has::Common::hasGetClusterConfig($crsHome);
  return $confref->{$crsHome}{cluster_name} if $confref and $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{cluster_name};
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{cluster_name} if $confref->{$ch}{cluster_name};
    }
  }
  warn "WARN:has::Common::hasGetClusterName Failed to get the cluster name for the cluster ";
   
  my $cookedName;
  if ( ( $ENV{EM_TARGET_TYPE} and $ENV{EM_TARGET_TYPE} =~ /cluster|has/i  )  or
       ( $ENV{TARGET_TYPE} and $ENV{TARGET_TYPE} =~ /cluster|has/i  ) )
  {
     $cookedName = $ENV{TARGET_NAME} if $ENV{TARGET_NAME};
     $cookedName = $ENV{EM_TARGET_NAME} if $ENV{EM_TARGET_NAME};
  
  }
  my $hostname = has::Common::hasGetLocalHostName() unless $cookedName;
  $cookedName = "host_$hostname" if $hostname and not $cookedName;
  
  return $cookedName if $cookedName;
  return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetNodeList
#
# DESC
# return the coma seperated list of nodes
#
# ARGUMENTS
# crsHome if known
#
#------------------------------------------------------------------------------
sub hasGetNodeList(;$)
{
  my ( $crsHome ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{nodelist} 
   if $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{nodelist};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{nodelist}
       if $confref->{$ch}{nodelist};
    }
  }
  $confref = has::Common::hasGetClusterConfig($crsHome);
  return $confref->{$crsHome}{nodelist} if $confref and $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{nodelist};
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{nodelist} if $confref->{$ch}{nodelist};
    }
  }
  warn "WARN:has::Common::hasGetNodeList Failed to get list of nodes" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetNodeStatus
#
# DESC
# return the status for the node passed or local node is null
#
# ARGUMENTS
# crsHome if known
# nodeName is known
#
#------------------------------------------------------------------------------
sub hasGetNodeStatus(;$$)
{
  my ( $crsHome,$nodeName ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{nodestatus} 
   if $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{nodestatus};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{nodestatus}
       if $confref->{$ch}{nodestatus};
    }
  }
  $confref = has::Common::hasGetClusterConfig($crsHome);
  return $confref->{$crsHome}{nodestatus} if $confref and $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{nodestatus};
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{nodestatus} if $confref->{$ch}{nodestatus};
    }
  }
  warn "DEBUG:has::Common::hasGetNodeStatus Failed to get status for node" and return unless $nodeName;
  warn "DEBUG:has::Common::hasGetNodeStatus Failed to get status for node $nodeName" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterVersion
#
# DESC
# return the Clusterware Acive Version 
#
# ARGUMENTS
# crsHome if known
# dynamicProperty - not null valud if function is called for dynamic prop 
#   computation
#
#------------------------------------------------------------------------------
sub hasGetClusterVersion(;$$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{crs_version} 
   if $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{crs_version};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{crs_version}
       if $confref->{$ch}{crs_version};
    }
  }
  if ( $dynProp )
  {
   $confref = has::Common::hasGetClusterConfigPre11g($crsHome,$dynProp);
   return $confref->{$crsHome}{crs_version} 
    if $crsHome
    and $confref->{$crsHome}
    and $confref->{$crsHome}{crs_version};
   
   warn "WARN:has::Common::hasGetClusterVersion Failed to get cluster active version " and return;
  }
  $confref = has::Common::hasGetClusterConfig($crsHome);
  return $confref->{$crsHome}{crs_version} if $confref and $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{crs_version};
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{crs_version} if $confref->{$ch}{crs_version};
    }
  }
  warn "WARN:has::Common::hasGetClusterVersion Failed to get cluster active version " and return;
}
sub hasGetClusterActiveVersion(;$$)
{
  my ( $crsHome,$dynProp ) = @_;
  if ( $dynProp )
  {
   return hasGetClusterVersion($crsHome,$dynProp);
  }
  else
  {
   return hasGetClusterVersion($crsHome);
  }
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterVendor
#
# DESC
#  return the cluster vendor
#
# ARGUMENTS
# crsHome if known
# dynamicProperty - not null valud if function is called for dynamic prop 
#   computation
#
# RETURNS
# vendor name
#------------------------------------------------------------------------------
sub hasGetClusterVendor(;$$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{vendor} 
   if $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{vendor};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{vendor}
       if $confref->{$ch}{vendor};
    }
  }
  if ( $dynProp )
  {
    $confref = has::Common::hasGetClusterConfigPre11g($crsHome,$dynProp);
  }
  else
  {
    $confref = has::Common::hasGetClusterConfig($crsHome);
  }
  return $confref->{$crsHome}{vendor} if $confref and $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{vendor};
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{vendor} if $confref->{$ch}{vendor};
    }
  }
  warn "DEBUG:has::Common::hasGetClusterVendor Failed to get Vendor Name" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetIsVendorCluster
#
# DESC
#  return 1 if vendor cluster , 0 for oracle
#
# ARGUMENTS
# crsHome if known
# dynamicProperty - not null valud if function is called for dynamic prop 
#   computation
#
# RETURNS
# 1 or 0
#------------------------------------------------------------------------------
sub hasGetIsVendorCluster(;$$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{isvendorcw} 
   if $crsHome
   and $confref->{$crsHome}
   and defined $confref->{$crsHome}{isvendorcw};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{isvendorcw}
       if defined $confref->{$ch}{isvendorcw};
    }
  }
  if ( $dynProp )
  {
   $confref = has::Common::hasGetClusterConfigPre11g($crsHome,$dynProp);
  }
  else
  {
    $confref = has::Common::hasGetClusterConfig($crsHome);
  }
  return $confref->{$crsHome}{isvendorcw} 
   if $confref and $crsHome
   and $confref->{$crsHome}
   and defined $confref->{$crsHome}{isvendorcw};
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{isvendorcw} if defined $confref->{$ch}{nodename};
    }
  }
  warn "DEBUG:has::Common::hasGetIsVendorCluster Failed to check if Vendor clusterware" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterSoftwareVersion
#
# DESC
#  return the software version for a node
#
# ARGUMENTS
# node name or null for local node
# crsHome if known
# dynamicProperty - not null valud if function is called for dynamic prop 
#   computation
#
# RETURNS
#  cluster software version for that node
#------------------------------------------------------------------------------
sub hasGetClusterSoftwareVersion(;$$$)
{
  my ( $nodename,$crsHome,$dynProp ) = @_;
  my $activ_version;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{node}{details}{$nodename}{version}
    if $nodename and $crsHome 
    and $confref->{$crsHome}{node} 
    and $confref->{$crsHome}{node}{details}
    and $confref->{$crsHome}{node}{details}{$nodename} 
    and $confref->{$crsHome}{node}{details}{$nodename}{version};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
   for my $ch ( %{$confref} )
   {
     return $confref->{$ch}{node}{details}{$nodename}{version}
      if $nodename 
       and $confref->{$ch}{node} 
       and $confref->{$ch}{node}{details}
       and $confref->{$ch}{node}{details}{$nodename} 
       and $confref->{$ch}{node}{details}{$nodename}{version};
   }
  }
  if ( $dynProp )
  {
   $confref = has::Common::hasGetClusterConfigPre11g($crsHome,$dynProp);
   return $confref->{$crsHome}{node}{details}{$nodename}{version}
     if $nodename and $crsHome 
     and $confref->{$crsHome}{node} 
     and $confref->{$crsHome}{node}{details}
     and $confref->{$crsHome}{node}{details}{$nodename} 
     and $confref->{$crsHome}{node}{details}{$nodename}{version};
 
   if ( $confref and ref($confref) and keys %{$confref} )
   {
    for my $ch ( %{$confref} )
    {
      return $confref->{$ch}{node}{details}{$nodename}{version}
       if $nodename 
        and $confref->{$ch}{node} 
        and $confref->{$ch}{node}{details}
        and $confref->{$ch}{node}{details}{$nodename} 
        and $confref->{$ch}{node}{details}{$nodename}{version};
    }
   }
   $activ_version = has::Common::hasGetClusterActiveVersion($crsHome,$dynProp);
  }
  else
  {
    $confref = has::Common::hasGetClusterConfig($crsHome);
  
    return $confref->{$crsHome}{node}{details}{$nodename}{version}
      if $nodename and $crsHome 
      and $confref->{$crsHome}{node} 
      and $confref->{$crsHome}{node}{details}
      and $confref->{$crsHome}{node}{details}{$nodename} 
      and $confref->{$crsHome}{node}{details}{$nodename}{version};
  
    if ( $confref and $confref and ref($confref) and keys %{$confref} )
    {
      for my $ch ( keys %{$confref} )
      {
        return $confref->{$ch}{node}{details}{$nodename}{version}
          if $nodename 
          and $confref->{$ch}{node} 
          and $confref->{$ch}{node}{details}
          and $confref->{$ch}{node}{details}{$nodename} 
          and $confref->{$ch}{node}{details}{$nodename}{version};
      }
    }
  
    $activ_version = has::Common::hasGetClusterActiveVersion($crsHome);
  }
  # return active version if software version is not available
  if (  $activ_version )
  {
    warn "DEBUG:has::Common::hasGetClusterSoftwareVersion Failed to get software version for node $nodename, returning active version" if $nodename;
    warn "DEBUG:has::Common::hasGetClusterSoftwareVersion Failed to get software version for local node, returning active version"; 
    return $activ_version;
  }
  warn "WARN:has::Common::hasGetClusterSoftwareVersion Failed to get software version for node $nodename" 
   and return if $nodename;
  warn "WARN:has::Common::hasGetClusterSoftwareVersion Failed to get software version for local node" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    computeOcrType
#
# DESC
# return if OCR type is has or cluster
# based on ocr properties in 11.2 cluster and higher
#
# ARGUMENTS
# ref to config has for a cluster home
#
#------------------------------------------------------------------------------
# compute ocr type based on output from emcrsp
sub computeOcrType($)
{
 
  my ( $ref ) = @_;
  return unless $ref and keys %{$ref};
  return $ref->{ocr_type} if $ref->{ocr_type};
  return unless $ref->{ocr_configured} and $ref->{ocr_location};
  if ( $ref->{ocr_configured} =~ /TRUE/ and $ref->{ocr_location} =~ /CLUSTER/ )
  {
    $ref->{ocr_type} = 'cluster';
  }
  elsif ( $ref->{ocr_location} =~ /CLUSTER/ )
  {
    $ref->{ocr_type} = 'cluster';
  }
  elsif ( $ref->{ocr_location} =~ /LOCAL/ )
  {
    $ref->{ocr_type} = 'has';
  }
  else
  {
     $ref->{ocr_type} = 'cluster';
  }
}
#------------------------------------------------------------------------------
# FUNCTION :    hasEvalOcrType
#
# DESC
# return if OCR type is has or cluster
#
# ARGUMENTS
# crsHome if known
# dynamicProperty - not null valud if function is called for dynamic prop 
#   computation
#
#------------------------------------------------------------------------------
sub hasEvalOcrType($;$)
{
  my ( $crsHome,$dynProp ) = @_;
  warn "WARN:has::Common::hasEvalOcrType crsHome is not passed" and return unless $crsHome;
  return 'cluster' unless has::Common::hasCheckForEmcrsp($crsHome);
  my $confref = has::Common::getClusterCacheRef();
  if ( not defined $confref or not ref($confref) or not ref($confref) =~ /HASH/
         or not defined $confref->{$crsHome} )
  {
    # if this is for dynamic prop then get the emcrsp config information
    # only and not all the other config info
    $confref = has::Common::hasGetClusterConfigEmcrsp($crsHome);
  }
  return unless $confref and ref($confref) and ref($confref) =~ /HASH/ and $confref->{$crsHome};
  my $o;
  $o = has::Common::computeOcrType($confref->{$crsHome}) if $crsHome and $confref->{$crsHome};
  return $o if $o;
  return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetOcrType
#
# DESC
# return if OCR type is has or cluster
#
# ARGUMENTS
# crsHome if known
# dynamicProperty - not null valud if function is called for dynamic prop 
#   computation
#
#------------------------------------------------------------------------------
sub hasGetOcrType(;$$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $o;
  $o = has::Common::hasEvalOcrType($crsHome,$dynProp) if $crsHome;
  return $o if $o;
  #if no crs home is passed go thru all the discovered crs homes
  my $confref = has::Common::getClusterCacheRef(); 
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      $o = has::Common::hasEvalOcrType($ch,$dynProp);
      return $o if $o;
    }
  }
  # if this is for dynamic prop then get the emcrsp config information
  # only and not all the other config info
  $confref = has::Common::hasGetClusterConfigEmcrsp($crsHome);
  $o = has::Common::hasEvalOcrType($crsHome,$dynProp) if $crsHome; 
  return $o if $o;
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      $o = has::Common::hasEvalOcrType($ch,$dynProp);
      return $o if $o;
    }
  }
  warn "DEBUG:has::Common::hasGetOcrType:Failed to get OCR type, defaulting to a cluster";
  return 'cluster';
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetClusterNodes
#
# DESC
# return a has list of cluster nodes with config information
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# return a hash of nodes
#------------------------------------------------------------------------------
sub hasGetClusterNodes(;$)
{
  my ( $crsHome ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{node} 
   if $crsHome
   and $confref and ref($confref) and keys %{$confref} 
   and $confref->{$crsHome}
   and $confref->{$crsHome}{node}
   and $confref->{$crsHome}{node}{details};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{node}
       if   $confref->{$ch}
       and  $confref->{$ch}{node}
       and  $confref->{$ch}{node}{details};
    }
  }
  $confref = has::Common::hasGetClusterConfig($crsHome);
  return $confref->{$crsHome}{node} 
   if $confref and $crsHome
   and $confref->{$crsHome}
   and $confref->{$crsHome}{node}
   and $confref->{$crsHome}{node}{details};
  if ( $confref and $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{node} 
       if  $confref->{$ch}{node} and $confref->{$ch}{node}{details};
    }
  }
  warn "DEBUG:has::Common::hasGetClusterNodes:Failed to get the list of nodes for cluster" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetGnsInformation
#
# DESC
#  return the Gns Information
#
# ARGUMENTS
# crsHome 
#
# RETURNS
# hash for results
#------------------------------------------------------------------------------
sub hasGetGnsInformation(;$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{gns} 
   if $crsHome
   and $confref and ref($confref) and keys %{$confref} 
   and $confref->{$crsHome}
   and $confref->{$crsHome}{gns};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{gns}
       if   $confref->{$ch}
       and  $confref->{$ch}{gns};
    }
  }
  if ( not has::Common::hasCheckForEmcrsp($crsHome) )
  {
    warn "INFO:has::Common::hasGetGnsInformation:This is a pre 11gR2 cluster, binary emcrsp is not found, no GNS information";
    return;
  }
  my $ocrtype = has::Common::hasGetOcrType($crsHome,$dynProp);
  if ( $ocrtype and $ocrtype =~ /^has$/ )
  {
    warn "DEBUG:has::Common::hasGetGnsInformation:this is a Oracle restart install, no GNS information";
    return;
  }
  # get the scan information
  my %meta; 
  $meta{cmd}='emcrsp em config -e resource -p ora.gns.type';
  $meta{element_name}='entity';
  $meta{entity}{attrs}{entity_name}=1;
  $meta{entity}{name_value}{NAME}=1;
  
  my $gnslistref;
  $gnslistref = has::Common::hasGetEntityInformation(\%meta,$crsHome);
  # get the scan vip information
  %meta = ();
  $meta{cmd}='emcrsp em config -e resource -p ora.gns_vip.type';
  $meta{element_name}='entity';
  $meta{entity}{attrs}{entity_name}=1;
  $meta{entity}{name_value}{NAME}=1;
  $meta{entity}{name_value}{USR_ORA_VIP}=1;
  my $gnsvipref;
  $gnsvipref = has::Common::hasGetEntityInformation(\%meta,$crsHome);
  my %gnsres;
  
  # if we have eons data then farm the fields to make sure we have reqd data
  if ( $gnslistref and ref($gnslistref) and ref($gnslistref) =~ /HASH/ and keys  %{$gnslistref}  )
  {
    for my $id ( keys %{$gnslistref} )
    {
      $gnsres{GNS_NAME} = $gnslistref->{$id}{NAME} if defined $gnslistref->{$id}{NAME};
      last if $gnsres{GNS_NAME};
    }
  }
  if ( $gnsvipref and ref($gnsvipref) and ref($gnsvipref) =~ /HASH/ and keys  %{$gnsvipref}  )
  {
    for my $id ( keys %{$gnsvipref} )
    {
      $gnsres{GNS_VIP_NAME} = $gnsvipref->{$id}{NAME} if $gnsvipref->{$id}{NAME};
      $gnsres{GNS_VIP_IP} = $gnsvipref->{$id}{USR_ORA_VIP} if $gnsvipref->{$id}{USR_ORA_VIP};
      last if $gnsres{GNS_VIP_NAME} and $gnsres{GNS_VIP_IP};
    }
  }
  warn "DEBUG:has::Common::hasGetGnsInformation Failed to get NAME for GNS" unless $gnsres{GNS_NAME};
  warn "DEBUG:has::Common::hasGetGnsInformation Failed to get GNS_VIP_NAME for GNS" unless $gnsres{GNS_VIP_NAME};
  warn "DEBUG:has::Common::hasGetGnsInformation Failed to get GNS_VIP_IP for GNS" unless $gnsres{GNS_VIP_IP};
  # if gns is configured then try to get the gns domain using srvctl commands
  if ( keys %gnsres and $gnsres{GNS_NAME} )
  {
     my $gnsdomain;
     has::Common::hasSetCRSEnv($crsHome);
     my %command_args = (exit_failure_list => [()]);
     my $srvgnsres = has::Common::runsystemcommand('srvctl','config gns -d',\%command_args);
     has::Common::hasRestoreCRSEnv();
     #command failed so no gns domain
     if ( $command_args{command_return_status}  )
     {
      warn "DEBUG:has::Common::hasGetGnsInformation Failed to get DOMAIN for GNS using command srvctl config gns -d";
     }
     elsif ( $srvgnsres )
     {
       $srvgnsres =~ s/^\s+|\s+$// if $srvgnsres;
       
        # we look for the english string, if not for nls we go with the :
       ($gnsdomain) = ( $srvgnsres =~ /Domain served by GNS\s*:(.+)/i ) if $srvgnsres and $srvgnsres =~ /Domain served by GNS\s*:/;
       $gnsdomain =~ s/^\s+|\s+$// if $gnsdomain;
       if ( not $gnsdomain and $srvgnsres and $srvgnsres =~ /:/ )
       {
        ($gnsdomain) = ( $srvgnsres =~ /.+:([^:]+)$/i );
         $gnsdomain =~ s/^\s+|\s+$// if $gnsdomain;
       }
       $gnsdomain =~ s/^\.// if $gnsdomain =~ /^\./;
       $gnsres{GNS_DOMAIN} = $gnsdomain if $gnsdomain;
     }
     # if srvctl fails to give gns domain then use resolve on gns ip
     if ( $gnsres{GNS_VIP_IP} and not $gnsres{GNS_DOMAIN} )
     {  
       $gnsdomain = has::Common::hasGetDomainNameNSLookup($gnsres{GNS_VIP_IP});
       $gnsres{GNS_DOMAIN} = $gnsdomain if $gnsdomain;
     }
  }
  if ( $crsHome )
  {
    $confref->{$crsHome}{gns}=\%gnsres;
  }
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      $confref->{$ch}{gns}=\%gnsres;
    }
  }
  return \%gnsres if keys %gnsres;
  warn "DEBUG:has::Common::hasGetGnsInformation Failed to get Information for GNS resource" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetScanInformation
#
# DESC
#  return the scan Information
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# hash for results
#------------------------------------------------------------------------------
sub hasGetScanInformation(;$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{scan} 
   if $crsHome
   and $confref and ref($confref) and keys %{$confref} 
   and $confref->{$crsHome}
   and $confref->{$crsHome}{scan};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{scan}
       if   $confref->{$ch}
       and  $confref->{$ch}{scan};
    }
  }
  if ( not has::Common::hasCheckForEmcrsp($crsHome) )
  {
    warn "INFO:has::Common::hasGetScanInformation:This is a pre 11gR2 cluster, binary emcrsp is not found, no Scan information";
    return;
  }
  my $ocrtype = has::Common::hasGetOcrType($crsHome,$dynProp);
  if ( $ocrtype and $ocrtype =~ /^has$/ )
  {
    warn "DEBUG:has::Common::hasGetScanInformation:this is a Oracle restart install, no Scan information";
    return;
  }
  # get the scan information
  my %meta; 
  $meta{cmd}='emcrsp em config -e resource -p ora.scan_listener.type';
  $meta{element_name}='entity';
  $meta{entity}{attrs}{entity_name}=1;
  $meta{entity}{name_value}{PORT}=1;
  $meta{entity}{name_value}{NAME}=1;
  
  my $scanlistref;
  $scanlistref = has::Common::hasGetEntityInformation(\%meta,$crsHome);
  # get the scan vip information
  %meta = ();
  $meta{cmd}='emcrsp em config -e resource -p ora.scan_vip.type';
  $meta{element_name}='entity';
  $meta{entity}{attrs}{entity_name}=1;
  $meta{entity}{name_value}{SCAN_NAME}=1;
  $meta{entity}{name_value}{USR_ORA_VIP}=1;
  my $scanvipref;
  $scanvipref = has::Common::hasGetEntityInformation(\%meta,$crsHome);
  my %scanres;
  
  # if we have eons data then farm the fields to make sure we have reqd data
  if ( $scanlistref and ref($scanlistref) and ref($scanlistref) =~ /HASH/ and keys  %{$scanlistref}  )
  {
    for my $id ( keys %{$scanlistref} )
    {
      $scanres{SCAN_PORT} = $scanlistref->{$id}{PORT} if defined $scanlistref->{$id}{PORT};
      $scanres{RESOURCE_NAME} = $scanlistref->{$id}{NAME} if defined $scanlistref->{$id}{NAME};
      $scanres{RESOURCE_NAME} = $scanlistref->{$id}{entity_name} if defined $scanlistref->{$id}{entity_name} and not $scanres{RESOURCE_NAME};
      last if $scanres{SCAN_PORT} and $scanres{RESOURCE_NAME};
    }
  }
  if ( $scanvipref and ref($scanvipref) and ref($scanvipref) =~ /HASH/ and keys  %{$scanvipref}  )
  {
    for my $id ( keys %{$scanvipref} )
    {
      $scanres{SCAN_NAME} = $scanvipref->{$id}{SCAN_NAME} if $scanvipref->{$id}{SCAN_NAME};
      $scanres{USE_ORA_VIP} = $scanvipref->{$id}{USR_ORA_VIP} if $scanvipref->{$id}{USR_ORA_VIP};
      last if $scanres{SCAN_NAME} and $scanres{USE_ORA_VIP};
    }
  }
  warn "DEBUG:has::Common::hasGetScanInformation Failed to get SCAN PORT for scan listener" unless $scanres{SCAN_PORT};
  warn "DEBUG:has::Common::hasGetScanInformation Failed to get NAME for scan" unless $scanres{SCAN_NAME};
  warn "DEBUG:has::Common::hasGetScanInformation Failed to get USE_ORA_VIP for scan" unless $scanres{USR_ORA_VIP};
  if ( $scanres{SCAN_NAME}  and $scanres{SCAN_NAME} !~  /\./ )
  {
   # check if gns is defined and get the gns domain
   my $gnsref = has::Common::hasGetGnsInformation($crsHome);
   if ( $gnsref and ref($gnsref) and ref($gnsref) =~ /HASH/i and keys %{$gnsref} and $gnsref->{GNS_NAME} )
   {
     $scanres{SCAN_NAME_WITH_FQDN} = "$scanres{SCAN_NAME}.$gnsref->{GNS_DOMAIN}" if $gnsref->{GNS_DOMAIN};
   }
   else
   {
     # if not gns get domain name for scan, using scan address using net utilities nslookup
     my $scanDomain;
     $scanDomain = has::Common::hasGetDomainNameNSLookup($scanres{USE_ORA_VIP}) if $scanres{USR_ORA_VIP};
     $scanDomain = has::Common::hasGetDomainNameNSLookup($scanres{SCAN_NAME}) unless $scanDomain;
     if ( $scanDomain )
     {
      $scanres{SCAN_NAME_WITH_FQDN} = "$scanres{SCAN_NAME}.$scanDomain";
     }
     else 
     {
      my $hostDomain = has::Common::hasGetLocalHostDomain();
      if ( $hostDomain )
      {
       $scanres{SCAN_NAME_WITH_FQDN} = "$scanres{SCAN_NAME}.$hostDomain";
      }
      else
      {
       warn "DEBUG:has::Common::hasGetScanInformation Failed to domain for scan";
      }
     }
   }
  }
  $scanres{SCAN_PORT} = 0 unless $scanres{SCAN_PORT};
  if ( $crsHome )
  {
    $confref->{$crsHome}{scan}=\%scanres;
  }
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      $confref->{$ch}{scan}=\%scanres;
    }
  }
  return \%scanres if keys %scanres;
  warn "DEBUG:has::Common::hasGetScanInformation Failed to get Information for Scan resource" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetScanVIPInformation
#
# DESC
#  return the scan VIP Information
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# hash for results
#------------------------------------------------------------------------------
sub hasGetScanVIPInformation(;$)
{
  my ( $crsHome ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{scan_vip} 
   if $crsHome
   and $confref and ref($confref) and keys %{$confref} 
   and $confref->{$crsHome}
   and $confref->{$crsHome}{scan_vip};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{scan_vip}
       if   $confref->{$ch}
       and  $confref->{$ch}{scan_vip};
    }
  }
  if ( not has::Common::hasCheckForEmcrsp($crsHome) )
  {
    warn "INFO:has::Common::hasGetScanVIPInformation:This is a pre 11gR2 cluster, binary emcrsp is not found, no Scan information";
    return;
  }
  # get the scan information
  my %meta; 
  $meta{cmd}='emcrsp em config -e resource -p ora.scan_vip.type';
  $meta{element_name}='entity';
  $meta{entity}{attrs}{entity_name}=1;
  $meta{entity}{name_value}{NAME}=1;
  $meta{entity}{name_value}{SCAN_NAME}=1;
  $meta{entity}{name_value}{ID}=1;
  $meta{entity}{name_value}{USR_ORA_VIP}=1;
  $meta{entity}{name_value}{TYPE}=1;
  $meta{entity}{name_value}{BASE_TYPE}=1;
  $meta{entity}{name_value}{HOSTING_MEMBERS}=1;
  $meta{entity}{name_value}{LAST_SERVER}=1;
  $meta{entity}{name_value}{VERSION}=1;
  
  my $resref;
  $resref = has::Common::hasGetEntityInformation(\%meta,$crsHome);
  # if we have eons data then farm the fields to make sure we have reqd data
  if ( keys  %{$resref}  )
  {
    for my $id ( keys %{$resref} )
    {
      $resref->{$id}{SCAN_IP} = $resref->{$id}{USR_ORA_VIP} if defined $resref->{$id}{USR_ORA_VIP};
      $resref->{$id}{SCAN_IP}='scanIP_TBD' unless $resref->{$id} and $resref->{$id}{SCAN_IP};
      warn "DEBUG:has::Common::hasGetScanVIPInformation Failed to get SCAN IP for scan VIP resource" 
       unless $resref->{$id} and $resref->{$id}{SCAN_IP};
    }
  }
  else
  {
   warn "DEBUG:has::Common::hasGetScanVIPInformation Failed to get information for scan vip resource";
   $resref->{NO_ID}{SCAN_IP}='scanIP_TBD';
  }
  if ( $crsHome )
  {
    for my $id ( keys %{$resref} )
    {
      $confref->{$crsHome}{scan_vip}=$resref->{$id};
    }
    return $confref->{$crsHome}{scan_vip};
  }
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      for my $id ( keys %{$resref} )
      {
        $confref->{$ch}{scan_vip}=$resref->{$id};
      }
      return $confref->{$ch}{scan_vip}
       if   $confref->{$ch}
       and  $confref->{$ch}{scan_vip};
    }
  }
  return $resref if $resref and keys %{$resref};
  warn "DEBUG:has::Common::hasGetScanVIPInformation Failed to get Information for Scan VIP resource" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetScanPort
#
# DESC
#  return the scanPort
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# scanPort
#------------------------------------------------------------------------------
sub hasGetScanPort(;$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $ref;
  $ref = has::Common::hasGetScanInformation($crsHome,$dynProp);
  return $ref->{SCAN_PORT} if defined $ref->{SCAN_PORT};
  warn "DEBUG:has::Common::hasGetScanPort Failed to get Scan Port" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetScanName
#
# DESC
#  return the scanName
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# scanPort
#------------------------------------------------------------------------------
sub hasGetScanName(;$)
{
  my ( $crsHome,$dynProp ) = @_;
  my $ref;
  $ref = has::Common::hasGetScanInformation($crsHome,$dynProp);
  return $ref->{SCAN_NAME_WITH_FQDN} if defined $ref->{SCAN_NAME_WITH_FQDN};
  return $ref->{SCAN_NAME} if defined $ref->{SCAN_NAME};
  warn "DEBUG:has::Common::hasGetScanPort Failed to get Scan Name" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetScanIP
#
# DESC
#  return the scan IP Addres
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# scanPort
#------------------------------------------------------------------------------
sub hasGetScanIP(;$)
{
  my ( $crsHome ) = @_;
  my $ref;
  $ref = has::Common::hasGetScanVIPInformation($crsHome);
  return $ref->{SCAN_IP} if defined $ref->{SCAN_IP};
  warn "DEBUG:has::Common::hasGetScanPort Failed to get Scan IP" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetEONSInformation
#
# DESC
#  return the eONS Information
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# has to be returned
#------------------------------------------------------------------------------
sub hasGetEONSInformation(;$)
{
  my ( $crsHome ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{eons} 
   if $crsHome
   and $confref and ref($confref) and keys %{$confref} 
   and $confref->{$crsHome}
   and $confref->{$crsHome}{eons};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{eons}
       if   $confref->{$ch}
       and  $confref->{$ch}{eons};
    }
  }
  if ( not has::Common::hasCheckForEmcrsp($crsHome) )
  {
    warn "INFO:has::Common::hasGetEONSInformation:This is a pre 11gR2 cluster, binary emcrsp is not found, no eons information";
    return;
  }
  # get the eons information
  my %meta; 
  $meta{cmd}='emcrsp em config -e resource_instance -p ora.eons.type';
  $meta{element_name}='entity';
  $meta{entity}{attrs}{entity_name}=1;
  $meta{entity}{name_value}{NAME}=1;
  $meta{entity}{name_value}{ID}=1;
  $meta{entity}{name_value}{LAST_SERVER}=1;
  $meta{entity}{name_value}{PORT}=1;
  $meta{entity}{name_value}{TYPE}=1;
  $meta{entity}{name_value}{BASE_TYPE}=1;
  $meta{entity}{name_value}{HOSTING_MEMBERS}=1;
  
  my $resref;
  $resref = has::Common::hasGetEntityInformation(\%meta,$crsHome);
  # if we have eons data then farm the fields to make sure we have reqd data
  if ( keys  %{$resref}  )
  {
    for my $id ( keys %{$resref} )
    {
      $resref->{$id}{ID} = $resref->{$id}{entity_name} if not $resref->{$id}{ID} and $resref->{entity_name}; 
      # split id of form 'ora.eons stbdq16 1
      my $name;
      my $node;
      my $instance;
      ($name,$node, $instance) = ( $resref->{$id}{ID} =~ /^([^\s]+)\s+([^\s]+)\s+([^\s]*)/) 
        if $resref->{$id}{ID} and $resref->{$id}{ID} =~ /^[^\s]+\s+[^\s]+\s+[^\s]*/;
      $resref->{$id}{NAME} = $name if $name and not $resref->{$id}{NAME}; 
      $resref->{$id}{NAME} = $resref->{$id}{ID} if $resref->{$id}{ID} and not $resref->{$id}{NAME};
      $resref->{$id}{NAME} = 'ora.eons UNKNOWN' unless $resref->{$id}{NAME};
      $resref->{$id}{EONS_PORT} = $resref->{$id}{PORT} if defined $resref->{$id}{PORT};
      $resref->{$id}{EONS_PORT} = 0 unless defined $resref->{$id}{EONS_PORT};
      $resref->{$id}{NODE_NAME} = $resref->{$id}{LAST_SERVER} if $resref->{$id}{LAST_SERVER};
      $resref->{$id}{NODE_NAME} = $node if $node and not $resref->{$id}{NODE_NAME};
      $resref->{$id}{NODE_NAME} = "$resref->{$id}{NAME} $instance" if $resref->{$id}{NAME} and $instance and not $resref->{$id}{NODE_NAME};
      $resref->{$id}{NODE_NAME} = $resref->{$id}{ID} if $resref->{$id}{ID} and not $resref->{$id}{NODE_NAME};
      $resref->{$id}{NODE_NAME} = $resref->{$id}{HOSTING_MEMBERS} if $resref->{$id}{HOSTING_MEMBERS} and not $resref->{$id}{NODE_NAME};
      $resref->{$id}{NODE_NAME} = "UNKNOWN $instance" if $instance and not $resref->{$id}{NODE_NAME};
      
      warn "DEBUG:has::Common::hasGetEONSInformation Failed to get NODE_NAME for eons resource" unless $resref->{$id}{NODE_NAME};
      warn "DEBUG:has::Common::hasGetEONSInformation Failed to get NAME for eons resource" unless $resref->{$id}{NAME};
    }
    if ( $crsHome )
    {
      $confref->{$crsHome}{eons}=$resref;
      return $confref->{$crsHome}{eons};
    }
    if ( $confref and ref($confref) and keys %{$confref} )
    {
      for my $ch ( keys %{$confref} )
      {
        $confref->{$ch}{eons}=$resref;
  
        return $confref->{$ch}{eons}
         if   $confref->{$ch}
         and  $confref->{$ch}{eons};
      }
    }
  }
  return $resref if $resref and keys %{$resref};
  warn "DEBUG:has::Common::hasGetEONSInformation Failed to get eONS Information" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetEONSPort
#
# DESC
#  return the eONS Port
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# eons port to be returned
#------------------------------------------------------------------------------
sub hasGetEONSPort(;$)
{
  my ( $crsHome ) = @_;
  my $eonsref = has::Common::hasGetEONSInformation($crsHome);
  my $eonsPort;
  if ( $eonsref and ref($eonsref) )
  {
    for my $id ( keys %{$eonsref} )
    {
      $eonsPort = $eonsref->{$id}{EONS_PORT} if $eonsref->{$id}{EONS_PORT};
      $eonsPort = 0 unless $eonsPort;
    }
  }
  return $eonsPort if defined $eonsPort;
  warn "DEBUG:has::Common::hasGetEONSPort Failed to get eONS port" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetVIPInformation
#
# DESC
#  return the VIP Information
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# has to be returned
#
#------------------------------------------------------------------------------
sub hasGetVIPInformation(;$)
{
  my ( $crsHome ) = @_;
  my $resref;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{vip} 
   if $crsHome
   and $confref and ref($confref) and keys %{$confref} 
   and $confref->{$crsHome}
   and $confref->{$crsHome}{vip};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{vip}
       if   $confref->{$ch}
       and  $confref->{$ch}{vip};
    }
  }
  if ( not has::Common::hasCheckForEmcrsp($crsHome) )
  {
    warn "INFO:has::Common::hasGetVIPInformation:This is a pre 11gR2 cluster, binary emcrsp is not found";
    return ;
  }
  # get the vip information
  my %meta; 
  $meta{cmd}='emcrsp em config -e resource -p ora.cluster_vip_net1.type';
  $meta{element_name}='entity';
  $meta{entity}{attrs}{entity_name}=1;
  $meta{entity}{name_value}{NAME}=1;
  $meta{entity}{name_value}{ID}=1;
  $meta{entity}{name_value}{LAST_SERVER}=1;
  $meta{entity}{name_value}{HOSTING_MEMBERS}=1;
  $meta{entity}{name_value}{USR_ORA_VIP}=1;
  $meta{entity}{name_value}{TYPE}=1;
  $meta{entity}{name_value}{BASE_TYPE}=1;
  $meta{entity}{name_value}{HOSTING_MEMBERS}=1;
  
  $resref = has::Common::hasGetEntityInformation(\%meta,$crsHome);
  # if we have eons data then farm the fields to make sure we have reqd data
  if ( keys  %{$resref}  )
  {
    for my $id ( keys %{$resref} )
    {
      $resref->{$id}{ID} = $resref->{$id}{entity_name} if not $resref->{$id}{ID} and $resref->{entity_name}; 
      # split id of form 'ora.stbdq16.vip 1 1 12
      my $node;
      my $name;
      my $instance;
      ($name, $instance) = ( $resref->{$id}{ID} =~ /^([^\s]+)\s+([^\s]+)/) 
        if $resref->{$id}{ID} and $resref->{$id}{ID} =~ /^[^\s]+\s+[^\s]+/;
      $resref->{$id}{NAME} = $name if not $resref->{$id}{NAME} and $name; 
      $resref->{$id}{NAME} = $resref->{$id}{ID} if $resref->{$id}{ID} and not $resref->{$id}{NAME}; 
      $resref->{$id}{NAME} = 'ora.cluster_vip UNKNOWN' unless $resref->{$id}{NAME};
      $resref->{$id}{VIP_IP} = $resref->{$id}{USR_ORA_VIP} if defined $resref->{$id}{USR_ORA_VIP};
      $resref->{$id}{VIP_IP} = 0 unless defined $resref->{$id}{VIP_IP};
      ($node) = ( $name =~ /ora\.([^'\.vip']+)\.vip/) if $name and $name =~ /ora\..+\.vip/;
      $resref->{$id}{NODE_NAME} = $resref->{$id}{LAST_SERVER} if $resref->{$id}{LAST_SERVER};
      $resref->{$id}{NODE_NAME} = $resref->{$id}{HOSTING_MEMBERS} 
       if $resref->{$id}{HOSTING_MEMBERS} and not $resref->{$id}{NODE_NAME};
      $resref->{$id}{NODE_NAME} = $node if $node and not $resref->{$id}{NODE_NAME};
      $resref->{$id}{NODE_NAME} = "$resref->{$id}{NAME} $instance" 
       if $resref->{$id}{NAME} and $instance and not $resref->{$id}{NODE_NAME};
      $resref->{$id}{NODE_NAME} = $resref->{$id}{ID} if $resref->{$id}{ID} and not $resref->{$id}{NODE_NAME};
      $resref->{$id}{NODE_NAME} = "UNKNOWN $instance" if $instance and not $resref->{$id}{NODE_NAME};
      
      warn "DEBUG:has::Common::hasGetVIPInformation Failed to get NODE_NAME for vip resource" 
       unless $resref->{$id}{NODE_NAME};
      warn "DEBUG:has::Common::hasGetVIPInformation Failed to get NAME for vip resource" 
       unless $resref->{$id}{NAME};
      warn "DEBUG:has::Common::hasGetVIPInformation Failed to get IP Address for vip resource" 
       unless $resref->{$id}{VIP_IP};
    }
    if ( $crsHome )
    {
      $confref->{$crsHome}{vip}=$resref;
      return $confref->{$crsHome}{vip};
    }
    if ( $confref and ref($confref) and keys %{$confref} )
    {
      for my $ch ( keys %{$confref} )
      {
        $confref->{$ch}{vip}=$resref;
  
        return $confref->{$ch}{vip}
         if   $confref->{$ch}
         and  $confref->{$ch}{vip};
      }
    }
  }
  return $resref if $resref and keys %{$resref};
  warn "DEBUG:has::Common::hasGetVIPInformation Failed to get VIP Information" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetVotingDiskInformation
#
# DESC
#  return the Voting Disk Information
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# hash for results
#------------------------------------------------------------------------------
sub hasGetVotingDiskInformation(;$)
{
  my ( $crsHome ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{votingdisk} 
   if $crsHome
   and $confref and ref($confref) and keys %{$confref} 
   and $confref->{$crsHome}
   and $confref->{$crsHome}{votingdisk};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{votingdisk}
       if   $confref->{$ch}
       and  $confref->{$ch}{votingdisk};
    }
  }
  # get the scan information
  my %results;
  $results{'voting_disk_location_tbd'}{LOCATION}='voting_disk_location_tbd';
  return \%results;
  warn "DEBUG:has::Common::hasGetVotingDiskInformation Failed to get Voting Disk Information " and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetOCRInformation
#
# DESC
#  return the OCR Information
#
# ARGUMENTS
# crsHome if known
#
# RETURNS
# hash for results
#------------------------------------------------------------------------------
sub hasGetOCRInformation(;$)
{
  my ( $crsHome ) = @_;
  my $confref = has::Common::getClusterCacheRef();
  return $confref->{$crsHome}{OCR} 
   if $crsHome
   and $confref and ref($confref) and keys %{$confref} 
   and $confref->{$crsHome}
   and $confref->{$crsHome}{OCR};
  if ( $confref and ref($confref) and keys %{$confref} )
  {
    for my $ch ( keys %{$confref} )
    {
      return $confref->{$ch}{OCR}
       if   $confref->{$ch}
       and  $confref->{$ch}{OCR};
    }
  }
  # get the scan information
  my %results;
  $results{'ocr_location_tbd'}{LOCATION}='ocr_location_tbd';
  $results{'ocr_location_tbd'}{MIRRORS}=1;
  return \%results;
  warn "DEBUG:has::Common::hasGetOCRInformation Failed to get Voting Disk Information " and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetDBResourceInformation
#
# DESC
#   use emcrsp to get resource information for a db resource type
#
# ARGUMENTS
# crsHome 
#
# RETURNS
# hash for database attributes by database resource name
#------------------------------------------------------------------------------
sub hasGetDBResourceInformation($)
{
  my ( $crsHome ) = @_;
  if ( not has::Common::hasCheckForEmcrsp($crsHome) )
  {
    warn "INFO:has::Common::hasGetDBResourceInformation:This is a pre 11gR2 cluster, binary emcrsp is not found, no Scan information";
    return;
  }
  # get the db target type information
  my %meta; 
  $meta{cmd}='emcrsp em config -e resource -p ora.database.type';
  $meta{element_name}='entity';
  $meta{entity}{attrs}{entity_name}=1;
  $meta{entity}{name_value}{USR_ORA_DB_UNIQUE_NAME}=1;
  $meta{entity}{name_value}{USR_ORA_INST_NAME}=1;
  $meta{entity}{name_value}{NAME}=1;
  $meta{entity}{name_value}{ORACLE_HOME}=1;
  
  my $resref;
  $resref = has::Common::hasGetEntityInformation(\%meta,$crsHome);
  return $resref if $resref and keys %{$resref};
  warn "DEBUG:has::Common::hasGetDBResourceInformation Failed to get database resource information from crs $crsHome" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetLsnrResourceInformation
#
# DESC
#   
#   use emcrsp to get resource information for a listener target type
#
# ARGUMENTS
# crsHome 
#
# RETURNS
# hash for listener attributes by listener resource name
#------------------------------------------------------------------------------
sub hasGetLsnrResourceInformation($)
{
  my ( $crsHome ) = @_;
  if ( not has::Common::hasCheckForEmcrsp($crsHome) )
  {
    warn "INFO:has::Common::hasGetLsnrResourceInformation:This is a pre 11gR2 cluster, binary emcrsp is not found, no Scan information";
    return;
  }
  # get the lsnr target type information
  my %meta; 
  $meta{cmd}='emcrsp em config -e resource -p ora.listener.type';
  $meta{element_name}='entity';
  $meta{entity}{attrs}{entity_name}=1;
  $meta{entity}{name_value}{PORT}=1;
  
  my $resref;
  $resref = has::Common::hasGetEntityInformation(\%meta,$crsHome);
  return $resref if $resref and keys %{$resref};
  warn "DEBUG:has::Common::hasGetLsnrResourceInformation Failed to get database resource information from crs $crsHome" and return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasIsDBManagedByHas
#
# DESC
#  return FALSE or HAS Name
#
# ARGUMENTS
# username
# password
# connect string
# role
# dbOracleHome
# dbVersion - this helps in choosing the righ sql if sqls are versioned
#
# RETURNS
#  return FALSE or HAS Name
#------------------------------------------------------------------------------
#------------------------------------------------------------------------------
# FUNCTION :    hasIsDBManagedByHas
#
# DESC
#  return FALSE or HAS Name
#
# ARGUMENTS
# username
# password
# connect string
# role
# dbOracleHome
# sid
# dbVersion - this helps in choosing the righ sql if sqls are versioned
#
# RETURNS
#  return FALSE or HAS Name
#------------------------------------------------------------------------------
sub hasIsDBManagedByHas($$$$;$$$)
{
   my ( $username,$password,$address,$role,$dbOracleHome,$sid,$dbVersion ) = @_;
   my $dynProp = 'TRUE';
   # define the other configuration variables
   my $sql=  
           "
       SELECT \'YES\'\"has_db\"
         FROM v\$session vs /*+ all_rows use_concat */
        WHERE (vs.client_info = \'oraagent\')
          AND ROWNUM = 1";      
   my $sql_par = "SELECT parallel \"parallel\" FROM v\$instance";
   my $dbsql = "
           SELECT database_name \"db_name\"
           FROM
           (
             SELECT name,
                    value database_name 
             FROM   v\$parameter
             WHERE (name = \'db_unique_name\'
                    OR name = \'db_name\')
              AND value IS NOT NULL
              ORDER BY name DESC
            )
            WHERE ROWNUM = 1";
   # order of fields in the results
   my %fieldOrder;
   $fieldOrder{$sql} = {has_db=>1 };
   $fieldOrder{$sql_par} = {parallel=>1 };
   $fieldOrder{$dbsql} = { db_name =>1 };
   # pupulate the credentials array
   my %credentials;
   $credentials{username} = $username if $username;
   $credentials{password} = $password if $password;
   $credentials{address} = $address if $address;
   $credentials{role} = $role if $role;
   $credentials{sql} = [ ( $sql,$sql_par,$dbsql ) ];
   #-----------------------------------------------------------------------------------------
   # subs declared
   my $hasDB = 'NO';
   my $parallel = 'NO';
   my $ocrType = 'cluster';
   my $clusterOcr = 1;
   my $hasManaged = 'N/A';
   my $oracleName = $ENV{EM_ORACLE_SID} if $ENV{EM_ORACLE_SID};
   # if the verson category is less than 11gR2 then db cannot be HA managed
   if ( keys %ENV and $ENV{EM_VERSION_CATEGORY} and $ENV{EM_VERSION_CATEGORY} =~ /^(pre8|8i|8iR2|9i|9iR2|10gR1|10gR2|10gR203|11gR1)$/ )
   {
       return wantarray? ($hasManaged,undef):$hasManaged;
   }
   my $results_ref = has::Common::hasGetSQLResults(\%credentials,\%fieldOrder);
   my $array_ref;
   if ( $results_ref and keys %{$results_ref} )
   {
    for my $sq ( keys %{$results_ref} )
    {
  
      # check if db is had managed
      $array_ref = $results_ref->{$sq};
    
      if ( $array_ref and ref($array_ref) 
           and ref($array_ref) =~ /LIST|ARRAY/ 
           and @{$array_ref} 
         )
      {
         # parse query results to check if db has has
         for my $row ( @{$array_ref} )
         {
           my %rowhash = %{$row};
   
           $hasDB = $rowhash{has_db} if $rowhash{has_db};
           $parallel = $rowhash{parallel} if $rowhash{parallel};
           $oracleName = $rowhash{db_name} if $rowhash{db_name};
   
           #last if $hasDB and $parallel;
         }
      }
    }
   }
   # cluster database is not has managed
   if ( $parallel and $parallel =~ /YES/i )
   {
       $hasManaged = 'N/A';
       return wantarray? ($hasManaged,undef):$hasManaged;
   }
  #bug fix 9350140
   # added function to check id has/cluster inventory exists
   my $hasclcheck = has::Common::hasCheckForHasClusterInventory();
   if ( $hasclcheck and ref($hasclcheck) and ref($hasclcheck) =~ /HASH/i and keys %{$hasclcheck}
        and $hasclcheck->{type} and $hasclcheck->{type} =~ /has|cluster/i
        and $hasclcheck->{oracle_home} )
   {
    warn "DEBUG:has::Common::hasIsDBManagedByHas an $hasclcheck->{type} install is present on this host at $hasclcheck->{oracle_home}\n";
   }
   else
   {
     # there is no has/cluster installed on this box with inventory
     $hasManaged = 'N/A';
     return wantarray? ($hasManaged,undef):$hasManaged;
   }
   # if not single instance has managed database
   # check if it has a resource name
   my $resourceName = undef;
   if ( not $hasDB or $hasDB !~ /^YES$/i )
   {
     $resourceName = has::Common::hasGetResourceNameForOracleApp($dbOracleHome,'database',$oracleName);
   }
   my $cHome = undef;
   my $HasManagedCRSHome = undef;
   # failed to get from db try to find a crs home and get information  
   my $discResultsref = has::Common::hasDiscoverCluster($dynProp);
    
   if ( $discResultsref and keys %{$discResultsref} )
   {
     for my $crsHome ( keys %{$discResultsref} )
     {
      # code below this will not execute on nt as emcrsp is not in the nt shiphome for beta1 
      if (  has::Common::hasCheckForEmcrsp($crsHome) )
      {
         $cHome = $crsHome;
         $ocrType = has::Common::hasGetOcrType($crsHome,$dynProp);
         # there is a single instance ha on the box but the db table shows none
         if ( $ocrType and $ocrType =~ /has/i )
         {
           $HasManagedCRSHome = $crsHome;
             my $db_ref = has::Common::hasGetDBResourceInformation($crsHome);
  
             if ( $db_ref and keys %{$db_ref} )
             {
                for my $dbres ( keys %{$db_ref} )
                {
                   if ( $db_ref->{$dbres} )
                   {
                     if ( $dbOracleHome and $sid and $db_ref->{$dbres}{ORACLE_HOME}
                        and $db_ref->{$dbres}{ORACLE_HOME} eq $dbOracleHome
                        and $db_ref->{$dbres}{USR_ORA_INST_NAME}
                        and $db_ref->{$dbres}{USR_ORA_INST_NAME} =~ /^$sid$/
                       )
                     {
                       $hasManaged = "YES";
                       return wantarray? ($hasManaged,$HasManagedCRSHome):$hasManaged;
                     }
                     elsif ( $resourceName and $db_ref->{$dbres}{NAME}
                                and $db_ref->{$dbres} =~ /^$resourceName$/
                             )
                     {
                       $hasManaged = "YES";
                       return wantarray? ($hasManaged,$HasManagedCRSHome):$hasManaged;
                     }
                   }
                }
             }
         }
         elsif ( $ocrType and $ocrType =~ /cluster/ )
         {
           $clusterOcr = 1;
         }
       }
     }
   }
  # there is a single instance HAS
  if ( $HasManagedCRSHome )
  {
    $cHome = $HasManagedCRSHome;
    if ( $resourceName  )
    {
      $hasManaged = 'YES';
    }
    elsif ( $hasDB and $hasDB =~ /^YES$/i )
    {
      $hasManaged = 'YES';
    }
    else
    {
      $hasManaged = 'NO';
    }
  }
  else
  {
    $cHome = undef;
    if ( $resourceName  )
    {
      $hasManaged = 'N/A';
      warn "DEBUG:has::Common::hasIsDBManagedByHas: No HA install found \n" unless $clusterOcr;
    }
    elsif ( $hasDB and $hasDB =~ /^YES$/i )
    {
      $hasManaged = 'N/A';
      warn "DEBUG:has::Common::hasIsDBManagedByHas: No HA install found \n";
    }
    else
    {
      $hasManaged = 'N/A';
    }
  }
  return wantarray? ($hasManaged,$cHome):$hasManaged;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasIsLsnrManagedByHas
#
# DESC
#  return FALSE or HAS Name
#
# ARGUMENTS
#  oracle Home
#  Listener Port
#  listner name ( optional)
#
# RETURNS
#  return FALSE or HAS Name
#------------------------------------------------------------------------------
sub hasIsLsnrManagedByHas($$;$)
{
   my ( $oracleHome,$lsnrPort,$oracleName ) = @_;
   my $dynProp = 'TRUE';
   my $ocrType = 'cluster';
   my $clusterOcr = 1;
   my $hasManaged = 'N/A';
   my $cHome = undef;
   my $HasManagedCRSHome = undef;
   #bug fix 9350140
   # added function to check id has/cluster inventory exists
   my $hasclcheck = has::Common::hasCheckForHasClusterInventory();
   if ( $hasclcheck and ref($hasclcheck) and ref($hasclcheck) =~ /HASH/i and keys %{$hasclcheck}
        and $hasclcheck->{type} and $hasclcheck->{type} =~ /has|cluster/i
        and $hasclcheck->{oracle_home} )
   {
    warn "DEBUG:has::Common::hasIsLsnrManagedByHas an $hasclcheck->{type} install is present on this host at $hasclcheck->{oracle_home}\n";
   }
   else
   {
     # there is no has/cluster installed on this box with inventory
      return wantarray? ($hasManaged,$cHome):$hasManaged;
   }
   # if not single instance has managed database
   # check if it has a resource name
   my ($resourceName,$isScan) = has::Common::hasGetResourceNameForOracleApp($oracleHome,'listener',$oracleName,$lsnrPort);
   # scan listener is not has managed 
   if ( $isScan and $isScan =~ /TRUE/ )
   {
      return wantarray? ($hasManaged,$cHome):$hasManaged;
   }
   # failed to get from db try to find a crs home and get information  
   my $discResultsref = has::Common::hasDiscoverCluster($dynProp);
    
   if ( $discResultsref and keys %{$discResultsref} )
   {
     for my $crsHome ( keys %{$discResultsref} )
     {
      # code below this will not execute on nt as emcrsp is not in the nt shiphome for beta1 
      if (  has::Common::hasCheckForEmcrsp($crsHome) )
      {
         $cHome = $crsHome;
         $ocrType = has::Common::hasGetOcrType($crsHome,$dynProp);
         # there is a single instance ha on the box but the db table shows none
         if ( $ocrType and $ocrType =~ /has/i )
         {
           $HasManagedCRSHome = $crsHome;
             my $db_ref = has::Common::hasGetLsnrResourceInformation($crsHome);
  
             if ( $db_ref and keys %{$db_ref} )
             {
                for my $dbres ( keys %{$db_ref} )
                {
                   if ( $db_ref->{$dbres} )
                   {
                     if ( $db_ref->{$dbres}{PORT} and $db_ref->{$dbres}{PORT} =~ /^$lsnrPort$/
                       )
                     {
                       $hasManaged = "YES";
                       return wantarray? ($hasManaged,$HasManagedCRSHome):$hasManaged;
                     }
                     elsif ( $resourceName and $db_ref->{$dbres}{NAME}
                                and $db_ref->{$dbres} =~ /^$resourceName$/
                             )
                     {
                       $hasManaged = "YES";
                       return wantarray? ($hasManaged,$HasManagedCRSHome):$hasManaged;
                     }
                   }
                }
             }
         }
         elsif ( $ocrType and $ocrType =~ /cluster/ )
         {
           $clusterOcr = 1;
         }
       }
     }
   }
  # there is a single instance HAS
  if ( $HasManagedCRSHome )
  {
    $cHome = $HasManagedCRSHome;
    if ( $resourceName  )
    {
      $hasManaged = 'YES';
    }
    else
    {
      $hasManaged = 'NO';
    }
  }
  else
  {
    $cHome = undef;
    if ( $resourceName  )
    {
      $hasManaged = 'N/A';
      warn "DEBUG:has::Common::hasIsDBManagedByHas: No HA install found \n" unless $clusterOcr;
    }
    else
    {
      $hasManaged = 'N/A';
    }
  }
  return wantarray? ($hasManaged,$cHome):$hasManaged;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasCheckAndReturnEmcrspResults
#
# DESC:
#  checks he command results to make sure they do not have anything
#  other than the stdout printed from emcrsp
#
# arg  :
#  cmdresults from emcrsp
#
# return:
#  return the ckecked cmd results
#------------------------------------------------------------------------------
sub hasCheckAndReturnEmcrspResults($)
{
  my ( $cmdresults ) = @_;
  return unless $cmdresults;
  my $temp_res;
  $temp_res = $cmdresults;
  #bug fix 9212119
  $temp_res =~ s/\n\s*[^\s|^<]+[^<]*<[^\n]+//g if $temp_res;
  $temp_res =~ s/^\s*[^\s|^<]+[^<]*<[^\n]+//g if $temp_res;
  return $temp_res if $temp_res;
  return;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasadm_resource_types_mark_base_types
#
# DESC
#  this function will mark the base type, 
#    build the base_type_chain
#    mark the base resource type for each attribute of a resource type
#
# arg  :
#  ref to hash list of resource types with attribs parsed from xml
#
# return:
#------------------------------------------------------------------------------
sub hasadm_resource_types_mark_base_types($)
{
  # name : hasadm_resourcetype_markbase_fn
  # desc : for each resource type element passed it keeps track of the base type
  #
  # arg  :
  #  ref to xml element to be filtered
  #``ref to the result hass entities  array
  # 
  sub hasadm_resourcetype_markbase_fn($$)
  {
    my ( $elref, $rsref ) = @_;
  
    return 1 unless $elref->{element} and $elref->{element} =~ /^(entity|attribute)$/i;
    # keep an index of resource_type to base_type
    if ( $elref->{element} =~ /^attribute$/i )
    {
       # attribute should have the name ,value .. child elements
       return 1 unless $elref->{children};
       my $entity_ref;
       #get the  resource_type entity
       $entity_ref = $elref->{parent}->{parent} if $elref->{parent} and $elref->{parent}->{parent};
       return 1 unless $entity_ref and $entity_ref->{attrs} and $entity_ref->{attrs}{entity_name}; 
       return 1 unless $entity_ref->{attrs}{entity_type} and $entity_ref->{attrs}{entity_type} =~ /^resource_type$/i; 
       # get the default_value for name= BASE_TYPE
       my $val = '';
       my $name;
       for my $elem ( @{$elref->{children}} )
       {
        next unless $elem and $elem->{element} and $elem->{element} =~ /^(name|default_value)$/i;
        $name = $elem->{name} and next if  $elem->{element} =~ /name/ and $elem->{name};
        $val = $elem->{name} and next if  $elem->{element} =~ /default_value/ and $elem->{name};
 
       }
        
       return 1 unless $name;
       #keep an look up list of resource attrib and default value
       #this is for marking as to the resource type a attrib comes from
       $has::Common::hasadm_restype_attrib{$entity_ref->{attrs}{entity_name}}{$name}=$val if $val;
       $has::Common::hasadm_restype_attrib{$entity_ref->{attrs}{entity_name}}{$name}='' unless $val;
       return 1 unless $name and $name =~ /^BASE_TYPE$/i;
       $has::Common::hasadm_restype_base{$entity_ref->{attrs}{entity_name}}{base_type}= $val;
       return 1;
    }
    return 1 unless $elref->{attrs} and $elref->{attrs}{entity_type} and $elref->{attrs}{entity_type} =~ /^resource_type$/i;
  
    #has::Common::append_element($rsref,$elref);
    # keep an has stack of resource types
    return 1 unless $elref->{attrs}{entity_name};
    $has::Common::hasadm_resource_type_list{$elref->{attrs}{entity_name}}=$elref;
  
    return 1;
  
  }
  # name : hasadm_resource_attrib_type_fn
  # desc : for each attrib passed it gets the source resource type for that attrib
  #
  # arg  :
  #  ref to xml element to be filtered
  #``ref to the result hass entities  array
  # 
  sub hasadm_resource_attrib_type_fn($$)
  {
    my ( $elref, $rsref ) = @_;
  
    return 1 unless $elref->{element} and $elref->{element} =~ /^(entity)$/i;
    return 1 unless $elref->{attrs} and $elref->{attrs}{entity_type} and $elref->{attrs}{entity_type} =~ /^resource_type$/i;
  
    return 1 unless $elref->{attrs}{entity_name};
   
    my $base_type_chain;
    my @base_types;
    $base_type_chain = $elref->{attrs}{base_type_chain} if $elref->{attrs}{base_type_chain};
    @base_types = split/,/,$base_type_chain if $base_type_chain; 
    # go thru each attrib and get the name and go thru base_type_chain to get the root resource type
    # for that attrib name
    for my $attribs ( @{$elref->{children}} )
    {
      for my $attrib ( @{$attribs->{children}} )
      {
         for my $elem ( @{$attrib->{children}} )
         {
          next unless $elem and $elem->{element} and $elem->{element} =~ /^(name)$/i;
  
          my $name;
          $name = $elem->{name} if  $elem->{element} =~ /name/ and $elem->{name};
           
          $name =~ s/^\s+|\s+$//g if $name;
          next unless $name;
  
          # for each attrib get the base name from where it came
          #  move from the leaf to the root, so the root for attrib sticks
          my $attribResType;
          my $attribResValue;
          for my $btype ( @base_types )
          {
             $btype =~ s/^\s+|\s+$//g if $btype;
             next unless $btype;
             next unless  $has::Common::hasadm_restype_attrib{$btype} and defined $has::Common::hasadm_restype_attrib{$btype}{$name};
             $attribResType = $btype;
             $attribResValue = $has::Common::hasadm_restype_attrib{$btype}{$name} if $has::Common::hasadm_restype_attrib{$btype}{$name};
          }
          if ( not $attribResType )
          {
            $attribResType = $elref->{attrs}{entity_name}; 
            $attribResValue =  $has::Common::hasadm_restype_attrib{$elref->{attrs}{entity_name}}{$name} 
              if $has::Common::hasadm_restype_attrib{$elref->{attrs}{entity_name}} 
                and  $has::Common::hasadm_restype_attrib{$elref->{attrs}{entity_name}}{$name};
          }
          $attribResType = '' unless $attribResType;
          $attribResValue = '' unless $attribResValue;
          my %attribResElement;
          my %attribResValueElement;
          my $attribResElementRef = has::Common::make_element('resource_type',$attribResType);
          my $attribResValueElementRef = has::Common::make_element('base_attrib_value',$attribResValue);
          has::Common::append_element($attrib,$attribResElementRef);
          has::Common::append_element($attrib,$attribResValueElementRef);
          last;
         }
      }  
    }
   return 1;
  
  }
  my ( $reslistref ) = @_;
  %has::Common::hasadm_restype_base = ();
  %has::Common::hasadm_resource_type_list = ();
  has::Common::traverse_xml($reslistref,\&has::Common::has_handle_error,\&hasadm_resourcetype_markbase_fn,$reslistref);
  # create the coma separated string list of resource types
  if ( keys %has::Common::hasadm_restype_base )
  {
    # for each resource type build the stack
    for my $res ( keys %has::Common::hasadm_restype_base )
    {
      next if $has::Common::hasadm_restype_base{$res}{chain};
      my @bstack;
      my $bres = $res;
      while ( $bres )
      {
        # build a stack of base resource types
        push @bstack,$bres;
        $bres =  $has::Common::hasadm_restype_base{$bres}{base_type};
        if ( not $bres )
        {
          # reached the base resource type, start building the chain
          my $chain='';
          my $pres = pop @bstack;
          while ( $pres )
          {
           $has::Common::hasadm_restype_base{$pres}{base_type_chain}=$chain
            unless $has::Common::hasadm_restype_base{$pres}{base_type_chain};
           $chain = "$pres,$chain" if $chain;
           $chain = $pres unless $chain;
           $pres = pop @bstack;
          }
        }
      }
    }  
  }
  # get the resource type chain for each resource
  for my $restype ( keys %has::Common::hasadm_resource_type_list )
  {
    my $resref = $has::Common::hasadm_resource_type_list{$restype};
    $resref->{attrs}{root_type} = '';
    $resref->{attrs}{base_type_chain} = '';
    $resref->{attrs}{root_type}=$has::Common::hasadm_restype_base{$restype}{base_type} 
      if $has::Common::hasadm_restype_base{$restype}{base_type};
    $resref->{attrs}{base_type_chain}=$has::Common::hasadm_restype_base{$restype}{base_type_chain}
      if $has::Common::hasadm_restype_base{$restype}{base_type_chain};
  }
 
  # for each resource type go up the base chain and mark the source resource type for each attrib
  has::Common::traverse_xml($reslistref,\&has::Common::has_handle_error,\&hasadm_resource_attrib_type_fn,$reslistref);
  return 1;
}
# name : hasGetResourceNameForOracleApp
# desc : return the resource name for oracle type, oracle name
#
# arg  :
#  oracleHome of the resource
#  type of resource (database,listener,service,scanlistener,asm,eons)
#  oracle name of the reesource
#
# return:
#
sub hasGetResourceNameForOracleApp($$;$$)
{
  my ($oracleHome,$oracleType,$oracleName,$oracleName2 ) = @_;
  my $resName = undef;
  my $type1;
  my $type2;
  my $nown;
  my $scanPortMatch=undef;
  my $isScan = undef;
  my %scanList;
  warn "ERROR:has::Common::hasGetResourceNameForOracleApp, ORACLE Home not passed for getting resource name\n" and return unless $oracleHome;
  warn "ERROR:has::Common::hasGetResourceNameForOracleApp , Type not passed for getting resource name\n" and return unless $oracleType;
  #bug fix 9350140
  # added function to check id has/cluster inventory exists
  my $hasclcheck = has::Common::hasCheckForHasClusterInventory();
  
  if ( $hasclcheck and ref($hasclcheck) and ref($hasclcheck) =~ /HASH/i and keys %{$hasclcheck} 
        and $hasclcheck->{type} and $hasclcheck->{type} =~ /has|cluster/i 
        and $hasclcheck->{oracle_home} )
  {
    warn "DEBUG:has::Common::hasGetResourceNameForOracleApp an $hasclcheck->{type} install is present on this host at $hasclcheck->{oracle_home}\n";
  }
  else
  {
     # there is no has/cluster installed on this box with inventory
     return wantarray? ('N/A',$isScan):'N/A';
  }
  if ( $oracleType =~ /database|oracle_database|rac_database/ )
  {
    $type1 = 'dbunique_name';
    $nown = 'database';
  }
  elsif ( $oracleType =~ /listener|oracle_listener/ )
  {
    $type1 = 'res_name';
    $nown = 'listener';
  }
  elsif ( $oracleType =~ /asm|asm_instance/ )
  {
    $type1 = 'asm_name';
    $nown = 'asm';
  }
  elsif ( $oracleType =~ /scan_listener/ )
  {
    $isScan = 'TRUE';
    $type1 = 'port';
    $type2 = 'lsnr_name';
    $nown = 'scan_listener';
  }
  else
  {
    warn "ERROR:has::Common::hasGetResourceNameForOracleApp unsupported type $oracleType, for getting resource name\n";
    return;
  }
  # set oracle home
  # execute srvctl config <> -S 1
  # if run is successful parse for 
  # unset oracle home
  # for oracle internal resources we do not want to throw an error if the srvctl command is not supported
  # for older versions, so check for srvctl and throw and debug message
  if ( $oracleHome )
  {
    my $srvctl_full_path;
    my $obindir = catfile($oracleHome,'bin');
    for my $extn ( ( '', '.exe', '.bat' ) )
    {
     my $tcmd = "srvctl$extn";
     $srvctl_full_path = catfile($obindir,$tcmd);
     last if has::Common::hasIsReadable($srvctl_full_path);
     
     undef $srvctl_full_path;
     next;
    
    }
    warn "DEBUG:has::Common::hasGetResourceNameForOracleApp , resource name is not supported, command srvctl is not presnt in ORACLE HOME $oracleHome\n" and return unless $srvctl_full_path;
  }
  has::Common::hasSetCRSEnv($oracleHome);
  my %command_args = (exit_failure_list => [()]);
  my $vo;
  if ( $nown =~ /database/ and $oracleName )
  {
    %command_args = (exit_failure_list => [()]);
    $vo = has::Common::runsystemcommand("srvctl","config $nown -d $oracleName -S 1",\%command_args);
  }
  elsif ( $nown =~ /^listener$/ and $oracleName )
  {
    # store information for all scan listeners
    %command_args = (exit_failure_list => [()]);
    $vo = has::Common::runsystemcommand("srvctl","config scan_listener -S 1",\%command_args);
    warn "DEBUG:has::Common::hasGetResourceNameForOracleApp Failed executing srvctl config scan_listener -S 1, no scan listener" 
     if $command_args{command_return_status};
 
    chomp($vo) if $vo;
    $vo =~ s/^\s+|\s+$// if $vo;
 
    my @scanrows = split /\n/,$vo if $vo;
 
    warn "DEBUG:has::Common::hasGetResourceNameForOracleApp Failed to get any output from command srvctl config scan_listener -S 1" unless @scanrows;
    #parse for res_name={resource name}
    for my $scanrow ( @scanrows )
    {
     $scanrow =~ s/^\s+|\s+$// if $scanrow;
 
     next unless $scanrow and $scanrow =~ /res_name\s*=\s*{[^}]+}/;
 
     my ( $scanResName ) = ( $scanrow =~ /res_name\s*=\s*{\s*([^}]+)\s*}/ );
     $scanResName =~ s/^\s+|\s+$// if $scanResName;
     
     $scanList{$scanResName}=$scanrow if $scanResName;
     # check if the scan listener name passed is in the resource name
     next unless $scanResName;
     next unless $oracleName;
     next unless $scanrow =~ /$type1={\s*ora\.$oracleName\.[^}]+\s*}/;
     # if listener port is provided check for port on the output of srvctl
     if ( $oracleName2 )
     {
        next unless $scanrow =~ /\{$oracleName2\}/ or $scanrow =~ /\{$oracleName2\,/ or $scanrow =~ /\,$oracleName2\}/
         or $scanrow =~ /\,$oracleName2\,/;
     }
     # if listenername and port are in the scanrow output it is a  scan listener
     $isScan = 'TRUE';
     return wantarray? ($scanResName,$isScan):$scanResName if $isScan;
    }
    undef $vo if $vo;
    %command_args = (exit_failure_list => [()]);
    $vo = has::Common::runsystemcommand("srvctl","config $nown -l $oracleName -S 1",\%command_args);
  }
  if ( $command_args{command_return_status} or not $vo )
  {
     %command_args = (exit_failure_list => [()]);
     $vo = has::Common::runsystemcommand("srvctl","config $nown -S 1",\%command_args);
  }
  has::Common::hasRestoreCRSEnv();
  warn "DEBUG:has::Common::hasGetResourceNameForOracleApp Error executing srvctl config $nown -S 1"
    if $command_args{command_return_status};
  return wantarray? ($resName,$isScan):$resName if $command_args{command_return_status};
  chomp($vo) if $vo;
  $vo =~ s/^\s+|\s+$// if $vo;
                                                         
  my @rows = split /\n/,$vo if $vo;
  
  warn "DEBUG:has::Common::hasGetResourceNameForOracleApp Failed to get any output from command srvctl config $nown -S 1" unless @rows;
  #parse for res_name={resource name}
  for my $row ( @rows )
  {
     $row =~ s/^\s+|\s+$// if $row;
     next unless $row =~ /res_name=/;
     if ( $nown =~ /database|listener|scan_listener/ )
     {
      next unless $row =~ /$type1=/;
      if ( $nown =~ /database/ )
      {
       next unless $oracleName;
       next unless $row =~ /$type1={\s*$oracleName\s*}/;
      }
      elsif ( $nown =~ /listener|scan_listener/ )
      {
       next unless $oracleName;
       next unless $row =~ /$type1={\s*ora\.$oracleName\.[^}]+\s*}/;
       # if listener port is provided check for port on the output of srvctl
       if ( $oracleName2 )
       {
        next unless $row =~ /\{$oracleName2\}/ or $row =~ /\{$oracleName2\,/ or $row =~ /\,$oracleName2\}/ or $row =~ /\,$oracleName2\,/;
       }
      }
     }
     ( $resName ) = ( $row =~ /res_name=\s*{\s*([^}]+)\s*}/ );
     $isScan = 'TRUE' if $nown =~ /listener|scan_listener/ and $resName and keys %scanList and $scanList{$resName};
     last if $resName;
  }
  warn "DEBUG:has::Common::hasGetResourceNameForOracleApp Failed to get resource name from command srvctl config $nown -S 1" unless $resName;
  return wantarray? ($resName,$isScan):$resName;
}
# name : hasListenerGetScanName
# desc : return the scan name for scan listener
#
# arg  :
#  oracleHome of the listener
#  name of the listener
#
# return:
# scan name
#
sub hasListenerGetScanName($;$)
{
  my ($oracleHome,$oracleName ) = @_;
  # yet to implement the srvctl implementation
  return has::Common::hasGetScanName();
}
# name : hasIsListenerScan
# desc : return true if listener is scan, false otherwise
#
# arg  :
#  oracleHome of the listener
#  name of the listener
#
# return:
# scan name
#
sub hasIsListenerScan($;$)
{
  my ($oracleHome,$oracleName ) = @_;
  # yet to implement the srvctl implementation
  my $scanname = has::Common::hasGetScanName();
  return 1 if $scanname;
  return 0;
}
# name : has_dependencies
# desc : for each element passed it gets the dependencies and instruments them
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
# 
sub has_dependencies($$$)
{
  my ( $elref, $rsref, $metric_name ) = @_;
  my $parent_metric_name;
  # storage an empty value, if there are no custom attribs the metric exec 
  #  should not fail
  $rsref->{$metric_name} = [()] unless $rsref->{$metric_name};
  # this is a function for dependent metrics, get the parent metric_name
  warn "WARN:Failed, parent metric is not defined for dependent metric $metric_name\n"
   and return
    unless $has::Common::has_metric_config{parent_metric}{$metric_name};
  $parent_metric_name = $has::Common::has_metric_config{parent_metric}{$metric_name};
  # this should have already have has the elemm attribs instrumented
  return 1 unless $elref->{elem_attribs};
  # instrument both the start and stop dependencies
  for my $depStartStop ( sort qw ( START_DEPENDENCIES STOP_DEPENDENCIES ) )
  {
    next unless $elref->{elem_attribs}{$depStartStop};
    # get the start or stop dependency string
    my $depls =  $elref->{elem_attribs}{$depStartStop};
    $depls =~ s/^\s+|\s+$//g if $depls;
    next unless $depls;
    # replace end brace with )== to easily split them
    $depls =~ s/\)/\)\=\=/g;
    my @deplist = split/\=\=/,$depls;
    for my $dps ( @deplist )
    {
      my $pref;
      chomp $dps;
      $dps =~ s/^\s+|\s+$//g;
      next unless $dps;
      # get the dependency attrib 1 from attrib>(,,..)
      my ( $dependentAttrib1,$depnds) = ( $dps =~ /^(.+)\((.*)\)$/);
      $dependentAttrib1 =~ s/^\s+|\s+$//g if $dependentAttrib1;
      $depnds =~ s/^\s+|\s+$//g if $depnds;
      warn "WARN:has_metrics.pl::has_dependencies Failed to get dependency type from $dps" and next unless $dependentAttrib1;
      warn "WARN:has_metrics.pl::has_dependencies Failed to get dependent name list from $dps" and next unless $depnds;
      # get the list of dependents from , seperated list
      my @dependentList = split/,/,$depnds;
      for my $dependents ( @dependentList )
      {
       my $dependentName;
       my $dependentAttrib2;
       chomp $dependents;
       $dependents =~ s/^\s+|\s+$//g;
       warn "WARN:has_metrics.pl::has_dependencies Got Null for dependents from $depnds" and next unless $dependents;
       $dependentName = $dependents;
       # get the second attrib if the dependent name has a :
       ($dependentAttrib2,$dependentName) =
         ( $dependentName =~ /(.+):(.*)/)
           if $dependentName =~ /:/;
       $pref->{DEPENDENCY_TYPE}=$depStartStop;
       $pref->{DEPENDENT_NAME}=$dependentName;
       $pref->{DEPENDENCY_ATTRIB1}=$dependentAttrib1;
       $pref->{DEPENDENCY_ATTRIB2}=$dependentAttrib2;
       $pref->{DEPENDENCY_ATTRIB1}='NA' unless $pref->{DEPENDENCY_ATTRIB1};
       $pref->{DEPENDENCY_ATTRIB2}='NA' unless $pref->{DEPENDENCY_ATTRIB2};
       # get the attrib 1 for the dependency
       for my $dpat1 ( qw ( hard weak attraction dispersion pullup ) )
       {
           $pref->{uc $dpat1} = 'YES' if  $dependentAttrib1 =~ /^$dpat1$/i;
           $pref->{uc $dpat1} = 'NO' unless $pref->{uc $dpat1};
       }
       # get the attrib 2 for the dependency
       for my $dpat2 ( qw ( global ) )
       {
           $pref->{uc $dpat2} = 'YES' if $dependentAttrib2 and $dependentAttrib2 =~ /^$dpat2$/i;
           $pref->{uc $dpat2} = 'NO' unless $pref->{uc $dpat2};
       }
       # build the em_results row
       has_em_results($elref,$rsref,$metric_name,$pref);
     }
    }
  }
 return 1;
}
# name : has_em_results
# desc : for each element passed it gets the metric cols
#         and build the em_result row after verufying the keys
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
#  ref to attribute element to be printed for dependent metrics
#
sub has_em_results ($$$;$)
{
  my ($elref,$rsref,$metric_name,$pref) = @_;
    # build the result string
    if ( $has::Common::has_metric_config{basic_fields}{$metric_name} )
    {
      my $retstr = 'em_result=';
      my %hashret;
      for my $attrpos ( sort 
   		    { $a <=> $b } 
   		    keys %{$has::Common::has_metric_config{basic_fields}{$metric_name}}  )
      {
       my $attr;
       my $metval = '';
       $attr = $has::Common::has_metric_config{basic_fields}{$metric_name}{$attrpos};
       # if the attr starts with # then its an attr element name 
       #  this is so as not to confuse things like entity name with attribute NAME
 
       if ( $attr =~ /NULLSTRING_/ )
       {
         $metval='';
       }
       elsif ( $attr !~ /^#/ )
       {
         $metval = $elref->{elem_attribs}{$attr} if defined $elref->{elem_attribs}{$attr} ;
         $metval  = $elref->{elem_attribs}{lc $attr} if defined $elref->{elem_attribs}{lc $attr} ;
     
         $metval  = $elref->{elem_attribs}{uc $attr} if defined $elref->{elem_attribs}{uc $attr} ;
       }
       else
       {
         my $nattr = $attr;
         $nattr =~ s/^#//;
         $metval = $pref->{$nattr} if defined $pref->{$nattr} ;
         $metval = $pref->{lc $nattr} if defined $pref->{lc $nattr} ;
         $metval = $pref->{uc $nattr} if defined $pref->{uc $nattr} ;
       }
       # | is special char for em, repalce it if it is in value
       $metval =~ s/\|+/_/g;
       # do this special for _NULL values change them to NULL
       $metval = 'NULL' if $metval =~ /^_NULL$/;
       # validate metric value , data type checks
       if ( $has::Common::has_metric_config{data_type}{$metric_name}{$attr} )
       {
          my $datatype = $has::Common::has_metric_config{data_type}{$metric_name}{$attr};
          if ( $datatype =~ /number/i )
          {
            $metval = 0 if $metval =~ /^NULL$/;
            $metval = 0 unless $metval;
            $metval = 0 unless defined $metval;
            warn "WARN:Invalid metric value $metval for metric column $attr for metic $metric_name not a number"
             and next
              unless $metval =~ /\d*/;
          }
          elsif ( $datatype =~ /boolean/ )
          {
            $metval = 1 if $metval =~ /YES|TRUE/i; 
            $metval = 0 if $metval =~ /NO|FALSE/i; 
            $metval = 0 if $metval =~ /^NULL$/;
            warn "WARN:Invalid metric value $metval for metric column $attr for metic $metric_name not a boolean"
              and next
               unless $metval =~ /\d+/ and ( $metval == 0 or $metval == 1 );
          }
       }
       $retstr .= "$metval|";
       $hashret{$attr} = $metval;
     }
     # we might have keys which are not in the list of columns
     # so pick key values fro them and store them in hashret
     for my $kno ( keys %{$has::Common::has_metric_config{key_fields}{$metric_name}} )
     {
       for my $keypos ( sort 
   		    { $a <=>$b }
   		    keys %{$has::Common::has_metric_config{key_fields}{$metric_name}{$kno}}  )
       {
  
         my $attr;
         my $metval = '';
         $attr = $has::Common::has_metric_config{key_fields}{$metric_name}{$kno}{$keypos};
         next if $hashret{$attr};
  
         # if the attr starts with # then its an attr element name 
         #  this is so as not to confuse things like entity name with attribute NAME
   
         if ( $attr =~ /NULLSTRING_/ )
         {
           $metval='';
         }
         elsif ( $attr !~ /^#/ )
         {
  
           $metval = $elref->{elem_attribs}{$attr} if defined $elref->{elem_attribs}{$attr} ;
  
           $metval  = $elref->{elem_attribs}{lc $attr} if defined $elref->{elem_attribs}{lc $attr} ;
       
           $metval  = $elref->{elem_attribs}{uc $attr} if defined $elref->{elem_attribs}{uc $attr} ;
  
         }
         else
         {
  
           my $nattr = $attr;
  
           $nattr =~ s/^#//;
  
           $metval = $pref->{$nattr} if defined $pref->{$nattr} ;
  
           $metval = $pref->{lc $nattr} if defined $pref->{lc $nattr} ;
  
           $metval = $pref->{uc $nattr} if defined $pref->{uc $nattr} ;
  
         }
  
         # | is special char for em, repalce it if it is in value
         $metval =~ s/\|+/_/g;
  
         # do this special for _NULL values change them to NULL
         $metval = 'NULL' if $metval =~ /^_NULL$/;
  
         # validate metric value , data type checks
         if ( $has::Common::has_metric_config{data_type}{$metric_name}{$attr} )
         {
            my $datatype = $has::Common::has_metric_config{data_type}{$metric_name}{$attr};
  
            if ( $datatype =~ /number/i )
            {
              $metval = 0 if $metval =~ /^NULL$/;
              $metval = 0 unless $metval;
              $metval = 0 unless defined $metval;
  
              warn "WARN:Invalid metric value $metval for metric column $attr for metic $metric_name not a number"
               and next
                unless $metval =~ /\d*/;
  
            }
            elsif ( $datatype =~ /boolean/ )
            {
              $metval = 1 if $metval =~ /YES|TRUE/i; 
              $metval = 0 if $metval =~ /NO|FALSE/i; 
              $metval = 0 if $metval =~ /^NULL$/;
  
              warn "WARN:Invalid metric value $metval for metric column $attr for metic $metric_name not a boolean"
                and next
                 unless $metval =~ /\d+/ and ( $metval == 0 or $metval == 1 );
            }
  
         }
  
         $hashret{$attr} = $metval;
  
       }
     }
     # check for keys before pushing
     my $key_valid = 1;
     for my $kno ( keys %{$has::Common::has_metric_config{key_fields}{$metric_name}} )
     {
       my $keyval = "";
       for my $keypos ( sort 
   		    { $a <=>$b }
   		    keys %{$has::Common::has_metric_config{key_fields}{$metric_name}{$kno}}  )
       {
         my $keycolval;
         my $keyattr;
         $keyattr = $has::Common::has_metric_config{key_fields}{$metric_name}{$kno}{$keypos};
         $keycolval = $hashret{$keyattr} if $hashret{$keyattr};
         $keycolval = $hashret{uc $keyattr} if $hashret{uc $keyattr};
         $keycolval = $hashret{lc $keyattr} if $hashret{lc $keyattr};
         if ( not $keycolval 
               and $has::Common::has_metric_config{data_type}{$metric_name} 
               and $has::Common::has_metric_config{data_type}{$metric_name}{$keyattr} 
               and $has::Common::has_metric_config{data_type}{$metric_name}{$keyattr} =~ /number/)
         {
           $keycolval =  $hashret{$keyattr} if defined $hashret{$keyattr}; 
         }
          # key value cannot be null
          warn "WARN:Key column $keyattr is null while instrumenting metric $metric_name "
            and $key_valid = 0 
             and last 
              unless defined $keycolval;
           $keyval = $keyval.'_'.$keycolval if defined $keycolval;
       }
       $keyval =~ s/^_// if $keyval;
       # if the key already exists then mark the key as invalid
        warn "WARN:Duplicate key $keyval while instrumenting metric $metric_name" 
         and $key_valid = 0 
          if $rsref->{$metric_name.'_key_value'}{$kno}{$keyval};
       # skip this metric row if key is invalid
       last unless $key_valid;
       # keep the list of keys
       $rsref->{$metric_name.'_key_value'}{$kno}{$keyval}=$retstr;
       $rsref->{hash_ref}{$metric_name}{$kno}{$keyval}=$elref;
     }
     # add the metric row only if key is valid
     push @{$rsref->{$metric_name}},$retstr if $key_valid;;
   }
   return 1;
}
# name : has_gen_key_value
# desc : generate a key value for an element passed
#
# arg  :
#  ref to hash opf element with attribs
#  metric_name being processed
#
#  return :
#   key value
#
sub has_gen_key_value($$;)
{
  my ( $elref,$metric_name) = @_;
  # for event metrics build the event key
  my $gen_key_value = '';
  if ( $has::Common::has_metric_config{build_key}{$metric_name} )
  {
    for my $ek ( sort keys %{$has::Common::has_metric_config{build_key_cols}{$metric_name}} )
    {
       my $keycol = $has::Common::has_metric_config{build_key_cols}{$metric_name}{$ek};
 
       my $keycolval = '';
 
       if ( $elref->{elem_attribs}{$keycol} )
       {
         $keycolval = $elref->{elem_attribs}{$keycol};
       }
       elsif ( $elref->{elem_attribs}{uc $keycol} )
       {
         $keycolval = $elref->{elem_attribs}{uc $keycol};
       }
       elsif ( $elref->{elem_attribs}{lc $keycol} )
       {
         $keycolval = $elref->{elem_attribs}{lc $keycol};
       }
       if ( $gen_key_value )
         {
           $gen_key_value = "$gen_key_value\_$keycolval";
         }
       else
         {
           $gen_key_value = $keycolval;
         }
    }
  }
  return $gen_key_value;
}
# name : has_entity_fn
# desc : for each element passed it filters our the enity element
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
#  crs home
#  filter metric column
#  filter metric column value
#  filter metric verb
#
sub has_entity_fn($$$;@)
{
  my ( $elref, $rsref, $metric_name ,$crsHome, $args) = @_;
  my $filter_col;
  my $filter_val;
  my $filter_verb;
	  my $dbPolicyManaged;
	  ($filter_col,$filter_val,$filter_verb ) = @{$args} if $args and ref($args) =~/ARRAY/i;
	  # check if this represents a element of interest
	  return 1 
	   unless $elref->{element} and $elref->{element} =~ /^entity$/i;
	  # does this metric need instrumenting an entity type
	  # if no entity type is defined the metric may be a cached from
	  #  another run
	  return 1 unless $has::Common::has_metric_config{entity_type}{$metric_name};
	  return 1
	   unless $elref->{attrs} and 
	    $elref->{attrs}{entity_type} and 
	     $elref->{attrs}{entity_name} and
	       $elref->{attrs}{entity_type} =~ 
		/^$has::Common::has_metric_config{entity_type}{$metric_name}$/i;
	  # save the cluster name and such globals as they are needed in every row to be printed
	  $elref->{elem_attribs}{CLUSTER_NAME}= has::Common::hasGetClusterName($crsHome);
	  $elref->{elem_attribs}{CLUSTER_ACTIVE_VERSION}= has::Common::hasGetClusterVersion($crsHome);
	  $elref->{elem_attribs}{OS_HOST_NAME}= has::Common::hasGetLocalHostName();
	  $elref->{elem_attribs}{EM_TARGET_NAME}= $has::Common::has_em_targetname;
	  $elref->{elem_attribs}{EM_TARGET_TYPE}= $has::Common::has_em_targettype;
	  $elref->{elem_attribs}{EM_TARGET_GUID}= $has::Common::has_em_targetguid;
	  $elref->{elem_attribs}{METRIC_SOURCE}= 'AGENT_GENERATED';
	  # event type for resource state change events
	  if ( $has::Common::has_metric_config{entity_type}{$metric_name} 
		and $has::Common::has_metric_config{entity_type}{$metric_name} =~ /resource|resource_instance/ )
	  {
	    $elref->{elem_attribs}{EONS_EVENT_NAME}= 'CRS_RESOURCE_STATE_CHANGE';
	  }
	  elsif ( $has::Common::has_metric_config{entity_type}{$metric_name} 
		and $has::Common::has_metric_config{entity_type}{$metric_name} =~ /node|server/ )
	  {
	    $elref->{elem_attribs}{EONS_EVENT_NAME}= 'CRS_SERVER_STATE_CHANGE';
	  }
	  # the timestamp for the metric generation in GMT
	  my $time = time;
	  my ($sec,$min,$hr,$mday,$mon,$year,$wday,$yday) = gmtime($time);
	  $year += 1900 if $year;
	  $mon += 1 if $mon;
	  my $timestring = sprintf("%04d-%02d-%02d %02d:%02d:%02d",$year,$mon,$mday,$hr,$min,$sec);
	  $elref->{elem_attribs}{TIMESTAMP}= $timestring;
	  # save entity type and entity name as elem attribs so they are 
	  # easy to print
	  # when iterating over attribs to print
	  $elref->{elem_attribs}{entity_type} = $elref->{attrs}{entity_type};
	  $elref->{elem_attribs}{entity_name} = $elref->{attrs}{entity_name};
	  # for now we kep oracle_name same as entity name
	  $elref->{elem_attribs}{oracle_name} = $elref->{attrs}{entity_name};
	  # keep the root type, base_type_chain attribs if present
	  $elref->{elem_attribs}{root_type} = $elref->{attrs}{root_type} if $elref->{attrs}{root_type};
	  $elref->{elem_attribs}{base_type_chain} = $elref->{attrs}{base_type_chain} if  $elref->{attrs}{base_type_chain};
	  # check if it has attributes
	  return 1 unless $elref->{children} and 
	   $elref->{child_elements}{attributes}
	    and ref($elref->{child_elements}{attributes}) =~ /ARRAY/i
	     and @{$elref->{child_elements}{attributes}};
	  my $attribsref = @{$elref->{child_elements}{attributes}}[0];
	  return 1 unless $attribsref->{children} and
	    $attribsref->{child_elements}{attribute}
	     and  ref($attribsref->{child_elements}{attribute}) =~ /ARRAY/i
	      and @{$attribsref->{child_elements}{attribute}};
	  my $valuefld = 'value';
	  $valuefld = $has::Common::has_metric_config{$metric_name}{value_property} if $has::Common::has_metric_config{$metric_name}{value_property};
	  my $attribbody='';
	  # create a name value array from the attributes
	  for my $attribref ( @{$attribsref->{child_elements}{attribute}} )
	  {
	     # get ref to the  and <[value]> element from attrib children
	     my $nameref;
	     my $valueref;
	     next unless $attribref->{children};
	     for my $cref ( @{$attribref->{children}} )
	     {
	       next unless $cref and $cref->{element} and $cref->{element} =~ /^(name|$valuefld)$/;
	       $nameref = $cref if $cref->{element} =~ /^name$/;
	       $valueref = $cref if $cref->{element} =~ /^$valuefld$/;
	     }
	     # if we do not get xx then skip this attribute
	     next unless $nameref and $valueref and $nameref->{name};
	     my $value = $valueref->{name};
	     $elref->{elem_attribs}{$nameref->{name}}= $value;
	     # save all the elements of the  attribute as hash members in elref
	     # this can be used for dependency values
	     # added them as attribs not to confuse them with elem_attribs{name}
	     # we mitght have a attribute name like MANDATORY etc
	     #  which matches element name MANDATORY
	     for my $cref ( @{$attribref->{children}} )
	     {
	       next unless $cref and $cref->{element};
	       $elref->{elem_attribs_properites}{$nameref->{name}}{$cref->{element}}= $cref->{name};
	     }
	     $value='' unless defined $value;
	     if ( $attribbody )
	     {
	       $attribbody = "$attribbody$nameref->{name}=$value\;";
	     }
	     else
	     {
	       $attribbody = "$nameref->{name}=$value\;";
	     }
	     #cook up an RESOURCE_LOCATION attribute in attrib body
	     # as eons event sends RESOURCE_LOCATION but not LAST_SERVER
	     # and the gpnp callback looks for RESOURCE_LOCATION
	     if ( $nameref->{name} =~ /^LAST_SERVER$/ )
	     {
	       $attribbody = "$attribbody"."RESOURCE_LOCATION=$value\;";
	     }
	  }
	  # do the filteration based on filters passed
	  if ( $filter_col and $filter_val )
	  {
	    my $filter_column_value;
	       if ( $elref->{elem_attribs}{$filter_col} )
	       {
		 $filter_column_value = $elref->{elem_attribs}{$filter_col};
	       }
	       elsif ( $elref->{elem_attribs}{uc $filter_col} )
	       {
		 $filter_column_value = $elref->{elem_attribs}{uc $filter_col};
	       }
	       elsif ( $elref->{elem_attribs}{lc $filter_col} )
	       {
		 $filter_column_value = $elref->{elem_attribs}{lc $filter_col};
	       }
	    if ( $filter_verb and $filter_verb =~ /NO/i )
	    {
	      # filter rows with column values not like 
	      if ( not defined $filter_column_value and defined $filter_val )
	      {
		  return 1;
	      }
	      if ( defined $filter_column_value and not defined $filter_val )
	      {
		  return 1;
	      }
	      if ( defined $filter_column_value and defined $filter_val )
	      {
		  return 1 if $filter_column_value =~ /$filter_val/;
	      }
	    }
	    else
	    {
	      # filter rows with column values like 
	      if ( defined $filter_column_value and defined $filter_val )
	      {
		  return 1 unless $filter_column_value =~ /$filter_val/;
	      }
	    }
	  }
	  # generate key value if required
	  my $gen_key_value = has_gen_key_value($elref,$metric_name);
	  
	  $elref->{elem_attribs}{GENERATED_KEY_1}=$gen_key_value;
	  
	  # mark relocated for vip
	  if ( $metric_name =~ /vip_relocation_metric|resource_instance_alert_metric/ )
	  {
	     if ( $elref->{elem_attribs}{TYPE} =~ /ora.cluster_vip_net1.type/ )
	     {
		if ( $elref->{elem_attribs}{NAME} and $elref->{elem_attribs}{LAST_SERVER} )
		{
		  if ( $elref->{elem_attribs}{NAME} =~ /ora\.$elref->{elem_attribs}{LAST_SERVER}\.vip/ )
		  {
		     $elref->{elem_attribs}{RELOCATED}='FALSE';
		  }
		  else
		  {
		     $elref->{elem_attribs}{RELOCATED}='TRUE';
		     $elref->{elem_attribs}{ADDITIONAL_ALERT_MESSAGE}=" Relocated to Node $elref->{elem_attribs}{LAST_SERVER}"
			if $elref->{elem_attribs}{STATE} =~ /INTERMEDIATE/;
		  }
		}
		else
		{
		     $elref->{elem_attribs}{RELOCATED}='NotDefined';
		}
	     }
	  }
	  #generate any dependent metrics if they are to be instrumented
	  # get the function name for instrumenting the dependent metrics from
	  # the config hash
	  for my $dp_metric_name 
	    ( keys %{$has::Common::has_metric_config{dependent_metric_functions}{$metric_name}} )
	      {
		warn "WARN:has::Common::has_dependencies:Failed to instrument dependent metric $dp_metric_name for $metric_name"
		  unless 
		    $has::Common::has_metric_config{dependent_metric_functions}{$metric_name}{$dp_metric_name};
		
		my $dp_retstr = 
		  &{$has::Common::has_metric_config{dependent_metric_functions}{$metric_name}{$dp_metric_name}}
		    ($elref,$rsref,$dp_metric_name)
		      or warn "WARN:has::Common::has_dependencies:Failed to instrument dependent metric $dp_metric_name for $metric_name"
	     and next;
		if ( $metric_name =~ /gpnp_alert_metric/ and $dp_metric_name =~ /gpnp_server_pool/ )
		{
		  my $sglist;
		  $sglist = $elref->{elem_attribs}{SERVER_POOLS} if $elref 
                      and ref($elref) =~ /HASH/i 
                      and $elref->{elem_attribs} 
		      and $elref->{elem_attribs}{SERVER_POOLS};
		  my $rtype;
		   $rtype = $elref->{elem_attribs}{TYPE} if $elref->{elem_attribs}{TYPE};
		  if ( $rtype and $rtype =~ /ora.database.type/ )
                  {
		   if ( $sglist )
                   {
                    my @sgs = split /\s+/,$sglist if $sglist;
                    for my $sg ( @sgs )
                    {
                      $sg =~ s/^\s+|\s+$//;
                      next unless $sg;
 
   		      if ( $rsref 
   			   and ref($rsref) =~ /HASH/i  and $rsref->{$dp_metric_name} 
                           and keys %{$rsref->{$dp_metric_name}} 
   			   and $rsref->{$dp_metric_name}{$sg} )
   		      {
   		        my $pp = $rsref->{$dp_metric_name}{$sg}{PARENT_POOLS} 
                         if $rsref->{$dp_metric_name}{$sg}{PARENT_POOLS};
   
   		        $pp =~ s/\s+//g if $pp;
   
                        # if the server pools do no thave parent pools then it is policy managed
                        # an admin managed db can have only one server pool
                        if ( $pp )
                        {
                          $dbPolicyManaged = 'NO' if @sgs < 2;
                          $dbPolicyManaged = 'YES' if @sgs > 1;
                        }
                        else
                        {
                          $dbPolicyManaged = 'YES';
                          last if @sgs > 1;
                        }
                      }
                      else
                      {
                         $dbPolicyManaged = 'NO' if @sgs < 2;
                         $dbPolicyManaged = 'YES' if @sgs > 1;
                      }
                    }
                   }
                   else
                   {
                       $dbPolicyManaged = 'NO';
                   }
                  }
                }
        
            }
  if ( $dbPolicyManaged )
  {
   if ( $attribbody )
   {
     $attribbody="DB_POLICY_MANAGED_COMPUTED=$dbPolicyManaged\;$attribbody";
   }
   else
   {
     $attribbody="DB_POLICY_MANAGED_COMPUTED=$dbPolicyManaged\;";
   }
  }
  if ( $attribbody )
  {
    $elref->{elem_attribs}{ATTRIB_BODY}=$attribbody;
    $elref->{elem_attribs}{ATTRIB_BODY1}=substr($attribbody,0,2000);
    
    if ( length($attribbody) > 2000 )
    {
      $elref->{elem_attribs}{ATTRIB_BODY2}=substr($attribbody,2000,2000); 
    }
  }
  # build the em_results row
  has::Common::has_em_results($elref,$rsref,$metric_name);
  
  return 1;
  
}
# name : has_gpnp_server_pool_fn
# desc : for each element passed it filters our the enity element
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
#  crs home
#  filter metric column
#  filter metric column value
#  filter metric verb
#
sub has_gpnp_server_pool_fn($$$;@)
{
  my ( $elref, $rsref, $metric_name ,$crsHome, $args) = @_;
  my $filter_col;
  my $filter_val;
  my $filter_verb;
  ($filter_col,$filter_val,$filter_verb ) = @{$args} if $args and ref($args) =~/ARRAY/i;
  # check if this represents a element of interest
  return 1 
   unless $elref->{element} and $elref->{element} =~ /^entity$/i;
  # does this metric need instrumenting an entity type
  # if no entity type is defined the metric may be a cached from
  #  another run
  return 1 unless $has::Common::has_metric_config{entity_type}{$metric_name};
  return 1
   unless $elref->{attrs} and 
    $elref->{attrs}{entity_type} and 
     $elref->{attrs}{entity_name} and
       $elref->{attrs}{entity_type} =~ 
        /^$has::Common::has_metric_config{entity_type}{$metric_name}$/i;
  # save entity type and entity name as elem attribs so they are 
  # easy to print
  # when iterating over attribs to print
  $elref->{elem_attribs}{entity_type} = $elref->{attrs}{entity_type};
  $elref->{elem_attribs}{entity_name} = $elref->{attrs}{entity_name};
  # check if it has attributes
  return 1 unless $elref->{children} and 
   $elref->{child_elements}{attributes}
    and ref($elref->{child_elements}{attributes}) =~ /ARRAY/i
     and @{$elref->{child_elements}{attributes}};
  my $attribsref = @{$elref->{child_elements}{attributes}}[0];
  return 1 unless $attribsref->{children} and
    $attribsref->{child_elements}{attribute}
     and  ref($attribsref->{child_elements}{attribute}) =~ /ARRAY/i
      and @{$attribsref->{child_elements}{attribute}};
  my $valuefld = 'value';
  $valuefld = $has::Common::has_metric_config{$metric_name}{value_property} if $has::Common::has_metric_config{$metric_name}{value_property};
  my $attribbody='';
  # create a name value array from the attributes
  for my $attribref ( @{$attribsref->{child_elements}{attribute}} )
  {
     # get ref to the  and <[value]> element from attrib children
     my $nameref;
     my $valueref;
     next unless $attribref->{children};
     for my $cref ( @{$attribref->{children}} )
     {
       next unless $cref and $cref->{element} and $cref->{element} =~ /^(name|$valuefld)$/;
       $nameref = $cref if $cref->{element} =~ /^name$/;
       $valueref = $cref if $cref->{element} =~ /^$valuefld$/;
     }
     # if we do not get xx then skip this attribute
     next unless $nameref and $valueref and $nameref->{name};
     my $value = $valueref->{name};
     $elref->{elem_attribs}{$nameref->{name}}= $value;
     # save all the elements of the  attribute as hash members in elref
     # this can be used for dependency values
     # added them as attribs not to confuse them with elem_attribs{name}
     # we mitght have a attribute name like MANDATORY etc
     #  which matches element name MANDATORY
     for my $cref ( @{$attribref->{children}} )
     {
       next unless $cref and $cref->{element};
       $elref->{elem_attribs_properites}{$nameref->{name}}{$cref->{element}}= $cref->{name};
     }
     $value='' unless defined $value;
  }
  if ( $elref and $elref->{elem_attribs} 
        and $elref->{elem_attribs}{NAME}
        and ref($elref->{elem_attribs}) =~ /HASH/i
        and keys %{$elref->{elem_attribs}} 
     )
  {
    for my $attrib ( keys %{$elref->{elem_attribs}} )
    {
     $rsref->{$metric_name}{$elref->{elem_attribs}{NAME}}
       {$attrib}  =  $elref->{elem_attribs}{$attrib};
    }
  }
  return 1;
  
}
# name : has_resource_crs_servers_fn
# desc : for each element passed it filters our the enity element
#
# arg  :
#  ref to xml element to be filtered
#  ref to xml ref to be returned back
#  metric_name being processed
#  crs home
#  filter metric column
#  filter metric column value
#  filter metric verb
#
sub has_resource_crs_servers_fn($$$;@)
{
  my ( $elref, $rsref, $metric_name ,$crsHome, $args) = @_;
  my $filter_col;
  my $filter_val;
  my $filter_verb;
  ($filter_col,$filter_val,$filter_verb ) = @{$args} if $args and ref($args) =~/ARRAY/i;
  # check if this represents a element of interest
  return 1 
   unless $elref->{element} and $elref->{element} =~ /^entity$/i;
  # does this metric need instrumenting an entity type
  # if no entity type is defined the metric may be a cached from
  #  another run
  return 1 unless $has::Common::has_metric_config{entity_type}{$metric_name};
  return 1
   unless $elref->{attrs} and 
    $elref->{attrs}{entity_type} and 
     $elref->{attrs}{entity_name} and
       $elref->{attrs}{entity_type} =~ 
        /^$has::Common::has_metric_config{entity_type}{$metric_name}$/i;
  # save the cluster name and such globals as they are needed in every row to be printed
  $elref->{elem_attribs}{METRIC_SOURCE}= 'AGENT_GENERATED';
  # event type for resource state change events
  $elref->{elem_attribs}{EONS_EVENT_NAME}= 'CRS_SERVER_STATE_CHANGE';
  # the timestamp for the metric generation in GMT
  my $time = time;
  my ($sec,$min,$hr,$mday,$mon,$year,$wday,$yday) = gmtime($time);
  $year += 1900 if $year;
  $mon += 1 if $mon;
  my $timestring = sprintf("%04d-%02d-%02d %02d:%02d:%02d",$year,$mon,$mday,$hr,$min,$sec);
  $elref->{elem_attribs}{TIMESTAMP}= $timestring;
  # save entity type and entity name as elem attribs so they are 
  # easy to print
  # when iterating over attribs to print
  $elref->{elem_attribs}{entity_type} = $elref->{attrs}{entity_type};
  $elref->{elem_attribs}{entity_name} = $elref->{attrs}{entity_name};
  # for now we kep oracle_name same as entity name
  $elref->{elem_attribs}{oracle_name} = $elref->{attrs}{entity_name};
  # check if it has attributes
  return 1 unless $elref->{children} and 
   $elref->{child_elements}{attributes}
    and ref($elref->{child_elements}{attributes}) =~ /ARRAY/i
     and @{$elref->{child_elements}{attributes}};
  my $attribsref = @{$elref->{child_elements}{attributes}}[0];
  return 1 unless $attribsref->{children} and
    $attribsref->{child_elements}{attribute}
     and  ref($attribsref->{child_elements}{attribute}) =~ /ARRAY/i
      and @{$attribsref->{child_elements}{attribute}};
  my $valuefld = 'value';
  $valuefld = $has::Common::has_metric_config{$metric_name}{value_property} if $has::Common::has_metric_config{$metric_name}{value_property};
  my $attribbody='';
  # create a name value array from the attributes
  for my $attribref ( @{$attribsref->{child_elements}{attribute}} )
  {
     # get ref to the  and <[value]> element from attrib children
     my $nameref;
     my $valueref;
     next unless $attribref->{children};
     for my $cref ( @{$attribref->{children}} )
     {
       next unless $cref and $cref->{element} and $cref->{element} =~ /^(name|$valuefld)$/;
       $nameref = $cref if $cref->{element} =~ /^name$/;
       $valueref = $cref if $cref->{element} =~ /^$valuefld$/;
     }
     # if we do not get xx then skip this attribute
     next unless $nameref and $valueref and $nameref->{name};
     my $value = $valueref->{name};
     $elref->{elem_attribs}{$nameref->{name}}= $value;
     # save all the elements of the  attribute as hash members in elref
     # this can be used for dependency values
     # added them as attribs not to confuse them with elem_attribs{name}
     # we mitght have a attribute name like MANDATORY etc
     #  which matches element name MANDATORY
     for my $cref ( @{$attribref->{children}} )
     {
       next unless $cref and $cref->{element};
       $elref->{elem_attribs_properites}{$nameref->{name}}{$cref->{element}}= $cref->{name};
     }
     $value='' unless defined $value;
     if ( $attribbody )
     {
       $attribbody = "$attribbody$nameref->{name}=$value\;";
     }
     else
     {
       $attribbody = "$nameref->{name}=$value\;";
     }
  }
  if ( $attribbody )
  {
    $elref->{elem_attribs}{ATTRIB_BODY}=$attribbody;
    $elref->{elem_attribs}{ATTRIB_BODY1}=substr($attribbody,0,2000);
    
    if ( length($attribbody) > 2000 )
    {
      $elref->{elem_attribs}{ATTRIB_BODY2}=substr($attribbody,2000,2000); 
    }
  }
  # generate key value if required
  my $gen_key_value = has_gen_key_value($elref,$metric_name);
  
  $elref->{elem_attribs}{GENERATED_KEY_1}=$gen_key_value;
  if ( $elref and $elref->{elem_attribs} 
        and $elref->{elem_attribs}{NAME}
        and ref($elref->{elem_attribs}) =~ /HASH/i
        and keys %{$elref->{elem_attribs}} 
     )
  {
    for my $attrib ( keys %{$elref->{elem_attribs}} )
    {
     $rsref->{$metric_name}{$elref->{elem_attribs}{NAME}}
       {$attrib}  =  $elref->{elem_attribs}{$attrib};
    }
  }
  return 1;
  
}
#------------------------------------------------------------------------------
# FUNCTION :    fnnpprnd
#
# DESC
#   print each node , to be called from the depth first tree traversal fn
#   prints the nested tree with braces
#
# ARGUMENTS
#  node, 
#  traverse tag
#  stack
#  string to print
#  hash of depth and nodes to be closed
#
# RETURN
#------------------------------------------------------------------------------
sub fnnpprnd ( $$$$$ )
{
  my ($node, $tvtag , $stack_ref, $strg_ref, $has_indnthsh) = @_;
  warn "WARN:has::Common::fnnpprnd:entity passed is not a reference\n" and return unless ref($node);
  $$strg_ref='' unless $$strg_ref;
  my $type = 'resource';
  #start to print from relationships, do not print root nodes
  if ( not $node->{entity_type} or $node->{entity_type} !~/relationship/ )
  {
    $$strg_ref = "$$strg_ref(" and return 1;
  }
  $$strg_ref="$$strg_ref(";
  if ( $node->{type} )
  {
    $$strg_ref="$$strg_ref$node->{type}:";
  }
  else
  {
    $$strg_ref="$$strg_ref:";
  }
  if ( $node->{attrib1} and $node->{attrib1} !~ /^NA$/ )
  {
    $$strg_ref="$$strg_ref$node->{attrib1}:";
  }
  else
  {
    $$strg_ref="$$strg_ref:";
  }
  if ( $node->{attrib2} and $node->{attrib2} !~ /NA/ )
  {
    $$strg_ref="$$strg_ref$node->{attrib2}:";
    $type = 'resourcetype' if $node->{attrib2} =~ /^type$/ and $tvtag !~ /^dependents$/;
  }
  else
  {
    $$strg_ref="$$strg_ref:";
  }
  $$strg_ref="$$strg_ref<$type>";
  if ( $node->{resource} )
  {
    $$strg_ref="$$strg_ref$node->{resource}";
  }
  $$strg_ref="$$strg_ref$type>";
  return 1 if defined $node->{$tvtag} and $node->{$tvtag} and keys %{$node->{$tvtag}};
  
  # close any parent nodes that need to be closed
  # How deep is the node
  for ( sort {$b<=>$a} keys %{$stack_ref} )
  {
    # if there are children at a particular level, then do not close the braces
    $$strg_ref="$$strg_ref, " and last if $has_indnthsh->{$_};
    $$strg_ref="$$strg_ref)";
  }    
  
  return 1;
    
}
#------------------------------------------------------------------------------
# FUNCTION :    fnnpprdp
#
# DESC
#   mark depth and parent for each node , to be called from the depth first tree traversal fn
#
# ARGUMENTS
#  node, 
#  traverse tag
#  stack
#  string to print
#  hash of depth and nodes to be closed
#
# RETURN
#------------------------------------------------------------------------------
sub fnnpprdp ( $$$$$ )
{
  my ($node, $tvtag , $stack_ref, $strg_ref, $has_indnthsh) = @_;
  warn "WARN:has::Common::fnnpprdp:entity passed is not a reference\n" and return unless ref($node);
  my $depth = 0;
  # close any parent nodes that need to be closed
  # How deep is the node
  for ( sort {$b<=>$a} keys %{$stack_ref} )
  {
    $depth = $_;
    last;
  }    
  $node->{depth}=$depth;
  
  return 1;
    
}
#------------------------------------------------------------------------------
# FUNCTION :    fnnpprnb
#
# DESC
#   print each node , to be called from the breadth first tree traversal fn
#   keeps an hash of the string of resources by depth
#
# ARGUMENTS
#  node, 
#  traverse tag
#  ref to hash of results 
#
# RETURN
#------------------------------------------------------------------------------
sub fnnpprnb ( $$$ )
{
  my ($node, $tvtag , $res_ref ) = @_;
  warn "WARN:has::Common::fnnpprnb:entity passed is not a reference\n" and return unless ref($node);
  warn "WARN:has::Common::fnnpprnb:entity passed does not have a name \n" and return unless $node->{resource};
  my $type = 'resource';
  my $strg;
  #start to print from relationships, do not print root nodes
  if ( not $node->{entity_type} or $node->{entity_type} !~/relationship/ )
  {
    return 1;
  }
  if ( $node and $node->{depth} and $node->{depth} == 1 )
  {
    return 1;
  }
  warn "WARN:has::Common::fnnpprnb:entity passed does not have a dependee \n" and return unless $node->{dependee};
  return 1 if $res_ref and $res_ref->{key} and $res_ref->{key}{$node->{dependee}} 
    and $res_ref->{key}{$node->{dependee}}{$node->{resource}};
  if ( $node->{type} )
  {
    $strg = 'START' if $node->{type} =~ /START_DEP/;
    $strg = 'STOP' if $node->{type} =~ /STOP_DEP/;
  }
  if ( $node->{attrib1} and $node->{attrib1} !~ /^NA$/ )
  {
    $strg="$strg:$node->{attrib1}:";
  }
  if ( $node->{attrib2} and $node->{attrib2} !~ /NA/ )
  {
    $type = 'resourcetype' if $node->{attrib2} =~ /^type$/ and $tvtag !~ /^dependents$/;
  }
  $strg="$strg<$type>";
  if ( $node->{resource} )
  {
    $strg="$strg$node->{resource}";
  }
  $strg="$strg$type>";
  $res_ref->{key}{$node->{dependee}}{$node->{resource}}=1;
  $res_ref->{results}{$node->{dependee}} = "$res_ref->{results}{$node->{dependee}},$strg" 
   if $res_ref->{results}{$node->{dependee}};
  $res_ref->{results}{$node->{dependee}}=$strg unless $res_ref->{results}{$node->{dependee}};
  return 1 if defined $node->{$tvtag} and $node->{$tvtag} and keys %{$node->{$tvtag}};
  
  $res_ref->{results}{$node->{resource}}='';
  return 1;
    
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetResourceDependencyString
#
# DESC
#  return the resource dependency as a string
#
# ARGUMENTS
#  crsHome if known
#
# RETURN
#  ref to the hash to be filled in with values
#------------------------------------------------------------------------------
sub hasGetResourceDependencyString($;)
{
  my ( $crsHome ) = @_;
  # get the resource information
  my $cmd='emcrsp em config -e resource';
  
  my $reslistref;
  # no cluster home is passed then 
  # get the cluster home from the env 
  #  and if not available or no emcrsp then discover one
  if ( not $crsHome )
  {
    $crsHome = has::Common::hasGetCRSHome();
    # get the cluster home by discovery
    if ( not $crsHome or not has::Common::hasCheckForEmcrsp($crsHome) )
    {
      $crsHome = has::Common::hasGetEnvCRSHome();
    }
  }
  return $reslistref 
   and warn "WARN:has::Common::hasGetResourceDependencyString:Cluster home is not passed or not set in env" 
   unless $crsHome;
  return $reslistref 
   and warn "DEBUG:has::Common::hasGetResourceDependencyString:binary emcrsp is not present in $crsHome" 
   unless has::Common::hasCheckForEmcrsp($crsHome);
  has::Common::hasSetCRSEnv($crsHome);
  my $cmdresults = has::Common::runsystemcommand($cmd);
  has::Common::hasRestoreCRSEnv();
  $cmdresults = has::Common::hasCheckAndReturnEmcrspResults($cmdresults);
  warn "WARN:has::Common::hasGetResourceDependencyString Failed to get results from command - $cmd" 
   and return $reslistref unless $cmdresults;
  my  %xmlvar =  has::Common::parse_xml($cmdresults);
  $has::Common::has_metric_config{dependent_metric_functions}{resources} =
  { 
   resource_dependencies => \&has::Common::has_dependencies,
  };
  my %resulthash;
  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&has_entity_fn,\%resulthash,'resources');
  return unless $resulthash{resource_dependencies};
  my @rows = @{$resulthash{resource_dependencies}};
  my $rel = 0;
  my $depends_rel_index;
  my $dependent_rel_index;
  my $depends_on_index;
  my $dependent_index;
  
  for my $row ( @rows )
  {
      $row =~ s/^\s+|\s+$// if $row;
      chomp($row);
  
      next unless $row;
  
      $row =~ s/^em_result=//g;
  
      my @rowcols = split/\|/,$row;
      my ( $cn,$res,$typ,$child,$atb1,$atb2) = @rowcols;
  
      $rel++;
       
      # e.g. this is the relationship rel where there is an dependency on emres2 for emres1
      $depends_rel_index->{$rel} = { entity_type=>'relationship', type=>$typ, resource=>$child, attrib1=>$atb1, attrib2=>$atb2 };
      # e.g. this is the relationship rel where there is an dependency where emres1 depends on emres2
      $dependent_rel_index->{$rel} = { entity_type=>'relationship', type=>$typ, resource=>$res, attrib1=>$atb1, attrib2=>$atb2 };
  
      # this is the start|stop dependencies of a resource
      # e.g. resource emres1 depends on resource emres2
      $depends_on_index->{depends_on}{$res}{depends_on}{$rel}=$rel;
      $depends_on_index->{depends_on}{$res}{resource}=$res;
      $depends_on_index->{depends_on}{$res}{entity_type}='resource';
  
      # resources that depend on a given resource
      # e.g. resource emres2 has resource emres1 dependent on it thru relation rel
      #$dependent_index->{dependents}{$child}{$rel}=$res;
      $dependent_index->{dependents}{$child}{dependents}{$rel}=$rel;
      $dependent_index->{dependents}{$child}{resource}=$child;
      $dependent_index->{dependents}{$child}{entity_type}='resource';
  
  }
  
  # for each resource that depends on a given resource
  # for each resource that are dependents of emres2
  if ( $dependent_index and $dependent_index->{dependents} )
  {
    for my $resnm ( keys %{$dependent_index->{dependents}} )
    {
      # are there other resources that this resource depends on
      # is emres2 in turn dependent on other resources
      next unless $depends_on_index and $depends_on_index->{depends_on} 
       and $depends_on_index->{depends_on}{$resnm};
    
      # for every relationship where you have another resource dependening on this
      # resource mark that it has dependencies on other resources in turn
      # take the dependents emres2 and slap them on to all relationships
      # where other resources are dependent on emres2, so you get the chain of dependents
      for my $rel ( keys %{$dependent_index->{dependents}{$resnm}{dependents}} )
      {
        for my $upperrel ( keys %{$depends_on_index->{depends_on}{$resnm}{depends_on}} )
        {
          $depends_rel_index->{$rel}{depends_on}{$upperrel}=$upperrel;
        }
      }
    
    }
  }
  
  # for each resource that depends on other resource
  # for each resource that emres1 depends on
  if ( $depends_on_index and $depends_on_index->{depends_on} )
  {
    for my $resnm ( keys %{$depends_on_index->{depends_on}} )
    {
      # are there other resources that are dependents of this
      # other resources which are dependent on emres2
      next unless $dependent_index and $dependent_index->{dependents} 
       and $dependent_index->{dependents}{$resnm};
    
      # for every relationship where you have another resource dependening on this
      # resource mark that it has dependencies on other resources in turn
      # take the dependents emres2 and slap them on to all relationships
      # where other resources are dependent on emres2, so you get the chain of dependents
      for my $rel ( keys %{$depends_on_index->{depends_on}{$resnm}{depends_on}} )
      {
        for my $upperrel ( keys %{$dependent_index->{dependents}{$resnm}{dependents}} )
        {
          $dependent_rel_index->{$rel}{dependents}{$upperrel}=$upperrel;
        }
      }
    
    }
  }
  
  my %has_sort_dep_index;
  my %res;
  for my $key ( keys %{$depends_on_index->{depends_on}} , keys %{$dependent_index->{dependents}} )
  {
    my $node;
    my $strg;
  
    next if $has_sort_dep_index{$key};
    $has_sort_dep_index{$key}=1;
  
    if ( $depends_on_index->{depends_on}{$key} )
    {
      $node = $depends_on_index->{depends_on}{$key};
  
      fntrdf('preorder',$node,'depends_on',$depends_rel_index,&fnnpprnd,\$strg);
      $res{$key}{dependent_on}=$strg if $strg;
      
    }
  
    next unless $dependent_index->{dependents}{$key};
  
    $node = $dependent_index->{dependents}{$key};
  
    $strg='';
  
    fntrdf('preorder',$node,'dependents',$dependent_rel_index,&fnnpprnd,\$strg);
  
    $res{$key}{dependents}=$strg if $strg;
  
  }
  
  return \%res;
}
#------------------------------------------------------------------------------
# FUNCTION :    hasGetResourceDependents
#
# DESC
#  return the resource dependets as a couple
#
# ARGUMENTS
#  crsHome if known
#
# RETURN
#  ref to the hash to be filled in with values
#------------------------------------------------------------------------------
sub hasGetResourceDependents($;)
{
  my ( $crsHome ) = @_;
  # get the resource information
  my $cmd='emcrsp em config -e resource';
  
  my $reslistref;
  # no cluster home is passed then 
  # get the cluster home from the env 
  #  and if not available or no emcrsp then discover one
  if ( not $crsHome )
  {
    $crsHome = has::Common::hasGetCRSHome();
    # get the cluster home by discovery
    if ( not $crsHome or not has::Common::hasCheckForEmcrsp($crsHome) )
    {
      $crsHome = has::Common::hasGetEnvCRSHome();
    }
  }
  return $reslistref 
   and warn "WARN:has::Common::hasGetResourceDependents:Cluster home is not passed or not set in env" 
   unless $crsHome;
  return $reslistref 
   and warn "DEBUG:has::Common::hasGetResourceDependents:binary emcrsp is not present in $crsHome" 
   unless has::Common::hasCheckForEmcrsp($crsHome);
  has::Common::hasSetCRSEnv($crsHome);
  my $cmdresults = has::Common::runsystemcommand($cmd);
  has::Common::hasRestoreCRSEnv();
  $cmdresults = has::Common::hasCheckAndReturnEmcrspResults($cmdresults);
  warn "WARN:has::Common::hasGetResourceDependents Failed to get results from command - $cmd" 
   and return $reslistref unless $cmdresults;
  my  %xmlvar =  has::Common::parse_xml($cmdresults);
  $has::Common::has_metric_config{dependent_metric_functions}{resources} =
  { 
   resource_dependencies => \&has::Common::has_dependencies,
  };
  my %resulthash;
  has::Common::traverse_xml(\%xmlvar,\&has::Common::has_handle_error,\&has_entity_fn,\%resulthash,'resources');
  return unless $resulthash{resource_dependencies};
  my @rows = @{$resulthash{resource_dependencies}};
  my $rel = 0;
  my $depends_rel_index;
  my $dependent_rel_index;
  my $depends_on_index;
  my $dependent_index;
  
  for my $row ( @rows )
  {
      $row =~ s/^\s+|\s+$// if $row;
      chomp($row);
  
      next unless $row;
  
      $row =~ s/^em_result=//g;
  
      my @rowcols = split/\|/,$row;
      my ( $cn,$res,$typ,$child,$atb1,$atb2) = @rowcols;
  
      $rel++;
       
      # e.g. this is the relationship rel where there is an dependency where emres1 depends on emres2
      $dependent_rel_index->{$rel} = { dependee=>$child, entity_type=>'relationship', type=>$typ, resource=>$res, attrib1=>$atb1, attrib2=>$atb2 };
  
      # e.g. this is the relationship rel where there is an dependency on emres2 for emres1
      $depends_rel_index->{$rel} = { entity_type=>'relationship', type=>$typ, resource=>$child, attrib1=>$atb1, attrib2=>$atb2 };
      # this is the start|stop dependencies of a resource
      # e.g. resource emres1 depends on resource emres2
      $depends_on_index->{depends_on}{$res}{depends_on}{$rel}=$rel;
      $depends_on_index->{depends_on}{$res}{resource}=$res;
      $depends_on_index->{depends_on}{$res}{entity_type}='resource';
  
      # resources that depend on a given resource
      # e.g. resource emres2 has resource emres1 dependent on it thru relation rel
      #$dependent_index->{dependents}{$child}{$rel}=$res;
      $dependent_index->{dependents}{$child}{dependents}{$rel}=$rel;
      $dependent_index->{dependents}{$child}{resource}=$child;
      $dependent_index->{dependents}{$child}{entity_type}='resource';
  }
  # for each resource that depends on other resource
  # for each resource that emres1 depends on
  if ( $depends_on_index and $depends_on_index->{depends_on} )
  {
    for my $resnm ( keys %{$depends_on_index->{depends_on}} )
    {
      # are there other resources that are dependents of this
      # other resources which are dependent on emres2
      next unless $dependent_index and $dependent_index->{dependents} 
       and $dependent_index->{dependents}{$resnm};
    
      # for every relationship where you have another resource dependening on this
      # resource mark that it has dependencies on other resources in turn
      # take the dependents emres2 and slap them on to all relationships
      # where other resources are dependent on emres2, so you get the chain of dependents
      for my $rel ( keys %{$depends_on_index->{depends_on}{$resnm}{depends_on}} )
      {
        for my $upperrel ( keys %{$dependent_index->{dependents}{$resnm}{dependents}} )
        {
          $dependent_rel_index->{$rel}{dependents}{$upperrel}=$upperrel;
        }
      }
    
    }
  }
  
  my %has_sort_dep_index;
  my %res;
  for my $key ( keys %{$dependent_index->{dependents}} )
  {
    my $node;
    my %dep_res = ();
    my $strg = '';
  
    next if $has_sort_dep_index{$key};
    $has_sort_dep_index{$key}=1;
  
    $node = $dependent_index->{dependents}{$key};
  
    #fntrdf('preorder',$node,'dependents',$dependent_rel_index,&fnnpprdp,\$strg);
    fntrbf('preorder',$node,'dependents',$dependent_rel_index,&fnnpprnb,\%dep_res);
    $res{$key}=$dep_res{results} if $dep_res{results};
  
  }
  
  return \%res;
}
1;