#!/usr/local/bin/perl
# 
# Copyright (c) 2002, 2009, Oracle and/or its affiliates.All rights reserved. 
#
#    NAME
#      dgutil_o.pl
#
#    DESCRIPTION
#      Data Guard 10.2 utility routines
#
#    NOTES
#      <other useful comments, qualifications, etc.>
#
#    MODIFIED   (MM/DD/YY)
#    ngade       01/30/09 - Code slap 10.2.0.5.0 -> 11.2 round 2
#    ngade       08/03/08 - Code Slap 10.2.0.5GC -> 11.2SA
#    ngade       06/29/08 - 
#    rimmidi     05/04/08 - Code slap from 11GC to 10.2.0.5
#    ngade       08/07/07 - fix bug 6211060
#    ngade       06/12/07 - Backport ngade_bug-4880147 from main
#    ngade       04/10/06 - escape quotes in password 
#    ngade       10/18/06 - Backport ngade_bug-5141850 from main
#    ngade       08/07/06 - Backport sjconnol_dg_ha_obs_051031 from main
#    ngade       09/12/06 - add observer 11g metric
#    ngade       06/06/06 - fix bug 5287905 
#    ngade       05/09/06 - reduce wait 
#    sjconnol    01/11/06 - Don't verify restart host is same as original
#    sjconnol    09/12/05 - Add retry to obs restart
#    sjconnol    08/12/05 - Move observer host check
#    sjconnol    07/05/05 - Verify observer host
#    sjconnol    06/17/05 - Bug 4439779: revert to dgutil.pl
#    sjconnol    06/14/05 - Observer restartability
#    sjconnol    06/03/05 - Bug 4407481: include dgutil_core.pl
#    sjconnol    03/21/05 - Bug 4252598
#    sjconnol    02/01/05 - Conditionalize observer debug logging
#    sjconnol    10/01/04 - Use unique name for observer lock file
#    sjconnol    09/13/04 - Creation
# 
use Sys::Hostname;
require "$ENV{EMDROOT}/sysman/admin/scripts/db/dg/dgutil.pl";

## This routine is intended to be called from a corrective action job
##  in response to a warning/error from the DG status metric
sub DGrestartObserver{

  local($SIG{__DIE__}) = \&handleError;
  local($SIG{__WARN__}) = \&handleWarning;
  my($old_fh) = select(STDOUT);
  $| = 1;
  select($old_fh);

  debug("dgutil_o.DGrestartObserver: begin");
  my($user, $pw, $tns, $home, $debug) = @_;
  
  ## Obtain DG status. If status indicates observer is down, proceed
  ## (Use full conn address here; db may be on a different host)
  my $mode = 2; # SYSDBA
  my($lda) = DBI->connect('dbi:Oracle:', "$user@".$tns , "$pw", 
                          {ora_session_mode => $mode,
                           PrintError => 0,
                           RaiseError => 1});

  if((defined($checkObserver) && $checkObserver > 1) || DGgetObserverStatus($lda)){
    debug("dgutil_o.DGrestartObserver: Attempting observer stop/start");
    print("\nUnobserved condition detected. Attempting observer restart...\n");
    #DGverifyObserverHost($lda);

    ## Retry operation once if it doesn't succeed first time
    ##my $retry = 0;
    ##while(1){

    my $dgmgrl = @_[3]. "/bin/dgmgrl";
    if($NT){
      $dgmgrl = @_[3]. "/bin/dgmgrl.exe";
    }

     ## If dgmgrl does not exist or the user does not have executable permission exit the perl script
    if(!( -e $dgmgrl) || !(-x $dgmgrl))
    {
    debug("dgutil_o.DGrestartObserver: Could not find or execute $dgmgrl");
    print("\n Could not locate $dgmgrl or the user does not have executable permissions \n");
      exit (1);
    }

      ## Stop observer
      DGstopObserver($lda);
      ## Start observer
      if(DGstartObserver(@_)){
        print("\nObserver restarted.\n");
        ## Give time for 16820 error to clear, so metric reload will catch it.
        sleep(60);
        ##last;
    }
      ##$retry++;
      ##if($retry > 1){
        ##last;
      ##}
      ##debug("dgutil_o.DGrestartObserver: Observer restart failed; retrying in 60 sec...");
      ##print("\nObserver restart failed; retrying...\n");
      ##sleep(60);
    ##}
  }
  else{
    debug("dgutil_o.DGrestartObserver: Observer restart not required");
    print("Unobserved condition not detected. Observer restart not required.\n");
  }
  
  debug("dgutil_o.DGrestartObserver: end");
}

sub DGverifyObserverHost{
  my ($lda) = @_;
  ## This job is running on the original observer host
  my ($orighost) = hostname();
  debug("dgutil_o.DGverifyObserverHost: begin");
  debug("dgutil_o.DGverifyObserverHost: original observer host: $orighost");
  ## Verify the current observer host is the same as the host
  ##  the job is running against
  my $sql =  "select FS_FAILOVER_OBSERVER_HOST from v\$database";
  my $dbcur = $lda->prepare($sql);
  $dbcur->execute;
  my @row = $dbcur->fetchrow_array();
  my($curhost) = $row[0];
  debug("dgutil_o.DGverifyObserverHost: current observer host: $curhost");

  ## Match up to first .
	my($idx) = index($orighost,".");  
  if($idx != -1){
    $orighost = substr($orighost,0,$idx);
    debug("dgutil_o.DGverifyObserverHost: truncating original observer host: $orighost");
  }
	$idx = index($curhost,".");  
  if($idx != -1){
    $curhost = substr($curhost,0,$idx);
    debug("dgutil_o.DGverifyObserverHost: truncating current observer host: $curhost");
  }

  if(lc($orighost) ne lc($curhost)){
    debug("dgutil_o.DGverifyObserverHost: current host doesn't match original");
    print("\nThe current observer host is not the same as when the corrective action job was originally configured. Enterprise Manager cannot restart the observer.\n");
    exit(1);
  }
  debug("dgutil_o.DGverifyObserverHost: end");
}
    

sub DGstopObserver{
  debug("dgutil_o.DGstopObserver: begin");
  my ($lda) = @_;
  
  # Retrieve the db resource STATUS property.
  my $indoc = "<DO_CONFIGURE_DRC><STOP_OBSERVER /></DO_CONFIGURE_DRC>";
  my $dbres_big_status = get_dg_document($lda, $indoc);
  # EMD_PERL_DEBUG("dbres_big_status=$dbres_big_status");

  my @dbres_status_tokens = split(/[><]+/, $dbres_big_status);
  # Token 0 is "", token 1 is "RESULT ", token 2 is "MESSAGE "
  my($dbres_status) = $dbres_status_tokens[3];
  my($dbres_status_text);
  my @temp_tokens = split(/\s+/, $dbres_status);
  $dbres_status = $temp_tokens[0];
  EMD_PERL_DEBUG("dgutil_o.DGstopObserver: dbres_status=$dbres_status");

  if ($dbres_status eq "SUCCESS")
  {
    EMD_PERL_DEBUG("dgutil_o.DGstopObserver: Observer stopped");
    return;
  }

  # Parse out the error from the big status.
  my $dbres_status_token;
  my $token_count = 0;
  foreach $dbres_status_token (@dbres_status_tokens)
  {
    # EMD_PERL_DEBUG("dbres_status_token=$dbres_status_token.");
    if ($dbres_status_token eq "ERROR_TEXT ")
    {
      $dbres_status_text = $dbres_status_tokens[$token_count + 1];
      last;
    }
    $token_count++;
  }

  printDebug("dgutil_o.DGstopObserver: Stop observer failed: $dbres_status_text");
  exit(1);
}

## Gather the status for each database
sub DGgetObserverStatus
{
  my ($lda) = @_;
  my $db_version = get_dbversion($lda);
  if (isDB11gOrHigher($db_version))
  { 
    my $sql =  "select count(1) from v\$database where database_role='PRIMARY' and FS_FAILOVER_STATUS !='DISABLED'";
    my $dbcur = $lda->prepare($sql);
    $dbcur->execute;
    my @row = $dbcur->fetchrow_array();
    $dbcur->finish;
    my($valid_count) = $row[0];
    if($valid_count > 0) 
    {
       my $get_observerHB_prop = q{
         begin
           :valid_count := sys.dbms_drs.get_property_obj(0, 'ObserverHB');
         end;
       };
      my $dbcur = $lda->prepare($get_observerHB_prop);
      $dbcur->bind_param_inout(":valid_count", \$valid_count, SQL_INTEGER);
      $dbcur->execute;
      $dbcur->finish;
      if($valid_count > 0)
      {
        EMD_PERL_DEBUG("dgutil_o.DGgetObserverStatus: observer needs to be started");
        return 1;
      }
      else 
      {
        EMD_PERL_DEBUG("dgutil_o.DGgetObserverStatus: observer normal");
        return 0;
      }
    }
    else 
    {
      EMD_PERL_DEBUG("dgutil_o.DGgetObserverStatus: observer disabled");
      return 0;
    }
  }
  else
  {
    # Retrieve the configuration object content.
    my $indoc = "<DO_CONFIGURE_DRC><GET_DRC/></DO_CONFIGURE_DRC>";
    my $drc_obj = get_dg_document($lda, $indoc);
  
    # Retrieve the list of sites.
    my (%site_list);
    my ($site_id, $site_obj);
    my $drc_start_pos = -1;
    while (($site_id = get_dg_token($drc_obj, "site_id", $drc_start_pos)) ne "")
    {
      EMD_PERL_DEBUG("dgutil_o.DGgetConfigStatus: site_id=$site_id");
      # Retrieve the site object content.
      $indoc = "<DO_CONFIGURE_SITE><GET_SITE site_id=\"$site_id\"/></DO_CONFIGURE_SITE>";
      $site_obj = get_dg_document($lda, $indoc);
      $site_list{$site_id} = $site_obj;
    }
  
    foreach $site_id (keys(%site_list))
    {
      $site_obj = $site_list{$site_id};
      my($status) = DGgetDatabaseStatus($lda, $site_obj);
      EMD_PERL_DEBUG("dgutil_o.DGgetConfigStatus: status for site $site_id: $status");
      ## ORA-16820 indicates an unobserved condition
      if($status =~ /16820/){
        EMD_PERL_DEBUG("dgutil_o.DGgetConfigStatus: found unobserved status; observer restart required");
        return 1;
      }
      ## ORA-16824/16825 multiple FSFO errors; must get full status report
      if($status =~ /16824|16825/){
        EMD_PERL_DEBUG("dgutil_o.DGgetConfigStatus: found 16824/16825 status; getting StatusReport");
        my $dbres_id = get_dg_property($lda, $site_id, "DBRESOURCE_ID");
        my $statusrpt = get_dg_property($lda, $dbres_id, "StatusReport");
        #EMD_PERL_DEBUG("dgutil_o.DGgetConfigStatus: StatusReport = $statusrpt");
        my @dbres_status_tokens = split(/[><]+/, $statusrpt);
        my $dbres_status_token;
        foreach $dbres_status_token (@dbres_status_tokens)
        {
          # EMD_PERL_DEBUG("dbres_status_token=$dbres_status_token.");
          if ($dbres_status_token =~ /ORA-16820/)
          {
            EMD_PERL_DEBUG("dgutil_o.DGgetConfigStatus: found unobserved status; observer restart required");
            return 1;
          }
        }
      }
    }
  }
  EMD_PERL_DEBUG("dgutil_o.DGgetConfigStatus: no observer down status");
  return 0;
}

# Process a site object.
sub DGgetDatabaseStatus
{
  my ($lda, $site_obj) = @_;
  my $site_start_pos = -1;

  # Parse out the site id.
  my $site_id = get_dg_token($site_obj, "site_id", $site_start_pos);
  EMD_PERL_DEBUG("dgutil_o.DGgetDatabaseStatus: site_id=$site_id");

  # Retrieve the site DBRESOURCE_ID property.
  my $dbres_id = get_dg_property($lda, $site_id, "DBRESOURCE_ID");
  EMD_PERL_DEBUG("dgutil_o.DGgetDatabaseStatus: dbres_id=$dbres_id");

  # Retrieve the db resource ENABLED property.
  my $dbres_enabled = get_dg_property($lda, $dbres_id, "ENABLED");
  EMD_PERL_DEBUG("dgutil_o.DGgetDatabaseStatus: dbres_enabled=$dbres_enabled");
  # The db resource must be enabled.
  if (!($dbres_enabled =~ /YES/i))
  {
    EMD_PERL_DEBUG("dgutil_o.DGgetDatabaseStatus: The db resource must be enabled.");
    return "";
  }
  # Retrieve the db resource INTENDED_STATE property.
  my $dbres_state = get_dg_property($lda, $dbres_id, "INTENDED_STATE");
  EMD_PERL_DEBUG("dgutil_o.DGgetDatabaseStatus: dbres_state=$dbres_state");
  # The db resource intended state must not be offline.
  if (($dbres_state =~ /OFFLINE/i))
  {
    EMD_PERL_DEBUG("dgutil_o.DGgetDatabaseStatus: The db resource must not be offline.");
    return "";
  }

  # Retrieve the db resource status.
  my $dbres_status = "";
  my $dbres_status_text = "";
  get_dg_dbres_status($lda, $dbres_id, $dbres_status, $dbres_status_text);

  EMD_PERL_DEBUG("dgutil_o.DGgetDatabaseStatus: Site $dbres_id : returning $dbres_status");
  return($dbres_status_text);
}

## Start the AFO observer
sub DGstartObserver{
  debug("dgutil_o.DGstartObserver: begin");
  my($user, $pw, $tns, $home, $debug) = @_;
  set_env_var("$home", "");
  &tempLocFallback();
  my($tempdir) = "${home}${S}rdbms${S}log";
  my($afodir) = "${home}${S}dbs";
  if($NT){
    $tempdir = "${home}${S}rdbms${S}trace";
    $afodir = "${home}${S}database";
  }
  my($tempfile) = "${tempdir}${S}dgmgrl$$.log";
  my($afofile) = "file='${afodir}${S}afo$$.dat'";

  if(! -e $tempdir){
    debug("dgutil_o.DGstartObserver: $tempdir doesn't exist; creating...");
    if(!mkdir($tempdir, 0755)){
      debug("dgutil_o.DGstartObserver: $tempdir creation failed, using $TEMP");
      $tempfile = "${TEMP}${S}dgmgrl$$.log"; 
    }
  }

  ## Don't specify an afo dat file if can't make the directory under Ohome
  if(! -e $afodir){
    debug("dgutil_o.DGstartObserver: $afodir doesn't exist; creating...");
    if(!mkdir($afodir, 0755)){
      debug("dgutil_o.DGstartObserver: $afodir creation failed, using default");
      $afofile = ""; 
    }
  }

  debug("dgutil_o.DGstartObserver: log file: $tempfile");
  unlink($tempfile);

  my($flags);
  if($debug){
    $flags = "-debug -xml";
  }
  my($cmd) = "${home}/bin/dgmgrl $flags -logfile $tempfile";

  unless(open(DGIN, "|$cmd")) {
    printDebug("dgutil_o.DGstartObserver: ERROR: could not run ${home}/bin/dgmgrl");
    exit(1);
  }
  my $old_fh = select(DGIN);
  $| = 1;
  select($old_fh);
  debug("dgutil_o.DGstartObserver: preparing to start observer");
  
  ## escape characters before passing into shell
  $pw =~ s/(['"])/\\$1/g;

  print DGIN "connect ${user}/${pw}\@'${tns}';\n";
  print DGIN "start observer ${afofile};\n";

  ## Don't attempt to close the pipe; will block waiting for DGMGRL exit
  debug("dgutil_o.DGstartObserver: observer start sequence completed");

  ## Give obs time to start

  my $wait_count = 0;
  my $started = 0;
  my $failed = 0;

  while($wait_count < 12 && !$started && !$failed){
    $wait_count++;
    sleep(5);

    unless (open(OUTPUT, "$tempfile")) {
      next;
    }

    my(@output);

    #my($size) = -s $tempfile;
    #debug("dgutil_o.DGstartObserver: size of $tempfile : $size");

    if(! -e "$tempfile")
    {
      printDebug("dgutil_o.DGstartObserver: Cannot access observer logfile $tempfile");
      exit(1);
    }

    while(<OUTPUT>){
      #debug("dgutil_o.DGstartObserver: $_");
      push(@output, $_);
      if(/Observer started/i){
        debug("dgutil_o.DGstartObserver: Observer started successfully");
        $started = 1;
        last;
      }
      elsif(/Failed/i){
        printDebug("dgutil_o.DGstartObserver: Observer startup failed:");
        $failed = 1;
        my($out);
        foreach $out (@output){
          printDebug($out);
        }
        last;
      }
    }
    close(OUTPUT);
  }

  #debug("dgutil_o.DGstartObserver:wait time: $wait_count: ".($wait_count*5)." seconds\n");

  if(!$started && !$failed){
    printDebug("dgutil_o.DGstartObserver: Unable to determine observer status");
  }

  debug("dgutil_o.DGstartObserver: end");
  return(!$failed);
}

1;