#!/usr/bin/perl
# 
# $Header: has/install/crsconfig/rootcrs.pl /st_has_11.2.0/41 2011/08/17 23:22:20 xyuan Exp $
#
# rootcrs.pl
# 
# Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. 
#
#    NAME
#      rootcrs.pl - root configuration script for Oracle Clusterware home
#
#    DESCRIPTION
#      rootcrs.pl - root configuration script for Oracle Clusterware home
#
#    NOTES
#      This is run once during configuration or patching of Oracle
#      clusterware home.
#      This script does the following:
#      1) Setup permissions of binaries and DSOs in Oracle Clusterware home
#      2) Setup OLR for storing Oracle local registry data
#      3) Setup GPnP wallet and profile
#      4) Setup and copy files for CLSINIT daemon
#      5) Start CLSINIT daemon
#      6) Copy required CRS resources for CLSINIT to start
#
#      NOTE: This is run during Oracle Clusterware home configurations only.
#            It should not be used for any other homes such as SI HA home,
#            or Oracle RAC homes.
#
#    MODIFIED   (MM/DD/YY)
#    sidshank    07/29/11 - Move help option processing before other tasks
#                           begin (Bug 12802690)
#    xyuan       07/18/11 - Fix bug 12727247
#    xyuan       07/14/11 - Fix bug 12698968,12728939
#    xyuan       07/12/11 - Fix bug 12585291 & add -keepdg option
#    xyuan       07/11/11 - Fix Bug 12701521
#    xyuan       06/04/11 - Fix Bug 12630162.
#    sidshank    05/31/11 - adding noterm option, to be used internally, only on Windows.
#    ksviswan    05/16/11 - Fix Bug 12554154.
#    sidshank    05/10/11 - Backport sidshank_bug-12393931 from main
#    rdasari     05/10/11 - remove OCR check
#    ksviswan    05/08/11 - Remove ORA_CRS_HOME env setting
#    sbezawad    05/03/11 - Backport sbezawad_bug-11936601 from main
#    ksviswan    05/01/11 - implement forceupgrade
#    rdasari     04/29/11 - check for OCR corruption before retrieving any
#                           config data
#    ksviswan    04/13/11 - XbranchMerge ksviswan_opauto_segregate from main
#    agraves     04/08/11 - Add oraacfs.pm to this.
#    dpham       04/06/11 - Backport dpham_bug-9842464 from main
#    rdasari     04/04/11 - add preUpgradeChecks
#    ksviswan    03/08/11 - Backport ksviswan_febbugs2 from main
#    ksviswan    01/20/11 - Backport ksviswan_janbugs2 from main
#    ksviswan    01/18/11 - Backport ksviswan_janbugs1 from main
#    dpham       11/23/10 - Fix bug 10327539 & 10325472
#    ksviswan    02/23/11 - Fix Bug 11787406,11719836
#    dpham       11/16/10 - XbranchMerge dpham_bug-10273727 from st_has_11.2.0
#                         - XbranchMerge dpham_bug-10273236 from st_has_11.2.0
#    dpham       11/09/10 - Fix bug 10273727
#    dpham       11/09/10 - Fix bug 10273236
#    dpham       09/29/10 - Fix bug 10056987
#    dpham       09/11/10 - Fix bug 10099673
#    dpham       08/10/10 - Fix bug 10029119
#    dpham       08/10/10 - Fix bug 10006758
#    ksviswan    08/03/10 - Fix Bug 9954172
#    samjo       07/07/10 - Disable CHM/OS in Windows 11.2.0.2.0
#    ksviswan    07/30/10 - Fix Bug 9957944
#    ksviswan    07/27/10 - Fix Bug 9945578
#    dpham       07/16/10 - Backport dpham_bug-9859193 from main
#    dpham       06/30/10 - Add set_perms_ocr_vdisk() function (9850696)
#    dpham       06/15/10 - Fix bug 9776829
#    ksviswan    06/03/10 - Fix Bugs 9732945,9737629,9773975,9723851 and 9785215
#    dpham       05/19/10 - XbranchMerge dpham_bug9282344 from st_has_11.2.0.1.0
#    khsingh     04/26/10 - add checkpoints
#    dpham       05/15/10 - Fix bug 9697577
#    dpham       05/03/10 - Fix REMOTENODE on NT
#    dpham       04/27/10 - Add UNLOCK (9446443
#    dpham       04/02/10 - Fix bug 7565887
#    dpham       03/18/10 - Call ModActionScript (bug 9257105) 
#    sujkumar    02/14/10 - BDB checks only on supported platforms
#    ksviswan    01/18/10 - Implemented logic for 11.2 patchset upgrade
#    dpham       12/07/09 - Increase wait time to 30 mins for ohasd & crsd
#    dpham       11/17/09 - Increase wait time for ohasd to start
#    dpham       01/07/09 - Execute 'oifcfg setif -global' when upgrade completes
#			  - backupOLR() should be unconditional (8573570)
#    dpham       12/28/09 - Remove -delete option
#    dpham       01/06/09 - Add start/stop "ora.asm" resource (8309620)
#    sujkumar    12/03/09 - Create config and db in symlink -> has_work/crf
#    dpham       11/18/09 - Remove obsolete options
#    dpham       11/24/09 - XbranchMerge dpham_bug-9080632 from st_has_11.2.0.1.0
#    sujkumar    11/16/09 - Dirs and files for IPD/OS in has_work
#    anutripa    11/10/09 - Create run directories for IPD
#    sukumar     11/05/09 - Add hostname to bdbloc to safely create bdb files
#    dpham       10/15/09 - Add '-remotenode' option
#    ksviswan    10/01/09 - CheckPoint Implementation for fresh  Clusterware install
#    sujkumar    09/25/09 - Add changes for IPD/OS
#    dpham       07/15/09 - XbranchMerge dpham_bug-8416640_2 from st_has_11.2.0.1.0
#    dpham       07/13/09 - Add 'no_auto_abbrev' to Getopt() function
#    ksviswan    06/26/09 - change default perl path
#    dpham       06/17/09 - Add '-deconfig' option
#    hchau       05/28/09 - Rewrite Step 9 to clean up the code for stack startup
#    dpham       05/25/09 - Call 'clscfg -lastnode' if it's last node to upgrade
#    dpham       05/21/09 - Add '-lastnode' option in "clscfg -upgrade"
#    hchau       05/12/09 - Start ctssd resource with env var CTSS_REBOOT=TRUE
#                           when starting the stack
#    mimili      05/10/09 - Bug 8496223 and 8488395: remove 'clscfg -upgradeevm', which 
#                           is only used for OCR permission problem in evmd upgrade
#    ksviswan    05/07/09 - Implement downgrade
#    dpham       05/12/09 - Call getUpgradeConfig, configNode, backupOLR
#    dpham       05/05/09 - update_ons_config() function should be called after 
#			    get_OldVipInfo or get_OldVipInfoFromOCRDump
#    dpham       05/05/09 - Backup OLR (8484172)
#    hchau       04/29/09 - Swap startup sequence to 'ctssd' then 'asm'
#    dpham       04/29/09 - Add start_resource(ora.asm) and stop_resource(ora.asm) (8309620).
#			  - Bypass isInterfaceValid() if it's new node (8438342).
#    dpham       04/27/09 - Call s_houseCleaning after successfully upgrade
#    ksviswan    04/25/09 - Add Patching support
#    dpham       04/22/09 - Call isLastNodeToUpgrade for upgrede
#    spavan      03/23/09 - install cvuqdisk rpm as part of root
#    dpham       03/17/09 - Check return code from configLastNode and configNewNode
#    dpham       03/10/09 - Add ORACLE_BASE
#    dpham       03/09/09 - isInterfaceValid() should be called if not UPGRADE
#    jleys       03/07/09 - Remove merge lines
#    jleys       03/07/09 - Remove merge lines
#    agraves     03/06/09 - Add -g option to pass ASMGRP for SYSTEM.ACFS to
#                           clscfg.
#    dpham       02/22/09 - Call isCRSAlreadyConfigured to check if CRS is
#                           already configured
#    jleys       02/06/09 - Remove expansion of VF discovery string
#    dpham       02/26/09 - Remove add_acfs_registry
#    dpham       02/23/09 - Call isInterfaceValid to check if interface is valid
#    dpham       02/22/09 - Call isCRSAlreadyConfigured to check if CRS is already configured
#    dpham       02/11/09 - Call add_acfs_registry
#    sravindh    02/11/09 - Bugfix 7714358
#    dpham       02/04/09 - Add isRAC_appropriate to check for rac_on/rac_off
#    dpham       01/27/09 - Add -unlock option
#    dpham       01/20/09 - Remove configure_ACFS
#    rsreekum    01/16/09 - fix bug 7678799
#    dpham       12/29/08 - Fix syntax error on isLastNodeToStart function
#    ksviswan    12/26/08 - Fix bug 7561694
#    jleys       11/29/08 - Set delete flag in $cfg
#    jleys       11/26/08 - Correct case of method
#    jleys       11/14/08 - Packagize perl scripts
#    dpham       12/11/08 - Remove -firstNode option
#                         - Configure HASD as "NT AUTHORITY\SYSTEM"
#                         - Call "crssetup install" to configure remote nodes
#    dpham       11/24/08 - Add 'clscfg -upgradeevm'
#    dpham       11/13/08 - remove ORA_CRS_HOME param from validate_9iGSD
#    dpham       10/27/08 - add 'srvctl config nodeapps' in get_OldVipInfo
#    psankara    10/23/08 - Use a new crsctl command to obtain VF Disc. string
#    ysharoni    10/20/08 - save results of make_vf_discovery_string in %param
#    rdasari     10/15/08 - remove the temporary changes to copy onsctl
#    dpham       10/09/08 - Trap error output from usm_root so that it gets written
#                           to the CRS log.
#    dpham       10/08/08 - The rootcrs.log is overwritten if ORACLE_HOME is shared.
#    jtellez     10/08/08 - fix asm start w/ storage_option==2
#    jleys       09/24/08 - Do not complete if initial config not successful
#    ppallapo    09/22/08 - Get OCRID and CLUSTER_GUID for GPnP Profile.
#    rdasari     09/12/08 - mv CH/bin/onsctl CH/opmn/bin/onsctli
#    dpham       08/12/08 - OCRCONFIGBIN is undefined.
#    jleys       07/11/08 - Put this program path into perl search path
#    jleys       07/07/08 - Correct typo
#    jleys       04/09/08 - Change css_start to CSS_start
#    jleys       04/05/08 - Change CSS package name
#    jleys       04/01/08 - Consolidate gpnp setup
#    jleys       03/29/08 - Make it easier to get debugging output
#    jleys       02/29/08 - Break out CSS functionality
#    dpham       06/27/08 - pass trace parameters to hasdconfig.pl (bug 6903750)
#    dpham       06/17/08 - fix usmca (bug 7159411)
#    dpham       06/16/08 - Add new node logic
#    averhuls    06/04/08 - Change externally visible USM text to ACFS.
#    srisanka    05/13/08 - replace ORA_CRS_HOME with ORACLE_HOME
#    averhuls    05/06/08 - Add usmfs.Add usmfs
#    samjo       05/02/08 - Use hostname in olr file name
#    hkanchar    05/05/08 - Remove CRS_HOME_ENV for upgrade
#    samjo       05/02/08 - Use hostname in olr file name
#    rxkumar     04/30/08 - remove srvctl add asm
#    srisanka    04/30/08 - Bug 7010466: fix typo
#    srisanka    04/30/08 - pass ASM_DISCOVERY_STRINGto USMCA only if non-NULL
#    dpham       04/28/08 - Add '-delete' for root deconfig  
#    jleys       04/21/08 - Add function call to determine if this is the last
#                           node
#    priagraw    04/16/08 - add gipcd agent start 
#    srisanka    04/09/08 - use USMCA path
#    samjo       04/03/08 - Pass owner and group during OLR format
#    ysharoni    03/31/08 - bug 6895319
#    skakarla    03/27/08 - fixing srvctl path
#    hkanchar    03/30/08 - Refactor upgrade scripts
#    srisanka    03/18/08 - handle stdout/stderr
#    samjo       02/25/08 - Add CTSSD start
#    jtellez     02/20/08 - trace files
#    jtellez     02/18/08 - srvctl clobbers rootcrs.pl output
#    srisanka    02/12/08 - do OSD actions
#    yizhang     02/18/08 - Fix bug 6822550
#    skakarla    01/18/08 - adding upgrade specific changes
#    srisanka    01/09/08 - separate generic and OSD code
#    jleys       01/22/08 - Tolerate failure to star in exclusive mode
#    jachang     01/13/08 - Modify the order of clscfg in script
#    jleys       01/07/08 - Remove cssvfupgd
#    yizhang     01/08/08 - Add scan listener as oracle owner
#    khsingh     01/08/08 - Add asm resource
#    yizhang     01/02/08 - Start scan listener after adding scan listener
#    ysharoni    12/27/07 - Static CSS_LEASEDURATION and ASM_SPFILE to paramf
#    jleys       12/21/07 - Workaround the lack of VF discovery string
#    yizhang     12/10/07 - Add scan and scan_listener
#    minzhu      12/19/07 - update css mode
#    ysharoni    12/14/07 - gpnp cont-d
#    jachang     12/07/07 - Add exclusive mode processing
#    samjo       12/07/07 - Making OCR and OLR names consistent
#    srisanka    11/30/07 - bug 6661111: fix CRS_NODEVIPS parsing
#    jachang     11/29/07 - Adding exclusive mode steps
#    ysharoni    11/28/07 - Add gpnp
#    srisanka    11/19/07 - add use Net::Ping
#    srisanka    08/23/07 - Creation
# 

################ Documentation ################

# The SYNOPSIS section is printed out as usage when incorrect parameters
# are passed

=head1 NAME

  rootcrs.pl - Configure Oracle Clusterware

=head1 SYNOPSIS

  rootcrs.pl [-verbose] [-upgrade [-force] | -patch]
             [-paramfile <parameter-file>] 
             [-deconfig [-deinstall] [-keepdg] | -downgrade] [-force] [-lastnode]
             [-downgrade] [-oldcrshome <old crshome path>] [-version <old crs version>]  
             [-unlock [-crshome <path to crs home>] [-nocrsstop]]

  Options:
   -verbose    Run this script in verbose mode
   -upgrade    Oracle HA is being upgraded from previous version
   -patch      Oracle HA is being upgraded to a patch version
   -paramfile  Complete path of file specifying HA parameter values
   -lastnode   Force the node this is executing on to be considered the
               last node of the install and perform actions associated
               with configuring the last node
   -downgrade  Downgrade the clusterware
   -version    For use with downgrade; special handling is required if
               downgrading to 9i. This is the old crs version in the format
               A.B.C.D.E (e.g 11.1.0.6.0).
   -deconfig   Remove Oracle Clusterware to allow it to be uninstalled or reinstalled.
   -force      Force the execution of steps in delete that cannot be verified 
	       to be safe
   -deinstall  Reset the permissions on CRS home during de-configuration
   -keepdg     Keep existing diskgroups during de-configuration
   -unlock     Unlock CRS home 
   -crshome    Complete path of crs home. Use with unlock option.
   -oldcrshome For use with downgrade. Complete path of the old crs home.
   -nocrsstop  used with unlock option to reset permissions on an inactive grid home


  If neither -upgrade nor -patch is supplied, a new install is performed

  To see the full manpage for this program, execute:
    perldoc rootcrs.pl

=head1 DESCRIPTION

  This script performs the operations necessary to install the Oracle
  Clusterware stack on a node of a cluster.  It must be run once on
  each node and it must be run by an authorized user, e.g. root on UNIX
  platforms, Administrator on Windows.

  Upon successful completion on each node, the Oracle Clusterware will
  be executing on that node.

=cut

################ End Documentation ################

use strict;
use English;
use File::Basename;
use File::Spec::Functions;
use Term::ANSIColor;

BEGIN {
  # Add the directory of this file to the search path
  push @INC, dirname($PROGRAM_NAME);
}

use Net::Ping;
use Getopt::Long qw(:config no_auto_abbrev);
use Pod::Usage;
use crsconfig_lib;
require crsdelete;
use crspatch;

use oracss;
use oraacfs;

my $OLD_CRS_HOME;

# Global variables
our $g_force = 0;
our $g_delete = 0;
our $g_lastnode = 0;
our $g_downgrade = 0;
our $g_unlock = 0;
our $g_version;
our $g_help = 0;
our $g_patch = 0;
our $g_noterm = 0;
our $g_deinstall = 0;
our $g_keepdg = 0;

our $DEBUG;

# pull all parameters defined in crsconfig_params and s_crsconfig_defs (if
# it exists) as variables in Perl
my $paramfile_default = catfile (dirname ($0), "crsconfig_params");

# pull all parameters defined in crsconfig_addparams
my $addparams = catfile (dirname ($0), "crsconfig_addparams");

# pull all definitions in s_crsconfig_defs (if it exists) as variables in Perl
# this file might not exist for all platforms
my $defsfile = catfile (dirname ($0), "s_crsconfig_defs");

my $UPGRADE;
my $firstNode;
our $PARAM_FILE_PATH = $paramfile_default;
my $unlock_crshome;
my $DOWNGRADE;
my $oldcrshome;
my $oldcrsver;
my $REMOTENODE;
my @oldcrs_ver;
my $ckptcrsver;
my $crsrelver;
my $destcrshome;
my $SUCC_REBOOT = 0;


# Parse command line args
# If an incorrect option is specified, the perl POD at the beginning
# of the file is parsed into a man page
# the return code to give when the incorrect parameters are passed
# noterm option is to be used internally by the installer in Win environment.
my $usage_rc = 1;

GetOptions('verbose!'    => \$DEBUG,
           'upgrade!'    => \$UPGRADE,
           'patch!'      => \$g_patch,
           'paramfile=s' => \$PARAM_FILE_PATH,
           'deconfig'    => \$g_delete,
           'force'       => \$g_force,
           'deinstall'   => \$g_deinstall,
           'keepdg'      => \$g_keepdg,
           'lastnode'    => \$g_lastnode,
           'downgrade'   => \$DOWNGRADE,
           'version=s'   => \$oldcrsver,
           'unlock'      => \$g_unlock,
           'crshome=s'   => \$unlock_crshome,
           'destcrshome=s'   => \$destcrshome,
           'oldcrshome=s'=> \$oldcrshome,
           'remotenode'  => \$REMOTENODE,
           'nocrsstop!'   => \$NOCRSSTOP,
           'help!'       => \$g_help,
	   'noterm!'	 => \$g_noterm) or pod2usage($usage_rc);

# Check validity of command line args
pod2usage(-msg => "Invalid extra options passed: @ARGV",
          -exitval => $usage_rc) if (@ARGV);

if ($g_help)   { pod2usage(0); }

#
# MAIN SCRIPT BODY
#
local $SIG{'__DIE__'} = sub { dietrap(@_); };
# Catch Control-C
local $SIG{INT} = sub { dietrap(@_); };
# Catch termination signal (kill)
# Kill signal (kill -9) can be neither trapped nor ignored
local $SIG{TERM} = sub { dietrap(@_); };

if (!$DEBUG) { $DEBUG = $ENV{'ORA_INSTALL_DEBUG'}; }

### Set this host name (lower case and no domain name)
our $HOST = tolower_host();

# Set the following vars appropriately for cluster env
### check if run as super user
our $SUPERUSER = check_SuperUser ();
if (!$SUPERUSER) {
  error("Insufficient privileges to execute this script.");
  error("root or administrative privileges needed to run the script.");
  exit 2;
}


# Read the config files and set up the configuration data for
# subsequent processing
my $cfg =
  crsconfig_lib->new(IS_SIHA             => FALSE,
                     paramfile           => $PARAM_FILE_PATH,
                     osdfile             => $defsfile,
                     addfile             => $addparams,
                     crscfg_trace        => TRUE,
                     CRSDelete           => $g_delete,
                     DEBUG               => $DEBUG,
                     HAS_USER            => $SUPERUSER,
                     HOST                => $HOST,
                     UPGRADE             => $UPGRADE,
                     UNLOCK		 => $g_unlock,
                     unlock_crshome      => $unlock_crshome,
                     CRSPatch            => $g_patch,
                     DOWNGRADE           => $DOWNGRADE,
                     oldcrshome          => $oldcrshome,
                     oldcrsver           => $oldcrsver,
                     force               => $g_force,
                     deinstall           => $g_deinstall,
                     keepdg              => $g_keepdg,
                     lastnode            => $g_lastnode,
                     REMOTENODE          => $REMOTENODE,
                     destcrshome         => $destcrshome
                     );

if($g_noterm)
{
# redirect stdout/stderr as appropriate for Windows only
s_redirect_souterr ($cfg->crscfg_trace_file . "_OUT");
}

my $ORACLE_HOME  = $cfg->params('ORACLE_HOME');
my $ORACLE_BASE  = $cfg->params('ORACLE_BASE');
my $ORA_HA_HOME  = $ORACLE_HOME;
my $ORA_CRS_HOME = $ORACLE_HOME;
my $asmgrp       = $CFG->params('ORA_ASM_GROUP');

$ENV{'ORACLE_HOME'}  = $ORACLE_HOME;
$ENV{'ORACLE_BASE'}  = $ORACLE_BASE;

# set some stuff we were not able to set until we had read params
my $CLSCFGBIN = catfile ($ORACLE_HOME, "bin", "clscfg");

if ($g_unlock && $destcrshome) {unlockPatchHome($destcrshome);}
elsif ($g_patch && $destcrshome)  {CRSPatchhome($destcrshome);}
elsif ($g_delete) { CRSDelete(); }
elsif ($g_unlock) { unlockCRSHome(); }
elsif ($g_patch)  { CRSPatch(); }
elsif ($DOWNGRADE){ CRSDelete(); }
else  
{

  if (isPrereqIgnored())
  {
   error("User ignored Prerequisites during installation");
   }
  # run directory creation, script instantiation, files creation/permissions
  # modules
  run_env_setup_modules ();

  rscPreChecks(); 

  #hack to get crs ver from sqlplus. use getcrsrelver once bug
  #11077430 is fixed.
  $crsrelver = getcrsrelver1();

  if (!isCkptexist("ROOTCRS_STACK"))
  {
    trace("Oracle clusterware configuration has started");
    writeCkpt("ROOTCRS_STACK", CKPTSTART);
    writeCkptProperty("ROOTCRS_STACK", "VERSION", $crsrelver);
    $wipCkptName = "ROOTCRS_STACK";
  }
  else
  {
    if (!isCkptPropertyExists("ROOTCRS_STACK", "VERSION"))
    {
       trace("Removing older checkpoints");
       remove_checkpoints();
       trace("Oracle clusterware configuration has started");
       writeCkpt("ROOTCRS_STACK", CKPTSTART);
       writeCkptProperty("ROOTCRS_STACK", "VERSION", $crsrelver);
       $wipCkptName = "ROOTCRS_STACK";
    }
    else {
       my $ckptcrsver = getCkptPropertyValue("ROOTCRS_STACK", "VERSION");
       if (!isVersionMatch($ckptcrsver, $crsrelver))
       {
          trace("Removing older checkpoints");
          remove_checkpoints();
          trace("Oracle clusterware configuration has started");
          writeCkpt("ROOTCRS_STACK", CKPTSTART);
          writeCkptProperty("ROOTCRS_STACK", "VERSION", $crsrelver);
          $wipCkptName = "ROOTCRS_STACK";
       }
       else
       {
         $isRerun = TRUE;
       }

       my $ckptStatus = getCkptStatus("ROOTCRS_STACK");

       if ($ckptStatus eq CKPTSTART)
       {
         ## It likely that the node crashed. Hence this ckpt is not labeled FAILED
         $isNodeCrashed = TRUE;
       }

       $wipCkptName = "ROOTCRS_STACK";
   }
  }

  my @nodes_to_del;

  if ($UPGRADE)
  {
    trace("Querying previous clusterware configuration");
    get_oldconfig_info();

    if (($CFG->force) && (FAILED == preForceUpgradeChecks(\@nodes_to_del)))
    {
      print color 'bold';
      print "The force upgrade pre-checks failed. Aborting the force upgrade\n";
      print color 'reset';
      exit(1);
    }

    if (!preUpgradeChecks())
    {
      print color 'bold';
      print "The pre-upgrade checks failed, aborting the upgrade\n";
      print color 'reset';
      exit 1;
    }
  }

  my $success = TRUE;

  # validate filesystem
  if (! isFilesystemSupported()) 
  {
     writeCkpt("ROOTCRS_STACK", CKPTFAIL);
     exit 1;
  }

  # validate RAC_ON/RAC_OFF
  if (! is_dev_env () && ! isRAC_appropriate()) 
  {
     writeCkpt("ROOTCRS_STACK", CKPTFAIL);
     exit 1;
  }

  if (! $UPGRADE &&
     (! isAddNode($HOST, $cfg->params('NODE_NAME_LIST'))) &&
     (! isInterfaceValid())) 
  {
     writeCkpt("ROOTCRS_STACK", CKPTFAIL);
     exit 1;
  }

  my $ckptstatus = getCkptStatus("ROOTCRS_STACK");

  trace("Saving the configuration parameter file data");

  trace("checkpoint status of ROOTCRS_STACK is $ckptstatus");
  if (!isCkptexist("ROOTCRS_PARAM") && ($ckptstatus ne CKPTSUC))
  {
    trace("inside rootcrs_param");
    writeCkpt("ROOTCRS_PARAM", CKPTSTART);
    writeCkptPropertyFile("ROOTCRS_PARAM", $PARAM_FILE_PATH);
    writeCkpt("ROOTCRS_PARAM", CKPTSUC);
  }

  if (! $UPGRADE) { 
     if (!isCkptexist("ROOTCRS_OSDSETUP")) {
        #This checkpoint is mainly required on windows
        trace("Writing a checkpoint for platform specific OSD setup");
        writeCkpt("ROOTCRS_OSDSETUP", CKPTSTART);
        $wipCkptName = "ROOTCRS_OSDSETUP";
     }
  
     if (!isCkptSuccess("ROOTCRS_OSDSETUP")) {
        $wipCkptName = "ROOTCRS_OSDSETUP";

        # perform platform-specific setup actions
        # only for fresh install
        if (SUCCESS != s_osd_setup ()) {
           error ("Platform-specific setup failed");
           writeCkpt("ROOTCRS_OSDSETUP", CKPTFAIL); 
           exit 1;
        }
        else {
           trace("Platform specific setup actions are done");
           writeCkpt("ROOTCRS_OSDSETUP", CKPTSUC);
           $wipCkptName = "ROOTCRS_STACK";
        }
     }
  }

  # perform platform-specific setup actions
  if (!isCkptexist("ROOTCRS_ONETIME"))
  {
    trace("Writing checkpoint for first node actions");
    writeCkpt("ROOTCRS_ONETIME", CKPTSTART);
    $wipCkptName = "ROOTCRS_ONETIME";
  }
  
  if (!isCkptSuccess("ROOTCRS_ONETIME"))
  {
    $wipCkptName = "ROOTCRS_ONETIME";

    # If $firstNode is set, perform cluster-wide one-time actions and exit
    if (SUCCESS != first_node_tasks ()) {
       error ("Cluster-wide one-time actions failed");
       writeCkpt("ROOTCRS_ONETIME", CKPTFAIL); 
       exit 1;
    }
    else {
       trace ("Cluster-wide one-time actions... Done!");
       writeCkpt("ROOTCRS_ONETIME", CKPTSUC); 
       $wipCkptName = "ROOTCRS_STACK";
    }
  }

  # on NT, set_perms_ocr_vdisk() function cannot be executed until 
  # s_osd_setup() and first_node_tasks() functions are finhish
  set_perms_ocr_vdisk();

  ## FIXME: This should not be required
  $ENV{'ORA_PCW_AGENTS'} = "true";

  ## Step3: Check if CRS is already configured

  # check_CRSConfig outers
  my $crsConfigured   = FALSE;
  my $gpnp_setup_type = GPNP_SETUP_BAD ;

  check_CRSConfig ($ORA_CRS_HOME, $cfg->HOST,
                   $cfg->params('ORACLE_OWNER'),
                   $cfg->params('ORA_DBA_GROUP'),
                   $cfg->params('GPNPGCONFIGDIR'),
                   $cfg->params('GPNPCONFIGDIR'),
                   $crsConfigured, $gpnp_setup_type)
    or die "Clusterware configuration check failed";

  # Verify bdb location. Initially its in CRS HOME. oclumon changes later
  my $bdbloc;
  $bdbloc = catfile("$ORA_CRS_HOME", "crf", "db", "$HOST");
  if (isCRFSupported() && !is_dev_env())
  {
    # no verification in ade, just create dirs
    trace("CHM/OS repository directory $bdbloc checks");
    s_crf_check_bdbloc($bdbloc, $cfg->params('NODE_NAME_LIST'));
  }

  if ($UPGRADE) 
  {
    trace("Querying previous clusterware configuration");
    queryClusterConfig();
  }

  if (!isCkptSuccess("ROOTCRS_STACK"))
  {
     #set $crscConfigured to false for checkpoint logic
     $crsConfigured = FALSE;
  }

  # In case of upgrade crs stack is always configured. But to upgrade to
  # 11.2 we still need to do same things as we do for fresh install.
  # TODO: going forward when we upgrade from 11.2 to higher versions
  # this step may not be required.

  if (!$crsConfigured) 
  {  
    ## Step5: Check existing configuration, Create and populate OLR and OCR keys
    perform_olr_config() or die("OLR configuration failed\n");
   
    if ((!$UPGRADE) || ($UPGRADE && !isVersion112()))
    {  
       # Initialize the SCR settings.
       s_init_scr ();
    
       # incase of upgrade  voting files would be upgraded from previous
       # version 
       ## create GPnP wallet/profile 
    
       initialize_local_gpnp($HOST, $gpnp_setup_type);
    }

      # Create CHM/OS config
      my $crfconfig;
      my $tmpcrfconfig;
      $crfconfig = catfile ("$ORA_CRS_HOME", "crf", "admin", "crf${HOST}.ora"); 

      trace ("Creating CHM/OS config file $crfconfig");
      $tmpcrfconfig = crf_config_generate($HOST, $bdbloc, $cfg->params('ORACLE_OWNER'), $cfg->params('NODE_NAME_LIST'));
      
      # Delete older existing CHM/OS repository for fresh start.
      # If not removed, leads to CHM/OS repository corruption.

      trace ("Deleting older CHM/OS repository at $bdbloc");
      crf_delete_bdb($bdbloc);
      copy_file($tmpcrfconfig, $crfconfig);
      unlink $tmpcrfconfig;

      # delete older CHM/OS installation
      crf_do_delete();

    if (!$crsConfigured || (($UPGRADE) && (!isVersion112())))
    {
       ## Step8: Setup OHASD service/daemon
       trace ("Registering ohasd");
       perform_register_service("ohasd") or die "Failed to register Oracle ohasd service";
    }
    
    # Start OHASD service/daemon
    trace ("Starting ohasd");
    perform_start_service("ohasd") or die "Failed to start Oracle ohasd service";

    ###FIXME: Change once we know types of resources to be created
    trace ("Creating CRS resources and dependencies");
    my $hasdconfig_superuser = $SUPERUSER;
    if ($OSNAME eq 'MSWin32' && is_dev_env ()) 
    {
       $hasdconfig_superuser = $cfg->params('ORACLE_OWNER');
    }

    trace("Configuring HASD");
    my $status;
    if ($cfg->platform_family eq "windows") {
       my $NT_AUTHORITY = '';
       if (is_dev_env()) {
          $NT_AUTHORITY = $cfg->params('ORACLE_OWNER');
       }

       $status = perform_configure_hasd('crs', $HOST, $NT_AUTHORITY,
                                $NT_AUTHORITY, 
                                $cfg->params('ORA_DBA_GROUP'));
    }
    else {
         $status = perform_configure_hasd('crs', $HOST,
                               $cfg->params('ORACLE_OWNER'),
                               $hasdconfig_superuser,
                               $cfg->params('ORA_DBA_GROUP'));
    }

    if ($status) 
    {
      trace ("Successfully created CRS resources for cluster and ASM");
    } else {
      error ("Failed to create CRS resources for cluster and ASM");
    }

    if (!isCkptexist("ROOTCRS_ACFSINST"))
    {
       trace("Writing checkpoint for USM driver install");
       writeCkpt("ROOTCRS_ACFSINST", CKPTSTART);
       $wipCkptName = "ROOTCRS_ACFSINST";
    }
    
    if (!isCkptSuccess("ROOTCRS_ACFSINST"))
    {
       $wipCkptName = "ROOTCRS_ACFSINST";
    
       # install USM driver 
       my $ret = installUSMDriver();
       if (FAILED == $ret) {
          error ("USM driver install actions failed");
          writeCkpt("ROOTCRS_ACFSINST", CKPTFAIL);
          exit 1;
       }
       else {
          trace ("USM drivers installation completed");
          writeCkpt("ROOTCRS_ACFSINST", CKPTSUC);
          $wipCkptName = "ROOTCRS_STACK";
          if (WARNING == $ret)
          {
            trace("Set reboot flag to 1");
            $SUCC_REBOOT = 1;
          }
       }
    }
 

    # perform CSS config or upgrade, propagate cluster-wide config if needed
    if ($UPGRADE) 
    {
      if (! perform_upgrade_config($gpnp_setup_type)) 
      {
         my $trace_file = $CFG->crscfg_trace_file;
         error ("Cluster configuration upgrade failed.  " . 
		"collect the  $trace_file and contact Oracle Support");
         exit 1;
      }
    } 
    elsif (! perform_init_config()) 
    {
      my $trace_file = $CFG->crscfg_trace_file;
      error ("Initial cluster configuration failed.  " . 
	     "See $trace_file for details");
      exit 1;
    }

    ## Step9: Start Oracle Clusterware stack
    perform_start_cluster();
  }

  if ($UPGRADE) {
    # execute clscfg -upgrade
    my $status;

    if ($CFG->force) {
       $status = run_crs_cmd('clscfg', '-upgrade', '-g', $asmgrp, '-lastnode');
    } else {
       $status = run_crs_cmd('clscfg', '-upgrade', '-g', $asmgrp);
    }

    if ($status == 0) {
       trace ("$CLSCFGBIN -upgrade completed successfully");
    } else {
       error ("$CLSCFGBIN -upgrade failed");
       exit 1;
    }

     # execute 'oifcfg setif -global'
     if (isLastNodeToUpgrade ($HOST, $CFG->params('NODE_NAME_LIST'))) {
        $status = run_crs_cmd('oifcfg', 'setif', '-global');

        # modify ACTION_SCRIPT
        ModActionScript();
     }

  }
  elsif (isAddNode($HOST, $cfg->params('NODE_NAME_LIST'))) 
  {
    if (!isCkptexist("ROOTCRS_ADDNODE"))
    {
      trace("Writing checkpoint for add node actions");
      writeCkpt("ROOTCRS_ADDNODE", CKPTSTART);
      $wipCkptName = "ROOTCRS_ADDNODE";
    }
    
    if (!isCkptSuccess("ROOTCRS_ADDNODE"))
    {
      $wipCkptName = "ROOTCRS_ADDNODE";
      writeCkpt("ROOTCRS_ADDNODE", CKPTSTART);
   
      my $status = run_crs_cmd('clscfg', '-add');
      if ($status == 0) 
      {
         writeCkpt("ROOTCRS_ADDNODE", CKPTSUC);
         trace ("$CLSCFGBIN -add completed successfully");
      } else {
         writeCkpt("ROOTCRS_ADDNODE", CKPTFAIL);
         error ("$CLSCFGBIN -add failed");
         exit 1;
      }
    }
  }

  # configure node
  $success = perform_configNode();

  if ($UPGRADE && (! isVersion112())) {
     # set permissions on voting disks
     if (! setperm_vdisks()) {
        trace ("Failed to set permission on voting disks during upgrade");
     }
  }

  #Workaround fix for bug 8309620 and 9954172 and 10056987. 
  #The start of asm resource pulled up acfs resource due to
  #startup dependency , but didn't get stopped as part of asm stop.
  if (isFirstNodeToStart() && ! $UPGRADE && 
      ! isAddNode($HOST, $CFG->params('NODE_NAME_LIST')) &&
      ! $CFG->ASM_STORAGE_USED)
  {
     start_asm();
     stop_asm();
     stop_resource("ora.registry.acfs", "-f");
  }

  backupOLR();

  configureCvuRpm();

  # This code is getting in 11203, so we only check old version < 11203 here.
  if ($UPGRADE && isVersionLT11203($CFG->oldconfig('ORA_CRS_VERSION')))
  {
    trace("Cleaning up ASM files ...");
    s_cleanASMFiles(); 
  }

  if ($success) {
     print color 'bold';
     print  "Configure Oracle Grid Infrastructure for a Cluster ... succeeded\n";
     if (($CFG->force) && scalar(@nodes_to_del) > 0)
     {
       my $nodelist = join(',', @nodes_to_del);
       print  "The following nodes:\n";
       print  "    $nodelist\n";
       print  "are not upgraded, and must be deleted from the cluster after the force upgrade\n";
     }
     
     if (1 == $SUCC_REBOOT)
     {
       print "A system reboot is recommended before using ACFS\n";
     }
     print color 'reset';
     trace ("Configure Oracle Grid Infrastructure for a Cluster ... succeeded");
     writeCkpt("ROOTCRS_STACK", CKPTSUC);
  } 
  else {
     print color 'bold';
     print  "Configure Oracle Grid Infrastructure for a Cluster ... failed\n";
     print color 'reset';
     trace ("Configure Oracle Grid Infrastructure for a Cluster ... failed");
     exit 100;
  }


  if ($success && ($cfg->platform_family eq "windows")) {
    if (!isCkptexist("ROOTCRS_REMOTENODES"))
    {
      trace("Writing checkpoint for remotenode actions");
      writeCkpt("ROOTCRS_REMOTENODES", CKPTSTART);
      $wipCkptName = "ROOTCRS_REMOTENODES";
    }
    
    if (!isCkptSuccess("ROOTCRS_REMOTENODES"))
    {
      $wipCkptName = "ROOTCRS_REMOTENODES";
      writeCkpt("ROOTCRS_REMOTENODES", CKPTSTART);
   
      # for windows only, configure all remote nodes
      $success = configureAllRemoteNodes();

      if ($success)
      {
         writeCkpt("ROOTCRS_REMOTENODES", CKPTSUC);
         trace ("Configure Oracle Grid Infrastructure on remote nodes... succeeded");
      }
      else
      {
         writeCkpt("ROOTCRS_REMOTENODES", CKPTFAIL);
         trace ("Configure Oracle Grid Infrastructure on remote nodes... failed");
         exit 100;
      }
    }
  }
}

if($g_noterm)
{
# restore stdout/stderr as appropriate
s_restore_souterr ();
}

0;