#
# $Header: has/install/crsconfig/oraocr.pm /st_has_12.2.0.1.0ocwpsu/1 2017/08/06 23:45:57 luoli Exp $
#
# oraocr.pm
#
# Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved.
#
#    NAME
#      oraocr.pm - <one-line expansion of the name>
#
#    DESCRIPTION
#      An OCR component to manage OCR related operations during install, 
#      upgrade, downgrade and deconfiguration.
#
#    NOTES
#      Required variables from the config objecj $CFG:
#      $CFG->params('OCR_LOCATIONS')
#      $CFG->ASM_STORAGE_USED
#      $CFG->params('CDATA_DISK_GROUP')
#      $CFG->ORA_CRS_HOME
#      $CFG->params('CLUSTER_NAME')
#      $CFG->params('ORACLE_HOME')
#
#    EXPORT METHODS
#      backupOCR - Backup OCR
#      checkOCR - Checks if the OCR is configured properly and not corrupted
#      ValidateOCR - Validating OCR locations based on existing ocr settings
#      validate_ocrconfig - Creates OCR config file if it does not exists.
#      downgradeOCR  - Downgrade OCR
#      localonlyOCR_exists - Verifies if the local-only OCR exists
#      createLocalOnlyOCR - Create local-only OCR
#      get_ocrdisk - Retrieves OCR location 1
#      get_ocrmirrordisk - Retrieves OCR location 2
#      get_ocrloc3disk - Retrieves OCR location 3
#      get_ocrloc4disk - Retrieves OCR location 4
#      get_ocrloc5disk - Retrieves OCR location 5
#      get_ocrlocaldisk - Retrieves Local-only OCR location
#      discoverOCRDiskgroups - Get all diskgroups that store the OCR
#      getOcrBackupOMF - Get OMF names of a given OCR backup file stored on DG.
#      removeOcrBackupFromDG - Remove OCR backup file from DG given its OMF name
#      backupOcr4Restore - Generate an OCR backup file and store it in OCR
#      removeOCRBeforeRestore - Remove OCR file on ASM before restoring the OCR
#                               of the old version
#      removeOCROnDG - Remove the OCR file and OCR mirrors in the diskgroups
#      get_softVersionfromOCR - Get the software version for the given node
#      writeOcrKeyPair - Write a key-value  pair in OCR.
#      removeOcrKeyPair - Remove a key-value pair from OCR.
#      createAndSetOcrKeyPair - Create and set a key-value pair in OCR
#      getOcrKeyValue - Obtain the value of a specific OCR key 
#
#    Depricated/Unused Methods:
#      get_ocrid - Returns OCRID
#
#    MODIFIED   (MM/DD/YY)
#    luoli      08/02/17 - Backport luoli_bug-25448476 from main
#    luoli      02/03/17 - Fix bug 25448476
#    xyuan      05/27/16 - Fix bug 23483199
#    xyuan      05/09/16 - Fix bug 22382998
#    muhe       03/21/16 - Modify sub restoreOcrFromBackup
#    luoli      02/23/16 - Fix a minor argc issue on message 183
#    luoli      02/17/16 - Fix bug 22700293
#    muhe       12/23/15 - Fix bug 22459788
#    xyuan      12/15/15 - Fix bug 22283181
#    luoli      12/15/15 - Fix bug 21921412
#    sbezawad   11/17/15 - Bug 21919798: Add cluster properties to clscfg
#    luoli      10/26/15 - Root script fix for bug 20775137
#    muhe       10/20/15 - Print the output when 'ocrconfig -manualbackup'
#                          fails during upgrade
#    bbeelamk   10/07/15 - Fix bug 21878673
#    muhe       09/09/15 - Check the ckpt first before copying OCR backup file
#                          to shared storage to avoid multiple copies
#    luoli      07/28/15 - Add -y -z -h to CLSCFG command
#    luoli      07/01/15 - Fix bug 21324323
#    jmarcias   06/25/15 - Fix bug 21137634
#    luoli      06/14/15 - Fix bug 21252763
#    muhe       05/13/15 - Modify sub configureOcrBackup
#    xyuan      05/04/15 - Fix bug 21023562
#    bbeelamk   04/27/15 - Fix bug 20950908
#    luoli      04/23/15 - Fix bug 20947747
#    muhe       04/14/15 - Add interface functions body for upgrade and patch
#    jmarcias   04/10/15 - Fix bug 20853464
#    sbezawad   04/10/15 - Bug 20863158: Fix SIHA upgrade
#    muhe       03/27/15 - Fix bug 20725601
#    muhe       03/20/15 - Fix bug 20717005
#    jmarcias   03/11/15 - Fix bug 20569941
#    luoli      02/26/15 - Fix bug 19232406
#    luoli      12/22/14 - rsc modeling for downgrade/deconfig
#    sbezawad   11/26/14 - Bug 20019354: Migrate OCR and OLR to new framework
#    xyuan      09/17/14 - Incorporate review comments
#    luoli      07/22/14 - Creation
#

package oraClusterwareComp::oraocr;

use parent 'oraClusterwareComp';

use strict;
use English;
use Carp;
use File::Copy;
use File::Path;
use File::Find;
use File::Basename;
use File::Spec::Functions;

use crsutils;
use s_crsutils;
use crsgpnp;
use s_oraocr;

# Steps to configure OCR on the first node
use constant INITIALIZE_OCRLOC_FILE     => 'ocr_ConfigFirstNode_step_1';
use constant CONFIGURE_OCR              => 'ocr_configFirstNode_step_2';
use constant DISTRIBUTE_ORCLOC_FILE     => 'ocr_configFirstNode_step_3';

# Steps to downgrade OCR on the first node
use constant GET_OCR_BACKUP_FILE        => 'ocr_downgradeLastNode_step_1';
use constant DOWNGRADE_OCR              => 'ocr_downgradeLastNode_step_2';
use constant RESTORE_OCR_LOC            => 'ocr_downgradeNonLastNode_step_1';

sub new
{
  my $class = shift;
  # Pass the component name into the constructor 
  my $componentName = @_;

  my $self = $class->SUPER::new(@_);
  $self->_initialize();

  return $self;
}

# Initialize OCR
sub _initialize
{
  my $self = shift;

  my $compName = $self->compName;
  trace("Perform initialization tasks before configuring $compName");
}

#
# Interface to be implemented
#
# Is OCR supported based on platform and user input
sub isSupported
{
  # Supported on all platforms
  return TRUE;
}

# Which component does OCR depend on
sub dependsOn
{
  my @dependency;
  # TODO
  # List all components which OCR depond on

  push @dependency, "ASM";
  return @dependency;
}

# Has the component already been configured
sub isConfigured
{
  #TODO
  return FALSE;
}

#
# Methods for install w.r.t OCR
#

# Configure action on first node
sub configureFirstNode
{
  my $self = shift;
  my $stepIndicator = shift;
  my $compName = $self->compName;

  trace("Executing the step [$stepIndicator] to configure $compName ".
        "on the first node");
  if (INITIALIZE_OCRLOC_FILE eq $stepIndicator)
  {
    # configure ocr.loc
    # Initialize OCR location using DG name
    my $OCR = $CFG->params('OCR_LOCATIONS');
    if ($CFG->ASM_STORAGE_USED)
    {
      trace("Use DG name as the OCR location");
      $OCR = "+" . $CFG->params('CDATA_DISK_GROUP');
    }

    trace("OCR locations: $OCR");
    $self->validateOCR($CFG->ORA_CRS_HOME,
                       $CFG->params('CLUSTER_NAME'),
                       $OCR) || die(dieformat(293, $OCR));

    return SUCCESS;
  }
  elsif (CONFIGURE_OCR eq $stepIndicator)
  {
    # configure OCR
    configureOCR() || die(dieformat(259));

    return SUCCESS;
  }
  elsif (DISTRIBUTE_ORCLOC_FILE eq $stepIndicator)
  {
    # push OCR location to sync file for non-first nodes
    if ($CFG->ASM_STORAGE_USED)
    {
      trace("Sync the OCR locations list to other nodes");
      pushOCRLocSyncFile(createOCRLocationList($CFG->params('ORACLE_HOME')));
    }

    return SUCCESS;
  }
  else
  {
    croak "Step indicator out of bounds";
  }
}

# Configure action on every node other than first node 
# w.r.t OCR
sub configureNonFirstNode
{
  my $self = shift;
  my $compName = $self->compName;

  my $OCR = $CFG->params('OCR_LOCATIONS');
  if (isAddNode($CFG->HOST))
  {
    trace("For addNode scenarios, the OCR location file " .
          "will get pulled from other nodes");
  }
  else
  {
    if ($CFG->ASM_STORAGE_USED)
    {
      trace("Retrieve OCR locations from the sync file");
      my $syncFile = getOCRLocSyncFile();
      if (! -e $syncFile)
      {
        my $localNode = tolower_host();
        my $installNode = getInstallNode();
        die(dieformat(507, $localNode, $installNode));
      }

      my @ocr_locations = read_file(getOCRLocSyncFile());
      $OCR = $ocr_locations[0];
      chomp($OCR);
    }
  }

  trace("OCR locations: $OCR");
  $self->validateOCR($CFG->ORA_CRS_HOME,
                     $CFG->params('CLUSTER_NAME'),
                     $OCR) || die(dieformat(293, $OCR));

  return SUCCESS;
}

# Configure action for SIHA
sub configureSIHA
{
  configureOCR_siha() || die(dieformat(259));
  return SUCCESS;
}

# Configure action on first node after the configured stack
# has been started
sub postConfigFirstNode
{
  if (isOCRonASM() && !configureOcrBackup())
  {
    return FAILED;
  }

  return SUCCESS; 
}

# Configure action on other nodes than the first node after
# the configured stack has been started
sub postConfigNonFirstNode
{
  return SUCCESS;
}

#
# Methods for patch
#

# The method for pre checks related to OCR component for patching 
sub prePatchCheck
{
  return SUCCESS;
}

# 
# Methods for upgrade w.r.t OCR
#

# The method for global checks related to OCR component before upgrade
# This method is called only on the first node when the stack is up.
sub preUpgradeCheck
{
  return SUCCESS;
}

# The method for local checks related to OCR component before upgrading the first node
sub upgradeCheckFirstNode
{
  return SUCCESS;
}

# The method for local checks related to OCR component before upgrading the middle node
sub upgradeCheckMiddleNode
{
  return SUCCESS;
}

# The method for local checks related to OCR component before upgrading the last node
sub upgradeCheckLastNode
{
  return SUCCESS;
}

# Upgrade action on first node
sub upgradeFirstNode
{
  # Upgrade OCR;  # Run as root "ocrconfig -upgrade" ;
  if(! upgradeOCR())
  {
    die(dieformat(248));
  }

  return SUCCESS;
}

# Upgrade action on every node other than first and last node
sub upgradeMiddleNode
{
  # NOOP
  return SUCCESS;
}

# Upgrade action on last node
sub upgradeLastNode
{
  # NOOP
  return SUCCESS;
}

# Upgrade action on the first node after the higher version stack
# has been started
sub postUpgradeFirstNode
{
  if (isOCRonASM() && !configureOcrBackup4upg())
  {
    return FAILED;
  }

  return SUCCESS;
}

# Upgrade action on every node other than first and last node
# after the higher version stack has been started 
sub postUpgradeMiddleNode
{
  return SUCCESS;
}

# Upgrade action on the last node after the higher version stack
# has been started
sub postUpgradeLastNode
{
  my $self = shift;

  $self->backupOCR();
  return SUCCESS;
}

#
# Methods for downgrade
#

# downgrade actions on nodes other than the last node
sub downgradeNonLastNode
{
  my $self = shift;
  my $stepIndicator = shift;
  my $compName = $self->compName;

  trace("Executing the step [$stepIndicator] to downgrade $compName ".
        "on nodes other than the last node.");

  if (RESTORE_OCR_LOC eq $stepIndicator)
  {
    s_restoreocrloc();
  }
  else
  {
    croak "Step indicator out of bounds";
  }

  return SUCCESS;
}

# downgrade actions on the last node
sub downgradeLastNode
{
  my $self = shift;
  my $stepIndicator = shift;
  my $compName = $self->compName;

  trace("Executing the step [$stepIndicator] to downgrade $compName ".
        "on the last node.");

  if (GET_OCR_BACKUP_FILE eq $stepIndicator)
  {
    # Get the OMF file name of the OCR backup file on DG.
    my @output = getOCRLoc();
    my $ocrType = shift @output;
    if ($ocrType == 2)
    {
      trace("Start to get the OMF name of the OCR backup file on DG.");
      my @oldCrsVer = @{$CFG->oldconfig('ORA_CRS_VERSION')};
      my $crsversion = join('.', @oldCrsVer);
      my $clustername = $CFG->params('CLUSTER_NAME');
      my $ocrBackupFileName = "$clustername"."_backup"."$crsversion".".ocr";
      $ocrBackupFileName = lc($ocrBackupFileName);

      my ($rc_OMF, @output_OMF) = 
        $self->getOcrBackupOMF($ocrBackupFileName, s_get_olr_file("crs_home"));
      my $count = scalar(grep(/$ocrBackupFileName/, @output_OMF));
      if (($rc_OMF == 0) && ($count == 1))
      {
        my $ocrBackupOMFName = join("", @output_OMF);
        $ocrBackupOMFName = trim($ocrBackupOMFName);
        trace("The OMF name of the OCR backup file on DG is: $ocrBackupOMFName");
        # putting a ':' in the OMF name between the DG name and the path 
        # of the OCR backup file in DG
        my $ocrBackupOMFName1;
        my @elems = split(/\//, $ocrBackupOMFName);
        $elems[0] = $elems[0].":";
        $ocrBackupOMFName1 = join("/",@elems);
        trace("Re-format the OMF name as: $ocrBackupOMFName1");
        $CFG->OCR_BACKUP_FILE_NAME($ocrBackupOMFName1);
      }
      else
      {
        trace("Failed to get the OMF name of the OCR backup file on DG.");
        ($rc_OMF == 0) ? die(dieformat(540, $ocrBackupFileName))
                       : die(dieformat(538));
      }
    }

    # Remove OCR from ASM before restoring OCR of the old version
    if (isOCRonASM && isOldVersionLT121())
    {
      stop_crsd_and_check($CFG->ORA_CRS_HOME);
      $self->removeOCRBeforeRestore();
    }
  }
  elsif (DOWNGRADE_OCR eq $stepIndicator)
  {
    s_restoreocrloc();

    # Downgrade OCR
    $self->downgradeOCR();
  }
  else
  {
    croak "Step indicator out of bounds";
  }

  return SUCCESS;
}

# Whether or not the system reboot is needed after OCR configuration
sub rebootRequired
{
  return FALSE;
}

#
# Export functions specific to oraocr class.
#

sub backupOCR
#------------------------------------------------------------------------------
# Function: Backup OCR
# Args    : None
# Returns : TRUE  if success
#           False if failed
#------------------------------------------------------------------------------
{
  my $self       = shift;
  my $ocrconfig  = catfile($CFG->params('ORACLE_HOME'), 'bin', 'ocrconfig');
  my $cmd        = "$ocrconfig -manualbackup";
  my @out        = system_cmd_capture($cmd);
  my $status     = shift @out;

  if ($status == 0)
  {
    trace ("Successfully generated OCR backup");
  }
  else
  {
    trace ("Failed to generate OCR backup");
    return FALSE;
  }

  return TRUE;
}

sub checkOCR
#------------------------------------------------------------------------------
# Function: Checks if the OCR is configured properly and not corrupted.
# Args    : [0] Path for Oracle CRS home
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $self         = shift;
  my $crs_home     = shift;
  my $cmd          = catfile($crs_home, 'bin', 'ocrcheck');
  my @out          = system_cmd_capture($cmd, "-debug");
  my $status       = shift @out;
  my $check_passed = FALSE;

  trace("checkOCR rc=$status");
  my @cmdout1 = grep(/(^PROT-708)/, @out);
  my @cmdout2 = grep(/(^PROT-715)/, @out);
  my @cmdout3 = grep(/(^PROT-721)/, @out);

  if ($status == 0 &&
      scalar(@cmdout1) == 0 &&
      scalar(@cmdout2) == 0 &&
      scalar(@cmdout3) == 0 )
  {
    $check_passed = TRUE;
    trace ("OCR check: passed");
  }
  else
  {
    trace ("OCR check: failed");
  }

  return $check_passed;
}

sub validateOCR
#------------------------------------------------------------------------------
# Function: Validating OCR locations based on existing ocr settings
# Args    : [0] Path for Oracle CRS home
#           [1] Cluster name
#           [2] Comma separated OCR locations
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $self         = shift;
  my $crshome      = shift;
  my $clustername  = shift;
  my $ocrlocations = shift;

  trace ("Oracle CRS home = $crshome");

  if (!$crshome)
  {
    print_error(4);
    return FALSE;
  }

  if (!(-d $crshome))
  {
    print_error(22, $crshome);
    return FALSE;
  }

  trace ("Oracle CRS home = $crshome");

  if (!($clustername))
  {
    print_error(6);
    return FALSE;
  }

  trace ("Oracle cluster name = $clustername");

  if (isAddNode($CFG->HOST))
  {
    if (! s_copyOCRLoc())
    {
      print_error(101);
      return FALSE;
    }
  }
  elsif (! $ocrlocations)
  {
    print_error(8);
    return FALSE;
  }

  trace ("OCR locations = $ocrlocations");
  trace ("Validating OCR");
  return s_validateOCR ($crshome, $clustername, $ocrlocations);
}

sub validate_ocrconfig
#------------------------------------------------------------------------------
# Function: Creates OCR config file if it does not exists.
# Args    : [0] List of OCR locations
#           [1] TRUE  - SIHA mode
#               FALSE - Cluster mode
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $self         = shift;
  my $ocrlocations = shift;
  my $isSIHA       = shift;

  if (!$ocrlocations)
  {
    print_error(9);
    return FALSE;
  }

  trace ("OCR locations = " . $ocrlocations);

  # OSD to validate OCR config
  s_validate_ocrconfig ($ocrlocations, $isSIHA) or return FALSE;

  return TRUE;
}

sub downgradeOCR
#------------------------------------------------------------------------------
# Function: Downgrade OCR
# Args    : None
# Returns : TRUE  if downgrade is successful
#           FALSE if failed to downgrade
#------------------------------------------------------------------------------
{
  my $self = shift;

  trace("Downgrading OCR ...");
  if (FALSE == ocrDowngrade())
  {
    exit(1);
  }
  else
  {
    trace("Check if the OCR is properly configured");
    if (FALSE == $self->checkOCR($CFG->OLD_CRS_HOME))
    {
      die(dieformat(361));
    }
  }
  return TRUE;
}

sub localonlyOCR_exists
#------------------------------------------------------------------------------
# Function: Verifies if the local-only OCR exists
# Args    : None
# Returns : TRUE  if local-only OCR exists
#           FALSE if local-only OCR is not found
#------------------------------------------------------------------------------
{
  my $self       = shift;
  my $found      = FALSE;
  my $local_only = s_get_config_key("ocr", "local_only");
  my $ocrcfg_loc = s_get_config_key("ocr", "ocrconfig_loc");
  my $db_home;

  if ($local_only =~ m/true/i)
  {
    $CFG->oldconfig('OCRCONFIG', $ocrcfg_loc);

    # get older version DBHOME path
    if ($ocrcfg_loc =~ m/(.+).cdata.localhost.local.ocr/)
    {
      $db_home = $1;
      if ($db_home)
      {
        $CFG->oldconfig('DB_HOME', $db_home);
        $found = TRUE;
        trace ("local_only config exists");
        #get old  releaseversion of SIHA/SI-CSS.
        my @oldCrsVer = get_has_version($db_home);
        $CFG->oldconfig('ORA_CRS_VERSION', \@oldCrsVer);
      }
      else
      {
        print_error(181);
      }
    }
    else
    {
      print_error(7);
    }
  }
  return $found;
}

sub createLocalOnlyOCR
#------------------------------------------------------------------------------
# Function: Create local-only OCR
# Args    : None
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $self       = shift;
  my $owner      = $CFG->params('ORACLE_OWNER');
  my $dba_group  = $CFG->params('ORA_DBA_GROUP');
  my $local_ocr  = catfile ($CFG->params('ORACLE_HOME'), "cdata",
                            "localhost", "local.ocr");

  trace ("create Local Only OCR...");
  s_createLocalOnlyOCR();

  # create local.ocr and set ownergroup
  open (FILEHDL, ">$local_ocr") or die(dieformat(207, $local_ocr, $!));
  close (FILEHDL);
  s_set_ownergroup ($owner, $dba_group, $local_ocr)
                    or die(dieformat(152, $local_ocr));
  s_set_perms ("0640", $local_ocr)
               or die(dieformat(153, $local_ocr));

  # validate local.ocr and update ocr.loc
  $self->validate_ocrconfig ($local_ocr, $CFG->SIHA)
                             or die(dieformat(301));
  return TRUE;
}

sub get_ocrdisk
#------------------------------------------------------------------------------
# Function: Retrieves OCR location 1
# Args    : None
# Returns : OCR location 1
#------------------------------------------------------------------------------
{
  trace ("Retrieving OCR main disk location");

  if ($CFG->platform_family ne "windows")
  {
    if (!(-r $CFG->params('OCRCONFIG')))
    {
      print_error(103, $CFG->params('OCRCONFIG'));
      return $CFG->params('OCRCONFIG');
    }
  }
  return s_get_config_key("ocr", "ocrconfig_loc");
}

sub get_ocrmirrordisk
#------------------------------------------------------------------------------
# Function: Retrieves OCR location 2
# Args    : None
# Returns : OCR location 2
#------------------------------------------------------------------------------
{
  trace ("Retrieving OCR mirror disk location");

  if ($CFG->platform_family ne "windows")
  {
    if (!(-r $CFG->params('OCRCONFIG')))
    {
      print_error(103, $CFG->params('OCRCONFIG'));
      return $CFG->params('OCRCONFIG');
    }
  }
  return s_get_config_key("ocr", "ocrmirrorconfig_loc");
}

sub get_ocrloc3disk
#------------------------------------------------------------------------------
# Function: Retrieves OCR location 3
# Args    : None
# Returns : OCR location 3
#------------------------------------------------------------------------------
{
  trace ("Retrieving OCR loc3 disk location");
  return s_get_config_key("ocr", "ocrconfig_loc3");
}

sub get_ocrloc4disk
#------------------------------------------------------------------------------
# Function: Retrieves OCR location 4
# Args    : None
# Returns : OCR location 4
#------------------------------------------------------------------------------
{
  trace ("Retrieving OCR loc4 disk location");
  return s_get_config_key("ocr", "ocrconfig_loc4");
}

sub get_ocrloc5disk
#------------------------------------------------------------------------------
# Function: Retrieves OCR location 5
# Args    : None
# Returns : OCR location 5
#------------------------------------------------------------------------------
{
  trace ("Retrieving OCR loc5 disk location");
  return s_get_config_key("ocr", "ocrconfig_loc5");
}

sub get_ocrlocaldisk
#------------------------------------------------------------------------------
# Function: Retrieves Local-only OCR location
# Args    : None
# Returns : Local-only OCR location
#------------------------------------------------------------------------------
{
  trace ("Retrieving OCR local_only location");
  return s_get_config_key("ocr", "local_only");
}

sub discoverOCRDiskgroups
#------------------------------------------------------------------------------
# Function: Get all diskgroups that store the OCR
# Args    : None
# Returns : An array containing the configured OCR diskgroups
#------------------------------------------------------------------------------
{
  my $self = shift;
  my @ocrDiskgroups;

  my $ocrcheck = catfile($CFG->ORA_CRS_HOME, 'bin', 'ocrcheck');
  my @cmd      = ($ocrcheck, '-config', '-debug');
  my @output   = system_cmd_capture(@cmd);
  my $rc       = shift(@output);

  if (0 == $rc)
  {
    trace("Parse the output for diskgroups with OCR");
    foreach my $line (@output)
    {
      if ($line =~ /PROT-709/)
      {
        trace("LINE: $line");
        my @elems = split(/:/, $line);
        my $ocrDG = trim($elems[2]);
        if ($ocrDG =~ /^\+/)
        {
          trace("OCR DG: $ocrDG");
          $ocrDG =~ s/^\+//;
          trace("OCR DG name: $ocrDG");
          push(@ocrDiskgroups, $ocrDG);
        }
      }
    }
  }
  else
  {
    print_lines(@output);
    trace("ocrcheck -config -debug failed with status $rc");
    if ($CFG->DECONFIG && $CFG->FORCE)
    {
      print_error(180, "ocrcheck -config -debug");
      $CFG->DECONFIG_ERROR(TRUE);
      return;
    }
    else
    {
      die(dieformat(180, "ocrcheck -config -debug"));
    }
  }

  trace("Diskgoups with OCR: @ocrDiskgroups");
  return @ocrDiskgroups;
}

sub removeOcrKeyPair
#------------------------------------------------------------------------------
# Function: Remove a key-value pair from OCR.
# Args    : [0] Keyname of the OCR key-value pair
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $self     = shift;
  my $keyName  = shift;
  my $crshome  = $CFG->ORA_CRS_HOME;
  my @program  = ('-exec', '-ocrdelete', '-key', $keyName);
  my $args_str = join(' ', @program);

  my ($rc, @capout) = cluutil(FALSE, $args_str, $crshome);# run as root

  if (0 != $rc)
  {
    if (2 == $rc)
    {
      trace("The key to be deleted doesn't exist");
      return TRUE;
    }

    print_lines(@capout);
    trace("Failed to remove the OCR key pair with the key:'$keyName'".
          "Error message is : @capout.  Error code is $rc");
    return FALSE;
  }
  else
  {
    trace("Succeeded in removing OCR key pair with the key:'$keyName'");
    return TRUE;
  }
}

sub getOcrKeyValue
#------------------------------------------------------------------------------
# Function: Obtain the value of a given OCR key from OCR 
# Args    : [0] Keyname of the OCR key-value pair
# Returns : (rc, Keyvalue)
#           rc = TRUE  if success
#           rc = FALSE if failed
#           Keyvalue: the value of the specific OCR key
#------------------------------------------------------------------------------
{
  my $self        = shift;
  my $keyName     = shift;
  my $keyValue    = "";
  my $crshome     = $CFG->ORA_CRS_HOME;
  my @program     = ('-exec', '-ocrgetval', '-key', $keyName);
  my $args_str    = join(' ', @program);

  my ($rc, @capout) = cluutil(TRUE, $args_str, $crshome);
  if (0 != $rc)
  {
    print_lines(@capout);
    trace("Failed to obtain the value of OCR key (SYSTEM.$keyName) ".
          "from OCR. \n Error message is : @capout.  Error code is $rc");
    $rc = FALSE;
  }
  else
  {
    $rc = TRUE;
    $keyValue = trim($capout[0]);
    trace("SYSTEM.$keyName : [$keyValue]");
  }

  return ($rc, $keyValue); 
}

sub createAndSetOcrKeyPair
#------------------------------------------------------------------------------
# Function: Create and set a key-value pair in OCR under the SYSTEM key.
#           If the OCR key to be written already exists, this function returns
#           without updating the value of the OCR key-value pair.
# Args    : [0] Keyname of the OCR key-value pair under the SYSTEM key
#           [1] Value of the OCR key-value pair
# Returns : TRUE      if success
#           FALSE     if failed
#------------------------------------------------------------------------------
{
  my $self        = shift;
  my $keyName     = shift;
  my $keyValue    = shift;
  my $crshome     = $CFG->ORA_CRS_HOME;
  my @program     = ('-exec', '-ocrcreateandset', '-key', $keyName,
                     '-value', $keyValue);
  my $args_str    = join(' ', @program);

  my ($rc, @capout) = cluutil(FALSE, $args_str, $crshome); #run as root

  if (0 == $rc)
  {
    trace("Succeeded in writing the key pair ".
          "(SYSTEM.$keyName:$keyValue) to OCR");
    return TRUE;
  }
  elsif (2 == $rc)
  {
    trace("The OCR key (SYSTEM.$keyName) already exists.");
    return TRUE;
  }
  else
  {
    print_lines(@capout);
    trace("Failed to write the key pair (SYSTEM.$keyName:$keyValue) ".
          "to OCR. \n Error message is : @capout.  Error code is $rc");
    return FALSE;
  }
}

sub writeOcrKeyPair
#------------------------------------------------------------------------------
# Function: Write a key-value pair in OCR under the SYSTEM key.
# Args    : [0] Keyname of the OCR key-value pair under the SYSTEM key
#           [1] Value of the OCR key-value pair
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $self        = shift;
  my $keyName     = shift;
  my $keyValue    = shift;
  my $crshome     = $CFG->ORA_CRS_HOME;
  my @program     = ('-exec', '-ocrsetval', '-key', $keyName, '-value',
                     $keyValue);
  my $args_str    = join(' ', @program);

  my ($rc, @capout) = cluutil(FALSE, $args_str, $crshome); #run as root

  if (0 != $rc)
  {
    print_lines(@capout);
    trace("Failed to write the key pair (SYSTEM.$keyName:$keyValue) ".
          "to OCR. \n Error message is : @capout.  Error code is $rc");
    return FALSE;
  }
  else
  {
    trace("Succeeded in writing the key pair ".
          "(SYSTEM.$keyName:$keyValue) to OCR");
    return TRUE;
  }
}

sub getOcrBackupWithKey
#------------------------------------------------------------------------------
# Function: Get the OCR backup filename name with ocrdump
# Args    : [0] OCR backup Keyname
# Returns : The value for the given backup keyname in OCR.
#------------------------------------------------------------------------------
{
  my $self       = shift;
  my $keyname    = shift;
  $keyname       = "SYSTEM.rootcrs.".$keyname;
  my $crsHome    = getCrsHome();
  my $OCRDUMPBIN = catfile ($crsHome, 'bin', 'ocrdump');

  trace("Get ocr backup file name with ocrdump");
  if ($CFG->platform_family eq "windows")
  {
    open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname $keyname |");
  }
  else
  {
    open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname '$keyname' |");
  }
  my @output = <OCRDUMP>;
  close (OCRDUMP);

  trace("ocrdump output: @output");

  if (scalar(grep (/$keyname/, @output)) > 0)
  {
    trace("The key pair with key name: $keyname exists in OCR.");
    my @txt = grep (/ORATEXT/, @output);
    my ($key, $value) = split (/:/, $txt[0]);
    $value = trim($value);
    trace("Ocrbackup info: $value");
    return $value;
  }
  else
  {
    # ocrdump doesn't work or the ocr backup info is null in the first place
    return "";
  }
}

sub getOcrBackupOMF
#------------------------------------------------------------------------------
# Function: Get the OMF name(s) of a given OCR backup file stored on DG.
# Args    : [0] The file name of the OCR backup file
#           [1] CRS Home
# Returns : Array containing both the return code and the captured output
#------------------------------------------------------------------------------
{
  my $self = shift; 
  my $ocrBackupFileName = $_[0];
  my $crsHome = $_[1];
  if (! defined $crsHome)
  {
    $crsHome = $CFG->ORA_CRS_HOME;
  }
  verify_gpnp_dirs($CFG->ORA_CRS_HOME,
                   $CFG->params('GPNPGCONFIGDIR'),
                   $CFG->params('GPNPCONFIGDIR'),
                   $CFG->HOST,
                   $CFG->params('ORACLE_OWNER'),
                   $CFG->params('ORA_DBA_GROUP'));
  my $asm_mode = getASMMode();
  setOraHomeSID($asm_mode, $crsHome);

  my $asmcmd = catfile($crsHome, "bin", "asmcmd");
  my @cmd;
  if (! isOldVersionLT121())
  {
    $asmcmd = $asmcmd . " --nocp";
  }

  if ($CFG->platform_family eq 'windows')
  {
    @cmd = ($asmcmd, "find", "--type", "'OCRBACKUP'", "'*'",
            "'*$ocrBackupFileName*'");
  }
  else
  {
    @cmd  = ($asmcmd, "find", "--type", "\"OCRBACKUP\"", "\"*\"",
             "\"*$ocrBackupFileName*\"");
  }
  my @capout;
  my $rc_asmcmd = run_as_user2($CFG->params('ORACLE_OWNER'), \@capout, @cmd);
  trace("Return code: $rc_asmcmd \nOutput: @capout");
  return ($rc_asmcmd, @capout);
}

sub removeOcrBackupFromDG
#------------------------------------------------------------------------------
# Function: Remove the OCR backup file from DG given its OMF name.
# Args    : [0] The OMF name of the OCR backup file on DG.
#           [1] CRS Home
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $self = shift;
  my $ocrBackupOMFName = $_[0];
  my $crsHome = $_[1];
  if (! defined $crsHome)
  {
    $crsHome = $CFG->ORA_CRS_HOME;
  }
  my $asm_mode = getASMMode();
  setOraHomeSID($asm_mode, $crsHome);

  my $asmcmd = catfile($crsHome, "bin", "asmcmd");
  my @capout;
  if (! isOldVersionLT121())
  {
    $asmcmd = $asmcmd . " --nocp";
  }

  my @cmd = ($asmcmd, "rm", $ocrBackupOMFName);
  my $rc_asmcmd = run_as_user2($CFG->params('ORACLE_OWNER'), \@capout, @cmd);
  if($rc_asmcmd == 0)
  {
    trace("Successfully removed the OCR backup file on DG ".
          "with OMF name: $ocrBackupOMFName");
    return SUCCESS;
  }
  else
  {
    trace("Failed to remove the OCR backup file on DG ".
          "with OMF name: $ocrBackupOMFName");
    return FAILED;
  }
}


sub get_ocrid
#------------------------------------------------------------------------------
# Function: Returns OCRID
# Args    : [0] Path for Oracle CRS home
# Returns : OCRID fetched from  ocrcheck or -1 in case of error
#------------------------------------------------------------------------------
{
  my $self    = shift;
  my $crshome = shift;
  my $id      = -1;
  my $ocrcheck;

  if ($CFG->platform_family eq "windows")
  {
    $ocrcheck = catfile($crshome, 'bin', 'ocrcheck.exe');
  }
  else
  {
    $ocrcheck = catfile($crshome, 'bin', 'ocrcheck');
  }

  trace("Executing ocrcheck to get ocrid");
  open(OCRCHECK, "$ocrcheck |" );
  my @output = <OCRCHECK>;
  close(OCRCHECK);

  my @txt = grep (/ ID /, @output);
  foreach my $line (@txt)
  {
    my ($idstring, $oid)  = split(/:/, $line);
    $id = trim($oid);
  }

  if ($id == -1)
  {
    trace("get_ocrid: Failed to get ocrid ");
  }
  else
  {
    $CFG->OCR_ID($id);
  }

  return $id;
}


=head2 backupOcr4Restore

  Use 'ocrconfig -manualbackup -filename <FILENAME>' to take OCR backup
  directly on shared storage.

=head3 Parameters

  [0] CRS home on which 'ocrconfig -manualbackup -filename <FILENAME>' is run

=head3 Returns

  SUCCESS or FAILED

=cut

sub backupOcr4Restore
{
  my $self = shift;
  my $crsHome = $_[0];
  my $ckptName = "ROOTCRS_OCRBACKUP";
  my @oldCrsVer = @{$CFG->oldconfig('ORA_CRS_VERSION')};
  my $crsversion = join('.', @oldCrsVer);
  my $clustername = $CFG->params('CLUSTER_NAME');
  my $ocrBackupFileName = "$clustername"."_backup"."$crsversion".".ocr";
  $ocrBackupFileName = lc($ocrBackupFileName);
  trace("OLD CRS HOME: " . $crsHome);

  # OCR backup file is directly created on DG during the firstnode upgrade.

  if (! isCkptexist($ckptName, "-global"))
  {
    trace("Writing checkpoint for OCR manual backup");
    writeGlobalCkpt($ckptName, CKPTSTART);
  }

  if (! isCkptSuccess($ckptName, "-global"))
  {
    # Existing backup file with the name "$clustername_backup$crsversion.ocr" 
    # is removed on the ASM diskgroup before creating the current backup.
    
    # Save the current ORACLE_HOME before setting it as the lower version 
    # ORACLE_HOME in following subroutine getOcrBackupOMF().
    my $currentHome = saveOraHome();
    trace("Remove any expired OCR backup files on DG with the name ".
          "$ocrBackupFileName");
    my ($rc_OMF, @output_OMF) = $self->getOcrBackupOMF($ocrBackupFileName,$crsHome);
    resetOraHome($currentHome);
    my $count = scalar(grep(/$ocrBackupFileName/, @output_OMF));
    if ($rc_OMF == 0)
    {
      if ($count >= 1)
      {
        trace("There is at least one expired OCR backup file with name ".
              "$ocrBackupFileName in DG.");
        trace("Start to remove the expired OCR backup file from DG.");
        my $rc_asmcmd;
        foreach my $omfName (@output_OMF)
        {
          trim($omfName);
          $rc_asmcmd = $self->removeOcrBackupFromDG($omfName,$crsHome);
          if ($rc_asmcmd eq FAILED)
          {
            # There is a Bug 20775137 on asmcmd command in 11.2. "asmcmd -rm"
            # successfully removes the OCR backup file on DG but returns a 
            # non-zero return code.
            trace("'asmcmd -rm' command returned non-zero return code. Need ". 
                  "to use 'asmcmd find' command to confirm whether the OCR " .
                  "backup file on DG has been removed.");  
          }
        }
        # make sure there is no OCR backup file on DG
        ($rc_OMF, @output_OMF) = $self->getOcrBackupOMF($ocrBackupFileName,$crsHome);
        resetOraHome($currentHome);
        $count = scalar(grep(/$ocrBackupFileName/, @output_OMF));
        if ($rc_OMF == 0)
        {
          if ($count >= 1)
          {
            trace("An existing OCR backup file found on DG."); 
            die(dieformat(539));
          }
        }
        else
        {
          trace("Failed to get the OMF name of the OCR backup file on DG.");
          die(dieformat(538));
        }
      }
      trace("Currently, there is no OCR backup file on DG with the name ".
            "$ocrBackupFileName");
    }
    else
    {
      trace("Failed to get the OMF name of the OCR backup file on DG.");
      die(dieformat(538));
    }

    trace("Start to do OCR manual backup.");

    my $ocrConfig = catfile($crsHome, 'bin', 'ocrconfig');

    print_info(515);

    # OCR backup file will be directly created on shared storage (DG)
    # get the OCR location
    my @output = getOCRLoc();
    my $ocrType = shift @output;
    my $value = shift @output;
    $value = "+" . $value . ":" if ($ocrType == 2);
    $ocrBackupFileName = "$clustername"."_backup"."$crsversion".".ocr";
    my $ocrBackupFileNamewithDG = lc($ocrBackupFileName);
    $ocrBackupFileNamewithDG = "$value" . "$ocrBackupFileNamewithDG";
   
    # Backup OCR using 'ocrconfig -manualbackup -filename <FILENAME>'
    my $cmd = "$ocrConfig -manualbackup -filename $ocrBackupFileNamewithDG";
    my ($backupStatus, @ocrBackups) = system_cmd_capture($cmd);
    if ($backupStatus == 0)
    {
      trace("Successfully create OCR backup on shared storage.");
      trace("The OCR manual backup file name is: $ocrBackupFileNamewithDG");
      writeCkptProperty($ckptName, "OCR_BACKUP_ON_SHARED_STORAGE",
                      "TRUE", "-global");
      writeGlobalCkpt($ckptName, CKPTSUC, "-transferfile");
      
      trace("Start to query the OMF file name of the OCR backup file: " .
            "$ocrBackupFileName, which was just being generated on DG.");
      my ($rc_OMF, @output_OMF) 
         = $self->getOcrBackupOMF($ocrBackupFileName,$crsHome);
      resetOraHome($currentHome);
      my $count = scalar(grep(/$ocrBackupFileName/, @output_OMF));
      if (($rc_OMF == 0) && ($count == 1))
      {
        my $ocrBackupOMFName = join("", @output_OMF);
        $ocrBackupOMFName = trim($ocrBackupOMFName);
        trace("The OMF name of the OCR backup file on DG is: $ocrBackupOMFName");
        # putting a ':' in the OMF name between the DG name and the path
        # of the OCR backup file in DG
        my $ocrBackupOMFName1;
        my @elems = split(/\//, $ocrBackupOMFName);
        $elems[0] = $elems[0].":";
        $ocrBackupOMFName1 = join("/",@elems);
        trace("Re-format the OMF name as: $ocrBackupOMFName1");
        $ocrBackupFileName = $ocrBackupOMFName1;
      }
      else
      {
        trace("Failed to get the OMF name of the OCR backup file on DG.");
        ($rc_OMF == 0) ? die(dieformat(540, $ocrBackupFileName))
                       : die(dieformat(538));
      }
    }
    else
    {
      print_lines(@ocrBackups) if (! isOldVersionLT12102());
      trace("Output: @ocrBackups");
      trace ("Failed to create OCR backup on shared storage.");
      if (isOldVersionLT12102())
      {
        trace("'ocrconfig -manualbackup -filename <FILENAME>' command is not ".
              "supported. Fall back to manualbackup and copying to shared " .
              "storage.");
        my $ocrBackupNodeName;
        trace("Start to do OCR manual backup and store backup info in global ".
              "checkpoint."); 
        # Backup OCR using 'ocrconfig -manualbackup'
        $cmd = "$ocrConfig -manualbackup";
        ($backupStatus, @ocrBackups) = system_cmd_capture($cmd);
        if ($backupStatus == 0)
        {
          trace ("Successfully generated OCR backup.");
          foreach my $ocrBackup (@ocrBackups)
          {
            #Get the OCR backup file name from the first line of previous output;
            if ($ocrBackup =~ /(\S+).+\s(.+)+\.ocr/)
            {
              $ocrBackupNodeName = $1;
              $ocrBackupNodeName = trim($ocrBackupNodeName);
              $ocrBackupFileName = $2.".ocr";
              $ocrBackupFileName = trim($ocrBackupFileName);
              if ($ocrBackupFileName !~ /^\+/)
              {
                trace("The OCR manual backup file: $ocrBackupFileName is ". 
                      "created on node: $ocrBackupNodeName");
              }
              trace("The OCR manual backup file name is: $ocrBackupFileName");
              last;
            }
            else
            {
              next;
            }
          }
        }

        if ( !defined($ocrBackupNodeName) || !defined($ocrBackupFileName))
        {
          print_lines(@ocrBackups);
          trace("Failed to resolve the node_Name & file_Name of OCR manual backup.");
          die(dieformat(183, $backupStatus));#OCR manual backup operation failed
        }

        if ($ocrBackupFileName !~ /^\+/)
        { 
          # MANUAL_BACKUP_NODE_NAME in global CKPT is used as a flat to indicate
          # whether OCR backp file was generated on local file system and need 
          # to be copied to DG.
          trace("Write OCR manual backup info into the global checkpoint file,".
                "so that the OCR backup file can be copied to DG.");
          writeCkptProperty($ckptName, "MANUAL_BACKUP_NODE_NAME", 
                            $ocrBackupNodeName, "-global");
        }
      }
      else
      {
        writeCkpt($ckptName, CKPTFAIL);
        die(dieformat(183, $backupStatus));
      }
    }
    # "MANUAL_BACKUP_FILE_NAME will store the OCR backup file name in DG,
    # so that it can be used to restore the OCR during lastnode downgrade.
    writeCkptProperty($ckptName, "MANUAL_BACKUP_FILE_NAME",
                          $ocrBackupFileName, "-global", "-transferfile");
    
    print_info(516);
  }
  else # checkpoint "ROOTCRS_OCRBACKUP" is SUCCESS
  {
    trace("Skip taking the OCR backup, the backup is already taken for this ".
          "upgrade.");
  }
  return SUCCESS;
}


sub removeOCRBeforeRestore
#------------------------------------------------------------------------------
# Function: Remove OCR file on ASM before restoring the OCR of the old version
# Args    : None
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $self     = shift;
  my $asm_mode = getASMMode();
  setOraHomeSID($asm_mode);

  # remove the OCR
  trace("Start to remove OCR before restoring the OCR of the old version.");

  # get the DG name
  my @output = getOCRLoc();
  my $ocrType = shift @output;
  return if ($ocrType != 2);
  my $dgname = shift @output; 

  my $dg = "+".$dgname;
  my $crsHome = getCrsHome();
  trace("CRS HOME: $crsHome");
  my $asmcmd = catfile($crsHome, "bin", "asmcmd");
  my $ocrfile = "\%REGISTRY.255.\%";
  my @cmd1 = ($asmcmd, "find", $dg, $ocrfile);
  my @capout1 = run_cmd_as_usr_with_backticks(\@cmd1, $CFG->params('ORACLE_OWNER'));
  my $rc1 = shift @capout1;
  trace("rc = $rc1");
  my $ocrdgfile = $capout1[0];
  chomp($ocrdgfile);
  trace("OCR file on the diskgroup: $ocrdgfile");
  my @cmd2 = ($asmcmd, "rm", "-f",  $ocrdgfile);
  my @capout2 = run_cmd_as_usr_with_backticks(\@cmd2, $CFG->params('ORACLE_OWNER'));
  my $rc2 = shift @capout2;
  trace("rc = $rc2");
  if ($rc2)
  {
    my @capout3 = run_cmd_as_usr_with_backticks(\@cmd1, $CFG->params('ORACLE_OWNER'));
    my $rc3 = shift @capout3;
    trace("rc = $rc3");
    my $ocrdgfile3 = $capout3[0];
    chomp($ocrdgfile3);
    if($ocrdgfile3 ne '')
    {
      trace("Failed to remove OCR.");
      trace("OCR file on the diskgroup: $ocrdgfile3");
      print_lines(@capout2);
      die(dieformat(427, $ocrdgfile, "@cmd2"));
    }
  }
  trace("Succesfully removed OCR.");
  return TRUE;
}

sub removeOCROnDG
#------------------------------------------------------------------------------
# Function: Remove the OCR file and OCR mirrors in the diskgroups
# Args    : [0] ASM Mode
# Returns : SUCCESS  if success
#           FAILED if failed
#------------------------------------------------------------------------------
{
  my $self     = shift;
  my $asm_mode = shift;
  my @ocrdgfiles;

  # Retrieve the complete OCR file name
  my $ocrcheck = catfile($CFG->ORA_CRS_HOME, 'bin', 'ocrcheck');
  my @cmd = ($ocrcheck, '-config', '-details', '-debug');
  my @output = system_cmd_capture(@cmd);
  my $rc = shift(@output);

  if (0 == $rc)
  {
    trace("Parse the output for the complete OCR file name");
    foreach my $line (@output)
    {
      if ($line =~ /PROT-709/)
      {
        trace("LINE: $line");
        my @elems = split(/:/, $line);
        my $ocrloc = trim($elems[2]);
        if ($ocrloc =~ /^\+.+registry/)
        {
          trace("ASM filename: $ocrloc");
          push(@ocrdgfiles, $ocrloc);
        }
      }
    }
  }
  else
  {
    print_lines(@output);
    if ($CFG->DECONFIG && $CFG->FORCE)
    {
      print_error(461);
      $CFG->DECONFIG_ERROR(TRUE);
      return FAILED;
    }
    else
    {
      die(dieformat(461));
    }
  }

  trace("OCR files on the diskgroup: @ocrdgfiles");

  setOraHomeSID($asm_mode);

  # Remove the OCR file on DG
  foreach my $ocrdgfile (@ocrdgfiles)
  {
    removeFileOnDG($asm_mode, $ocrdgfile);
  }
  return SUCCESS;
}

sub get_softVersionfromOCR
#------------------------------------------------------------------------------
# Function: Get the software version for the given node
# Args    : [0] Nodename
# Returns : Software version
#------------------------------------------------------------------------------
{
  my $self        = shift;
  my $nodename    = shift;
  my $OCRDUMPBIN  = catfile ($CFG->ORA_CRS_HOME, 'bin', 'ocrdump');

  if ($CFG->JOIN) { return get_softVersionfromOCR2($nodename); }

  trace("Getting crs software verison from ocrdump");
  if ($CFG->platform_family eq "windows")
  {
    open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
          "SYSTEM.version.hostnames.$nodename |");
  }
  else
  {
    open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
          "'SYSTEM.version.hostnames.$nodename'|");
  }
  my @output = <OCRDUMP>;
  close (OCRDUMP);

  trace("ocrdump output for software version on $nodename is @output");
  my @txt = grep (/ORATEXT/, @output);
  my ($key, $softver) = split (/:/, $txt[0]);
  $softver = trim($softver);
  trace ("key is $key");
  trace ("softver is $softver");

  return $softver;
}

#
# Private methods
#

sub upgradeOCR
#------------------------------------------------------------------------------
# Function: Upgrades OCR during clusterware upgrade
# Args    : None
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  # Execute 'ocrconfig -upgrade' command.
  if (! configureOCR_runocrconfig())
  {
    return FALSE;
  }

  return TRUE;
}

sub configureOCR
#------------------------------------------------------------------------------
# Function: Creates or upgrades OCR in Cluster mode
# Args    : None
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $self = $CFG->compOCR;

  # Abort if OCR is already configured
  if ((! $CFG->isRerun) && (! isReusedg()) &&
      $self->checkOCR($CFG->ORA_CRS_HOME))
  {
    trace("Existing OCR configuration found, aborting the configuration. "
          ."Rerun configuration setup after deinstall");
    die(dieformat(428));
  }

  if (! configureOCR_runocrconfig())
  {
    return FALSE;
  }

  if (! configureOCR_runclscfg())
  {
    return FALSE;
  }

  return TRUE;
}

sub configureOCR_runocrconfig
#------------------------------------------------------------------------------
# Function: Executes 'ocrconfig -upgrade' command to create or upgrade OCR
# Args    : None
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $status  = 0;

  my $OCRCONFIGBIN = catfile ($CFG->ORA_CRS_HOME, "bin", "ocrconfig");
  my @runocrconfig = ("$OCRCONFIGBIN", "-upgrade",
                      $CFG->params('ORACLE_OWNER'),
                      $CFG->params('ORA_DBA_GROUP'));

  trace ("Creating or upgrading OCR keys");
  print_info(482, "@runocrconfig");
  $status = system_cmd("@runocrconfig");
  if (0 != $status)
  {
    trace("ocrconfig -upgrade failed with status $status");
    print_error(157);
    return FALSE;
  }

  trace ("OCR keys are successfully populated");

  return TRUE;
}

sub configureOCR_runclscfg
#------------------------------------------------------------------------------
# Function: Executes 'clscfg' command to create required keys in OCR
# Args    : None
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $status  = 0;
  my $asmgrp  = $CFG->params('ORA_ASM_GROUP');

  my $CLSCFGBIN = catfile ($CFG->ORA_CRS_HOME, "bin", "clscfg");
  my @runclscfg = ("$CLSCFGBIN", "-install",
                   '-o', $CFG->ORA_CRS_HOME, 
                   '-g', $asmgrp);

  if (defined($CFG->clscfg_node2site_arg_h))
  {
    push @runclscfg, $CFG->clscfg_node2site_arg_h;
  }

  if (defined($CFG->clscfg_site2guid_arg_y))
  {
    push @runclscfg, $CFG->clscfg_site2guid_arg_y;
  }

  if (defined($CFG->clscfg_clsprop_ocr_arg_p))
  {
    push @runclscfg, $CFG->clscfg_clsprop_ocr_arg_p;
  }

  if ($CFG->CLSCFG_EXTRA_PARMS)
  {
    push @runclscfg, @{$CFG->CLSCFG_EXTRA_PARMS};
  }

  # clscfg - Initialize the Oracle Cluster Registry for the cluster. Should be
  #          done once per cluster install. Overwriting a configuration while
  #          any CRS daemon is running can cause serious issues.

  trace("Executing clscfg");

  my @out = system_cmd_capture("@runclscfg");
  $status = shift @out;

  # Get true return value of clscfg (i.e. rc for spawned process)
  if ((0 == $status) || ($status == 105 && (isReusedg() || $CFG->isRerun)))
  {
    trace ("Oracle Cluster Registry initialization completed");

    if ($CFG->CLSCFG_POST_CMD)
    {
      my @cmd = @{$CFG->CLSCFG_POST_CMD};
      system_cmd(@cmd);
    }
  }
  else
  {
    print_lines(@out);
    print_error(159);
    return FALSE;
  }

  return TRUE;
}

sub configureOCR_siha
#------------------------------------------------------------------------------
# Function: Creates or upgrades OCR in SIHA mode
# Args    : None
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $status = 0;
  my $asmgrp = $CFG->params('ORA_ASM_GROUP');
  my $host   = $CFG->HOST;

  # Create necessary configuration with 'clscfg -local'. Needed for
  # compatibility with older DBs (10.x, 11.1).
  my $clscfg = catfile ($CFG->ORA_CRS_HOME, "bin", "clscfg");
  my $cmdclscfg = "$clscfg -local -g $asmgrp";

  $status = system_cmd ($cmdclscfg);
  if ($status != 0)
  {
    print_error(160, $cmdclscfg);
    return FALSE;
  }
  else
  {
    trace ("Creating local-only OCR ($cmdclscfg) ... succeeded");
  }

  my $crsctl = crs_exec_path('crsctl');
  my $cmdcrsctl = "$crsctl pin css -n $host";

  $status = system_cmd ($cmdcrsctl);
  if ($status != 0)
  {
    print_error(161, $cmdcrsctl);
    return FALSE;
  }
  else
  {
    trace ("Pin node ($cmdcrsctl) ... succeeded");
  }

  return TRUE;
}

sub createOCRLocationList
#------------------------------------------------------------------------------
# Function : Parse the output of 'ocrcheck -config -details -debug' command,
#            and creates the list of OCR locations
# Args     : [0] Oracle Home
# Returns  : A vertical-bar separated list of OCR locations
#------------------------------------------------------------------------------
{
  my $crs_home = shift;
  my $ocr_locations;
  my @filenames;

  my $ocrcheck = catfile($crs_home, 'bin', 'ocrcheck');
  my @cmd      = ($ocrcheck, '-config', '-details', '-debug');
  my @output   = system_cmd_capture(@cmd);
  my $status   = shift(@output);

  if (0 == $status)
  {
    trace("Parse the output for OCR locations");
    foreach my $line (@output)
    {
      if ($line =~ /PROT-709/)
      {
        trace("LINE: $line");
        my @elems = split(/:/, $line);
        push(@filenames, trim($elems[2]));
      }
    }
  }
  else
  {
    print_lines(@output);
    die(dieformat(461));
  }

  if (scalar(@filenames) > 0)
  {
    foreach my $fname (@filenames)
    {
      $ocr_locations = ($ocr_locations) ? ($ocr_locations."|".$fname) : ($fname);
    }
  }

  trace("OCR locations: $ocr_locations");
  return $ocr_locations;
}

sub pushOCRLocSyncFile
#------------------------------------------------------------------------------
# Function : Creates OCR sync file and pushes it to all other nodes
# Args     : [0] List of OCR locations
# Returns  : None
#------------------------------------------------------------------------------
{
  my $ocr_locations = shift;
  my $syncfile      = getOCRLocSyncFile();

  trace("Create the file [$syncfile] for syncing OCR locations list");
  open(SYNCFILE, ">$syncfile") or die(dieformat(208, $syncfile, $!));
  print SYNCFILE "$ocr_locations\n";
  close(SYNCFILE);

  s_set_ownergroup($CFG->params('ORACLE_OWNER'),
                   $CFG->params('ORA_DBA_GROUP'),
                   $syncfile) or die(dieformat(152, $syncfile));
  s_set_perms("0750", $syncfile) or die(dieformat(153, $syncfile));

  unless (isOracleHomeShared())
  {
     copy_file_to_nodes($syncfile,
                        $syncfile,
                        TRUE,
                        $CFG->params('NODE_NAME_LIST')) or die(dieformat(462));
  }
}

sub getOCRLocSyncFile
#------------------------------------------------------------------------------
# Function : Populates the sync filename to store the list of OCR locations
# Args     : None
# Returns  : OCR Sync filename
#------------------------------------------------------------------------------
{
  my $sync_file = catfile($CFG->params('GPNPGCONFIGDIR'),
                          'gpnp', 'seed', 'asm', 'ocrloc.conf');
  trace("Sync file: $sync_file");
  return $sync_file;
}

sub restoreOcrFromBackup
#------------------------------------------------------------------------------
# Function : Restores OCR from the backup file
# Args     : [0] OCR backup filename
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $ocrBackupFileName = shift;
  my $OLD_CRS_HOME      = $CFG->oldcrshome;
  my $ocrconfigbin      = catfile ($OLD_CRS_HOME, 'bin', 'ocrconfig');

  trace("OCR backup file name is $ocrBackupFileName");

  my $option;
  $option = "-restore";

  my $cmd = "$ocrconfigbin $option $ocrBackupFileName";
  my @output = system_cmd_capture($cmd);
  my $rc = shift @output;
  if (0 !=  $rc)
  {
    print_lines(@output);
    return FALSE;
  }
  else
  {
    trace("Successfully restored OCR from the backup file");
    return TRUE;
  }
}


sub get_softVersionfromOCR2
#------------------------------------------------------------------------------
# Function: Get the software version for the given node
# Args    : [0] Nodename
# Returns : Software version
#------------------------------------------------------------------------------
{
  my $nodename    = shift;
  my $OCRDUMPBIN  = catfile ($CFG->ORA_CRS_HOME, 'bin', 'ocrdump');

  trace("Getting crs software verison from ocrdump");
  if ($CFG->platform_family eq "windows")
  {
    open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
          "SYSTEM.version.hostnames.$nodename |");
  }
  else
  {
    open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
          "'SYSTEM.version.hostnames.$nodename'|");
  }
  my @output = <OCRDUMP>;
  close (OCRDUMP);

  trace("ocrdump output for software version on $nodename is @output");
  my @txt = grep (/ORATEXT/, @output);
  my ($key, $softver) = split (/:/, $txt[0]);
  $softver = trim($softver);
  trace ("key is $key");
  trace ("softver is $softver");

  if (! $softver)
  {
    trace("Searching the OCR dump for software version");

    open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
          "SYSTEM.css.node_names.$nodename.nodenum |");
    my @output = <OCRDUMP>;
    close(OCRDUMP);

    trace("The ocrdump output for node num on $nodename is @output");
    my @val = grep(/UB4/, @output);
    if (0 == scalar(@val))
    {
      trace("Failed to retrieve the node num for $nodename");
      return $softver;
    }

    my ($valueType, $nodenum) = split(/:/, $val[0]);
    $nodenum = trim($nodenum);
    trace("The node num for $nodename is $nodenum");

    open (OCRDUMP, "$OCRDUMPBIN -stdout -noheader -keyname " .
          "SYSTEM.version.node_numbers.node$nodenum |");
    @output = <OCRDUMP>;
    close(OCRDUMP);

    trace("ocrdump output for software version on $nodename is @output");
    my @txt = grep(/ORATEXT/, @output);
    if (0 == scalar(@txt))
    {
      trace("Failed to retrieve the software version on $nodename");
      return $softver;
    }

    ($key, $softver) = split(/:/, $txt[0]);
    $softver = trim($softver);
    trace("The software version on $nodename is $softver");
  }
  return $softver;
}

sub ocrDowngrade
#------------------------------------------------------------------------------
# Function: Downgrade OCR
# Args    : None
# Returns : TRUE  if success
#           FALSE if failed
#------------------------------------------------------------------------------
{
  my $OLD_CRS_HOME  = $CFG->oldcrshome;
  my $ocrconfigbin  = catfile ($OLD_CRS_HOME, 'bin', 'ocrconfig');
  my $oldcrsver     = $CFG->oldcrsver;
  my $rc            = TRUE;
  my $backupFile4Restore;

  if (! -e $OLD_CRS_HOME)
  {
    print_error(46, $OLD_CRS_HOME);
    trace ("crshome: $OLD_CRS_HOME not found. " .
           "Re-try the command with right path for old crs home");
    exit 1;
  }

  if ($CFG->OCRBACKUPFILE)
  {
    trace("Start to restore OCR using the OCR backup file given by user.");

    my $ocrBackupFile = $CFG->OCRBACKUPFILE;
    if (FALSE == restoreOcrFromBackup($ocrBackupFile))
    {
      trace("Fail to restore OCR with the user given backup file. \n".
            "Start to restore OCR with the manual backup file.");
    }
    else
    {
      print_info(338, $oldcrsver);
      return $rc;
    }
  }

  # restore OCR
  trace("Start to restore OCR using the OCR backup file.");
  $backupFile4Restore = $CFG->OCR_BACKUP_FILE_NAME;

  if(FALSE == restoreOcrFromBackup($backupFile4Restore))
  {
    my $rootcrs = catfile($CFG->params('ORACLE_HOME'), 'crs', 'install',
                          'rootcrs.pl');
    print_error(204, $oldcrsver);
    print_info(560);
    $rc = FALSE;
    return $rc;
  }

  print_info(338, $oldcrsver);
  return $rc;
}

sub getOCRLoc
#------------------------------------------------------------------------------
# Function: Get the OCR location.
#           If the OCR is not on ASM (on NAS), return the NAS path;
#           If the OCR is on ASM (on DG), return the DG name.
# Args    : None
# Returns : An Array contains 2 values:
#           1. a code indicating the type of OCR location, i.e.:
#              0: Error, fail to get OCR location from /etc/oracle/ocr.loc file;
#              1: OCR is on NAS;
#              2: OCR is on DG.
#
#           2. Empty string if Code is '0';
#              The path to the location of OCR on NAS if Code is '1';
#              The DG name of OCR location if Code is '2'.
#------------------------------------------------------------------------------
{
  # read /etc/oracle/ocr.loc to get the NAS path of OCR
  my $ocr_loc = s_get_config_key("ocr", "ocrconfig_loc");
  # check if it's a symbolic link
  if (-l $ocr_loc)
  {
    $ocr_loc = s_getAbsLink($ocr_loc);
  }
  my $NASpath;
  my $DGname;

  if (!isOCRonASM())
  {
    trace("OCR is on NAS.");
    if ($ocr_loc eq "" || !($ocr_loc =~ /\//))
    {
      trace("Fail to get OCR location from /etc/oracle/ocr.loc file");  
      return (0, "");
    }
    my $pdir = dirname($ocr_loc);
    if (! -e $pdir) { die(dieformat(22, $pdir)); }

    trace ("The OCR location path is $pdir.");
    return (1, $pdir);
  }
  else
  {
    trace ("OCR is on ASM.");

    # for 12.1 and onward
    if ($ocr_loc =~ /\+([^\/]+)\//) 
    {
      $DGname = $1; 
    }
    # for pre 12.1
    elsif ($ocr_loc =~ /\+(.+)/) 
    {
      $DGname = $1; 
    }
    else
    {
      trace("Fail to get OCR location from /etc/oracle/ocr.loc file");
      return (0, "");
    }
    trace ("The DG containing OCR is $DGname.");
    return (2, $DGname);
  }
}

# Upgrade action for SIHA
sub upgradeSIHA
{
  my $self                = shift;

  trace("configure OCR for SIHA upgrade");
  configureOCR_siha() || die(dieformat(259));

  return SUCCESS;
}

sub configureOcrBackup4upg
{
  my $success = SUCCESS;
  my $ocrconfig = catfile($CFG->ORA_CRS_HOME, 'bin', 'ocrconfig');
  my $cmd = "$ocrconfig -showbackuploc"; 
  my ($rc, @output) = system_cmd_capture($cmd);
 
  if ( $rc == 0 )
  { 
    if ((scalar(grep(/\+/, @output)) > 0) && ($output[0] =~ /\[(.*?)\]/))
    {
      trace("OCR backup location is already set to DG $1");
      return $success;
    }
    else
    {
      my @backuploc = discoverOCRDiskgroups();
      $cmd = "$ocrconfig -backuploc +$backuploc[0] -rscupg";
      ($rc, @output) = system_cmd_capture($cmd);
      if (0 != $rc)
      {
        print_lines(@output);
        die(dieformat(606, $backuploc[0]));
        $success = FAILED;
      }
      else {
      trace("OCR backup location is succesfully set to  +$backuploc[0]");
      }
      return $success;
    }
  }
  else 
  {
    print_lines(@output);
    $success = FAILED;
    return $success;
  } 
}

sub configureOcrBackup
{
  my $backuploc;

  if ($CFG->params('CDATA_BACKUP_DISK_GROUP'))
  {
    $backuploc = $CFG->params('CDATA_BACKUP_DISK_GROUP');    
  }
  else
  {
    $backuploc = $CFG->params('CDATA_DISK_GROUP');
  }

  my $ocrconfig = catfile($CFG->ORA_CRS_HOME, 'bin', 'ocrconfig');
  my $cmd = "$ocrconfig -backuploc +$backuploc -rscupg";
  my $success = SUCCESS;
  my ($rc, @output) = system_cmd_capture($cmd);

  if (0 != $rc)
  {
    print_lines(@output);
    die(dieformat(606, $backuploc));
    $success = FAILED;
  }
  else {
   trace("OCR backup location is succesfully set to +$backuploc");
  }
  return $success;
}

sub copyOcrBackupToSharedStorage	
#------------------------------------------------------------------------------	
# Function : This function supports copying the OCR backup to shared 
#            storage(DG/NAS).	
# Returns :  TRUE if success	
#            FALSE if failed	
#------------------------------------------------------------------------------	
{	
  my $self = shift;
  
  # get old CRS version
  my @oldCrsVer = @{$CFG->oldconfig('ORA_CRS_VERSION')};
  my $crsversion = join('.', @oldCrsVer);
	
  my $clustername = $CFG->params('CLUSTER_NAME');
  my $oraowner = $CFG->params('ORACLE_OWNER');
  my $oraDbaGrp = $CFG->params("ORA_DBA_GROUP");
	
  my $localNode = tolower_host();
	
  # get the OCR location
  my @output = getOCRLoc();
  my $ocrType = shift @output;
  my $value = shift @output;

  my $ocrback_ckpt = "ROOTCRS_OCRBACKUP";
  my $ckpt_nodeName = "MANUAL_BACKUP_NODE_NAME";

  # Only the node containing OCR manual backup file can copy the backup file	
  # to shared storage.	
  if (isCkptPropertyExists($ocrback_ckpt, $ckpt_nodeName, "-global"))	
  {
    my $ocrBackupNodeName = trim(getCkptPropertyValue($ocrback_ckpt,
                            $ckpt_nodeName, "-global"));	
	
    if ($localNode eq $ocrBackupNodeName)	
    {	
      trace("Current node: $localNode contains the OCR manual backup file.");	

      # check the checkpoint property OCR_BACKUP_ON_SHARED_STORAGE,
      # if it's TRUE, the OCR backup has been copied to shared stroage,
      # there's no need to copy again.
      if (isCkptPropertyExists($ocrback_ckpt,"OCR_BACKUP_ON_SHARED_STORAGE","-global")
        && (getCkptPropertyValue($ocrback_ckpt,
                          "OCR_BACKUP_ON_SHARED_STORAGE", "-global") eq "TRUE"))
      {
        trace("OCR backup file has already been copied to shared storage.");
        return TRUE;
      }

      my $ocrKey_fileName = "MANUAL_BACKUP_FILE_NAME";	
      my $ocrBackupFileName = getCkptPropertyValue($ocrback_ckpt,	
      $ocrKey_fileName, "-global");	

      my $oldCrsHome = $CFG->OLD_CRS_HOME;	
 
      if ($ocrType == 2)	
      {	
        trace("OCR is on ASM");	
        trace("Start to copy OCR manual backup to DG...");	
        my $filenameOnDG = "$clustername"."_backup"."$crsversion".".ocr";	
        $filenameOnDG = lc($filenameOnDG);	
        $value = "+" . $value . ":";	
        $filenameOnDG = catfile ($value, $filenameOnDG);	
        my $ocrConfig = catfile($CFG->ORA_CRS_HOME, 'bin', 'ocrconfig');	
        my $cmd = "$ocrConfig -copy $ocrBackupFileName $filenameOnDG";	
    
        trace("Start to copy OCR backup file to DG: $value.");	
        my ($copyStatus, @copyOutput) = system_cmd_capture($cmd);	
        if ($copyStatus == 0)	
        {	
          trace ("Successfully copied OCR manual backup file from "
                ."$ocrBackupFileName to DG: $filenameOnDG.");	
        }	
        else	
        {	
          trace ("Failed to copy OCR manual backup file from "
                  ."$ocrBackupFileName to DG: $filenameOnDG.");
          trace ("Copy OCR manual backup to Shared Storage Fail!.");	
          die(dieformat(537, $localNode));	
        }	
      }	
      else	
      {	
        trace ("Copy OCR manual backup to shared storage failed!");	
        return FALSE;	
      }
      writeCkptProperty($ocrback_ckpt, "OCR_BACKUP_ON_SHARED_STORAGE",
                      "TRUE", "-global");
      writeGlobalCkpt($ocrback_ckpt, CKPTSUC, "-transferfile"); 	
    }
  }
  return TRUE;	
}

sub getOcrbackupFileNameOnDG
#------------------------------------------------------------------------------
# Function : Get the OCR backup file name on DG.
# Returns : The OCR backup file name if sucess,
#           The script would exit if it failed to get the OCR backup file name.
#------------------------------------------------------------------------------
{
  my $self = shift;
  my $ocrback_ckpt = "ROOTCRS_OCRBACKUP";
  my $omfname_prpty = "MANUAL_BACKUP_FILE_NAME";
  my $crshome = getCrsHome();

  if (isCkptPropertyExists($ocrback_ckpt, $omfname_prpty, "-global"))
  {
    my $ocrBackupName = trim(getCkptPropertyValue($ocrback_ckpt,
                             $omfname_prpty, "-global"));
    return $ocrBackupName;
  }
  else
  {
    # Somehow the global CKPT was corrupt causing failed to retrieve the
    # OCR backup file name on DG.
     trace("Start to get the OMF name of the OCR backup file on DG.");
    my @oldCrsVer = @{$CFG->oldconfig('ORA_CRS_VERSION')};
    my $crsversion = join('.', @oldCrsVer);
    my $clustername = $CFG->params('CLUSTER_NAME');
    my $ocrBackupFileName = "$clustername"."_backup"."$crsversion".".ocr";
    $ocrBackupFileName = lc($ocrBackupFileName);

    my ($rc_OMF, @output_OMF) =
       $self->getOcrBackupOMF($ocrBackupFileName, $crshome);
    my $count = scalar(grep(/$ocrBackupFileName/, @output_OMF));

    if (($rc_OMF == 0) && ($count == 1))
    {
      my $ocrBackupOMFName = join("", @output_OMF);
      $ocrBackupOMFName = trim($ocrBackupOMFName);
      trace("The OMF name of the OCR backup file on DG is: $ocrBackupOMFName");

      # putting a ':' in the OMF name between the DG name and the path
      # of the OCR backup file in DG
      my $ocrBackupOMFName1;
      my @elems = split(/\//, $ocrBackupOMFName);
      $elems[0] = $elems[0].":";
      $ocrBackupOMFName1 = join("/",@elems);
      trace("Re-format the OMF name as: $ocrBackupOMFName1");

      # Store the OCR backup file name on DG into global ckpt.
      writeCkptProperty($ocrback_ckpt, $omfname_prpty,
                        $ocrBackupOMFName1, "-global", "-transferfile");
      return $ocrBackupOMFName1;
    }
    else
    {
      trace("Failed to get the OMF name of the OCR backup file on DG.");
      ($rc_OMF == 0) ? die(dieformat(540, $ocrBackupFileName))
                     : die(dieformat(538));
    }
  }
}

1;