# 
# $Header: has/install/crsconfig/crsdowngrade.pm /st_has_12.2.0.1.0ocwpsu/1 2017/07/24 02:58:28 muhe Exp $
#
# crsdowngrade.pm
# 
# Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
#
#    NAME
#      crsdowngrade.pm
#
#    DESCRIPTION
#      Root downgrade script for Oracle Clusterware.
#
#    MODIFIED   (MM/DD/YY)
#    muhe        07/20/17 - Backport muhe_bug-25301463 from main
#    muhe        01/18/17 - Fix bug 25301463
#    luoli       07/05/16 - Fix bug 23740751
#    luoli       04/19/16 - Fix bug 23034456
#    luoli       04/14/16 - Replaced message 550 with 119 in
#                           checkRemoteStacks()
#    bbeelamk    04/13/16 - Fix bug 23095140
#    xyuan       04/07/16 - Fix bug 23016188
#    xyuan       03/29/16 - Fix bug 23014561
#    luoli       03/22/16 - Fix bug 22978901
#    agraves     03/16/16 - Set ORA_CRS_HOME before running acfsroot. This is
#                           so we pick up libraries from the old home as well.
#    luoli       03/15/16 - Fix bug 22894620
#    luoli       03/01/16 - Fix bug 22825685
#    luoli       02/04/16 - Fix RTI 18660895
#    bbeelamk    01/18/16 - Fix bug 22545495
#    muhe        12/22/15 - Fix bug 22453926
#    luoli       12/21/15 - Fix getOCRLoc undefined issue
#    muhe        12/17/15 - Fix bug 22383268
#    luoli       12/16/15 - Fix bug 22367681
#    xyuan       12/03/15 - Fix bug 22283181
#    bbeelamk    11/25/15 - Fix bug 22035716
#    luoli       11/23/15 - Fix bug 21971395
#    muhe        11/08/15 - Fix bug 22148653, 22162692
#    luoli       09/22/15 - Fix bug 21776812
#    muhe        09/17/15 - Fix bug 21823264
#    xyuan       09/05/15 - Fix bug 21470281
#    madoming    08/11/15 - Fix bug 21572384
#    luoli       08/11/15 - Fix bug 21564538
#    luoli       07/23/15 - Fix bug 21324323
#    bbeelamk    07/10/15 - Fix typo
#    shullur     06/21/15 - For migrating CHM modules to new root script
#                           changes
#    luoli       06/09/15 - Fix bug 21213444
#    luoli       05/26/15 - Fix bug 21133828
#    luoli       05/25/15 - Fix bug 20978142
#    muhe        04/21/15 - Fix bug 20916630
#    luoli       04/06/15 - Fix bug 20811547
#    muhe        03/20/15 - Fix bug 20726318
#    madoming    03/16/15 - Changes for new framework
#    luoli       03/11/15 - Fix bug 20607402
#    luoli       02/12/15 - Fix bug 20419288
#    luoli       02/10/15 - Fix bug 19232406
#    sbezawad    02/11/15 - Bug 20019354: Migrate OCR and OLR to new framework
#    jmarcias    02/09/15 - Remove global checkpoint file only on lastnode
#    luoli       01/12/15 - Separate downgrade/deconfig flow
#    luoli       01/12/15 - Creation
#
package crsdowngrade;

use strict;
use English;
use File::Copy;
use File::Path;
use File::Spec::Functions;
use Cwd;

# root scripts modules
use crsutils;
use crsgpnp;
use oracss;
use oraacfs;
use crska;
use oraafd;
use s_crsutils;
use crstfa;
use oraocr;
use oraolr;
use oraasm;
use oraohasd;
use orachm;

sub new {
   shift;
   crsutils->new(@_);

   $CFG->compACFS(oraClusterwareComp::oraacfs->new("ACFS"));
   ($CFG->compACFS)->checkPath();

   rscPreChecks();

   if (!$CFG->SIHA)
   {
     CRSDowngrade();
   }
}


my @CRS_DOWNGRADE_STAGES =
(
  {"name" => "DowngradeValidate",       "checkpoint" => "null",              "sub" => \&downgrade_validate},      # check if the node has been downgraded already
                                                                                                                  # check if the 'lastnode' 'online' option is given correctly
                                                                                                                  # get old crs home/version
                                                                                                                  # check if ASM version compatible
  {"name" => "DiscoverVF",              "checkpoint" => "null",              "sub" => \&discover_VF},             # for non-lastnode: no-op
                                                                                                                  # for lastnode: stop full stack & start CRS in exclusive mode
                                                                                                                  # for lastnode: discover Voting files
  {"name" => "DowngradeASM",            "checkpoint" => "null",              "sub" => \&downgrade_ASM},           # stop full stack
                                                                                                                  # for lastnode: downgrade ASM
                                                                                                                  # subroutine moved to oraasm.pm
  {"name" => "DropMgmtDB",              "checkpoint" => "null",              "sub" => \&drop_MgmtDB},             # for non-lastnode: no-op
                                                                                                                  # for lastnode: drop MgmtDB
  {"name" => "DeleteVF",                "checkpoint" => "null",              "sub" => \&delete_VF},               # for non-lastnode: no-op
                                                                                                                  # for lastnode: start CRS in exclusive mode from new home
                                                                                                                  # for lastnode: delete Voting Files
  {"name" => "PrepareToDowngradeOCR",   "checkpoint" => "null",              "sub" => \&prepare_to_downgrade_OCR},# for non-lastnode: no-op
                                                                                                                  # for lastnode: get OCR backup file name
                                                                                                                  # for lastnode: remove OCR
                                                                                                                  # for lastnode: stop full stack
                                                                                                                  # subroutine moved to oraocr.pm
  {"name" => "DeinstallACFS",           "checkpoint" => "null",              "sub" => \&deinstall_ACFS},          # deinstall ACFS
  {"name" => "RemoveAFD",               "checkpoint" => "null",              "sub" => \&remove_AFD},              # remove AFD
  {"name" => "DowngradeOLR",            "checkpoint" => "null",              "sub" => \&downgrade_OLR},           # create old olr.loc file  
                                                                                                                  # subroutine moved to oraolr.pm
  {"name" => "DeleteOHASD",             "checkpoint" => "null",              "sub" => \&delete_OHASD},            # delete OHASD
  {"name" => "DowngradeCHM",             "checkpoint" => "null",              "sub" => \&downgrade_CHM},            # downgrade CHM
  {"name" => "DowngradeOCR",            "checkpoint" => "null",              "sub" => \&downgrade_OCR},           # restore ocr.loc
                                                                                                                  # for lastnode: restore OCR
                                                                                                                  # for lastnode: start cluster in exclusive mode 
                                                                                                                  # without starting CRS from old home
                                                                                                                  # subroutine moved to oraocr.pm
  {"name" => "AddVF",                   "checkpoint" => "null",              "sub" => \&add_VF},                  # for non-lastnode: no-op
                                                                                                                  # for lastnode: start CRSD
                                                                                                                  # for lastnode: stop old stack
  {"name" => "ReinstallACFS",           "checkpoint" => "null",              "sub" => \&reinstall_ACFS},          # reinstall ACFS
  {"name" => "ReinstallAFD",            "checkpoint" => "null",              "sub" => \&reinstall_AFD},           # reinstall AFD
  {"name" => "PostTasks",               "checkpoint" => "null",              "sub" => \&post_tasks},              # restore init scripts, downgrade Oratab, remove ckpts,
                                                                                                                  # downgrade TFA
);


sub CRSDowngrade
{
  my $count = 0;

  trace ("Downgrading Oracle Clusterware on this node");

  foreach my $stage (@CRS_DOWNGRADE_STAGES)
  {
    my $name = $stage->{"name"};
    my $checkpoint = $stage->{"checkpoint"};
    my $func = $stage->{"sub"};

    trace("Executing the [$name] step with checkpoint [$checkpoint] ...");
    &$func();
    $count++;
  }

  if ($count == scalar(@CRS_DOWNGRADE_STAGES))
  {
    trace ("Successfully downgraded Oracle Clusterware stack on this node");

    set_bold();
    print_info(591);
    my $crshome = $CFG->OLD_CRS_HOME;
    if ($CFG->LASTNODE)
    {
      my $lastnode_status = getCkptStatus("ROOTCRS_LASTNODE", "-global");
      trace("Global ckpt 'ROOTCRS_LASTNODE' state: $lastnode_status");
      if ($lastnode_status eq CKPTSUC)
      {
        print_info(640, $crshome);
      }
      remove_global_checkpoints(); 
      print_info(592, $crshome);
    }
   
    # global CKPT also need to be removed for online downgrade 
    if ($CFG->ONLINE && isLastNodeToDowngrade())
    {
      remove_global_checkpoints();
      print_info(592, $crshome);
    }

    reset_bold();

    exit(0);
  }
  else
  {
    trace ("Failed to downgrade Oracle Clusterware stack on this node");
    set_bold();
    print_error(593);
    reset_bold();
    exit(1);
  }
}


sub downgrade_validate
{
   if (isSIHA())
   {
     die(dieformat(457));
   }

   if ($CFG->platform_family eq "unix")
   {
     my $OLRCONFIG = $CFG->params('OLRCONFIG');
     if (!(-r $OLRCONFIG))
     {
       # olr.loc should be checked before it's referred in s_get_olr_file().
       trace ("Either " . $OLRCONFIG . " does not exist or is not readable");
       trace ("Make sure the file exists and it has read and execute access");
       die(dieformat(103, $OLRCONFIG));
     }
   }

   if ((! isInstallNode()) && (! isUpgradedNode()) &&
        (! isFirstNodeUpgradeSuc()))
   {
     trace("The OUI node has not been upgraded and hence ".
           "downgrading the local node is not needed.");
     die(dieformat(613));
   }

   if (isNodeDowngraded())
   {
     my $localNodeName = tolower_host();
     print_info(548, $localNodeName);
     set_bold();
     print_info(335);
     reset_bold();
     exit(0);
   }

   if (! isUpgradedNode())
   {
     trace("The OLR has not been upgraded");
     $CFG->olrNotUpgraded(TRUE);
   }
   else
   {
     $CFG->olrNotUpgraded(FALSE);
   }

   TraceOptions();

   # get old crs home and old crs version
   getOldConfig();

   # check whether the '-online' is provided correctly and whether the 
   # last-node operations need to be performed
   lastnodeCheck();

   # Set the node attribute for downgrade based on the determination
   # if the current node is the last node to downgrade or not
   ($CFG->LASTNODE) ?
    $CFG->nodeAttributeDowngrade(LAST_NODE_TO_DOWNGRADE):
    $CFG->nodeAttributeDowngrade(NONLAST_NODE_TO_DOWNGRADE);

   $CFG->compOLR(oraClusterwareComp::oraolr->new("OLR"));
   $CFG->compOCR(oraClusterwareComp::oraocr->new("OCR"));
   $CFG->compCHM(oraClusterwareComp::orachm->new("CHM"));
   $CFG->compASM(oraClusterwareComp::oraasm->new("ASM"));
   $CFG->compOHASD(oraClusterwareComp::oraohasd->new("OHASD"));

   # If user downgrades lastnode with force option, root script assumes the OCR 
   # backup file is already on shared storage.
   checkIfOCRonSharedStorage() if ($CFG->LASTNODE && !$CFG->FORCE);

   verify_gpnp_dirs($CFG->ORA_CRS_HOME,
                    $CFG->params('GPNPGCONFIGDIR'),
                    $CFG->params('GPNPCONFIGDIR'),
                    $CFG->HOST,
                    $CFG->params('ORACLE_OWNER'),
                    $CFG->params('ORA_DBA_GROUP'));

   # validate ASM
   if((isOCRonASM()) &&
       (GI_STACK_UP == checkGIStack(s_get_olr_file("crs_home"))) &&
       (! isASMVersionCompatible()))
   {
     die(dieformat(487));
   }
}

sub discover_VF
{
  if ($CFG->LASTNODE)
  {
    my %ref;
    my @whereVoteDisks;
    my $useASM = discoverVotingFiles(\@whereVoteDisks);

    # check if the GI stack is down on all remote nodes. After this function 
    # returns, the stacks is running in CRS exclusive mode.
    checkRemoteStacks();

    if ($useASM)
    {
      trace("Downgrade with voting files on ASM");
      if (scalar(@whereVoteDisks) <= 0)
      {
        trace("No diskgroups to store voting file(s) found");
        die(dieformat(545));
      }

      %ref =
      (
        USE_ASM       => 1,
        RM_VOTEDISKS  => \&removeVotingDisk,
        ADD_VOTEDISKS => \&addVotingDisks,
        VOTEDISKS     => $whereVoteDisks[0],
      );
    }
    else
    {
      trace("Downgrade with voting files on filesystem");
      if (scalar(@whereVoteDisks) <= 0)
      {
        trace("No voting file(s) found");
        die(dieformat(545));
      }

       %ref =
      (
        USE_ASM       => 0,
        RM_VOTEDISKS  => \&removeVotingfiles,
        ADD_VOTEDISKS => \&addVotingFiles,
        VOTEDISKS     => \@whereVoteDisks,
      );
    }

    $CFG->dwn_voting_file(\%ref);
  }
}


sub downgrade_ASM
{
  if (isCkptexist("ROOTCRS_UPGRADEASM") &&
         isCkptSuccess("ROOTCRS_UPGRADEASM"))
  {
    my $oraasm = $CFG->compASM;
    $oraasm->downgradeCurrentNode($oraasm->DOWNGRADE_ASM);
  }

  trace("Stop the current CRS stack");
  stopFullStack("force", s_get_olr_file("crs_home")) || die(dieformat(349));
}


sub drop_MgmtDB
{
  if ($CFG->LASTNODE && (isOldVersionLT121() || isOldVersion121()))
  {
    my $lastnode_status = getCkptStatus("ROOTCRS_LASTNODE", "-global");
    trace("Global ckpt 'ROOTCRS_LASTNODE' state: $lastnode_status");
    if ($lastnode_status eq CKPTSUC)
    {
      my $savedPfile =
           catfile($CFG->ORA_CRS_HOME, 'dbs', 'init-dropmgmtdbSAVED.ora');
      trace("remove the saved pfile $savedPfile for mgmtdb");
      s_remove_file($savedPfile);

      # Drop MGMTDB before restoring OLR and OCR downgrade
      dropMgmtdbDowngrade();
    }
    else
    {
      trace("Last node operations have not completed, ".
            "hence not dropping the mgmtdb.");
    }
  }
}


sub delete_VF
{
  if ($CFG->LASTNODE)
  {
    my %ref = %{$CFG->dwn_voting_file};
    my $USE_ASM = $ref{USE_ASM};

    # delete voting disks
    trace("Start CRS in exclusive mode from active GI home");
    start_excl_crs(s_get_olr_file("crs_home")) || die(dieformat(260));

    trace("Delete voting disks. Voting disks on ASM: $USE_ASM");
    my $rm_func = $ref{RM_VOTEDISKS};
    if (FAILED == &$rm_func(s_get_olr_file("crs_home"), $ref{VOTEDISKS}))
    {
      die(dieformat(415));
    }
  }
}


sub prepare_to_downgrade_OCR
{
  if ($CFG->LASTNODE)
  {
    my $oraocr = $CFG->compOCR;
    $oraocr->downgradeLastNode($oraocr->GET_OCR_BACKUP_FILE);

    stopFullStack("force", s_get_olr_file("crs_home")) || die(dieformat(349));
  }
}


sub deinstall_ACFS
{
  if ($CFG->olrNotUpgraded())
  {
    trace("Skipping ACFS deinstall");
    return;
  }

  ($CFG->compACFS)->downgradeCurrentNode();
}


sub remove_AFD
{
  if ($CFG->olrNotUpgraded())
  {
    trace("Skipping AFD deinstall");
    return;
  }

  my $afdSupported = isAFDSupported();
  $CFG->AFDSupported($afdSupported);
  # AFD can be installed after GI install.
  # Remove AFD if installed.
  if (isAFDInstalled()) {
    rm_afd_conf();
    s_rm_afdinit_rclevel();
    s_rm_afdinit_init();
    removeAFDRoot(USECHKPOINTS);
  }
}


sub downgrade_OLR
{
  if ($CFG->olrNotUpgraded())
  {
    trace("Skipping OLR restoration");
    return;
  }

  my $oraolr = $CFG->compOLR;
  $oraolr->downgradeCurrentNode();
}


sub delete_OHASD
{
  if ($CFG->olrNotUpgraded()) 
  {
    trace("Skipping the deletion of OHASD");
    return;
  }

  my $oraohasd = $CFG->compOHASD;
  $oraohasd->downgradeCurrentNode();
}

sub downgrade_CHM
{
  if ($CFG->olrNotUpgraded()) 
  {
    trace("Skipping CHM downgrade");
    return;
  }

  my $orachm = $CFG->compCHM;
  $orachm->downgradeCurrentNode();
}

sub downgrade_OCR
{
  my $oraocr = $CFG->compOCR;

  if ($CFG->LASTNODE)
  {
    # Start old stack in exclusive mode without CRSD running
    if(! startExclNoCRS($CFG->OLD_CRS_HOME)) { die(dieformat(260)); }

    # unset ORACLE_HOME environment variable since this var when set to 
    # current home location causes issues with OCR downgrade.
    # Bug# 16467434
    
    $ENV{'ORACLE_HOME'}  = "";

    $oraocr->downgradeCurrentNode($oraocr->DOWNGRADE_OCR);
  }
  else
  {
    $oraocr->downgradeCurrentNode($oraocr->RESTORE_OCR_LOC);
  }
}


sub add_VF
{
  if ($CFG->LASTNODE)
  {
    my %ref = %{$CFG->dwn_voting_file};
    my $USE_ASM = $ref{USE_ASM};

    # Start CRSD
    trace("CRSD needs to be up for adding vote disks back");
    start_crsd_and_check($CFG->OLD_CRS_HOME) || die(dieformat(249));

    # Add Voting disks back
    trace("Add voting disks. Voting disks on ASM: $USE_ASM");
    if (1 == $USE_ASM)
    {
      trace("Attempting to start ora.asm ...");
      start_resource("ora.asm", "-init") || die(dieformat(247));
    }

    my $add_func = $ref{ADD_VOTEDISKS};
    if (FAILED == &$add_func($CFG->OLD_CRS_HOME, $ref{VOTEDISKS}))
    {
      die(dieformat(261));
    }

    trace("Stopping the old stack");
    stopClusterware($CFG->OLD_CRS_HOME, "crs") || die(dieformat(349));
  }
}


sub reinstall_ACFS
{
  if ($CFG->olrNotUpgraded()) 
  {
    trace("Skipping ACFS reinstall");
    return;
  }

  if ($CFG->ACFSSupported)
  {
    if (WARNING == reinstallACFSRoot())
    {
      print_error(453);
    }
  }
}


sub reinstall_AFD
{
  if ($CFG->olrNotUpgraded()) 
  {
    trace("Skipping AFD reinstall");
    return;
  }

  if ($CFG->AFDSupported)
  {
    if (WARNING == reinstallAFDRoot())
    {
      print_error(453);
    }
  }
}

sub isUpgradedNode
{
   my $configuredHome = s_get_olr_file ("crs_home");
   my $oracleHome = $CFG->ORA_CRS_HOME;
   if ($configuredHome ne $oracleHome) {
     trace("The node has not been upgraded.");
     return FALSE;
   }
   else {
     trace("The node has been upgraded.");
     return TRUE;
   }
}

sub isNodeDowngraded
{
  if ((! isCkptFileExists()) && (! isUpgradedNode()))
  {
    # The checkpoint file is removed after downgrade.
    trace("The current node has already been downgraded");
    return TRUE;
  }
  else
  {
    trace("The current node has not been downgraded");
    return FALSE;
  }
}

sub getOldConfig
{
  my $ckptName = "ROOTCRS_OLDHOMEINFO";
  my $ckptStatus;

  if (isCkptexist($ckptName, "-global"))
  {
    $ckptStatus = getCkptStatus($ckptName, "-global");
  }
  else
  {
    trace("The global checkpoint ROOTCRS_OLDHOMEINFO doesn't exist");
    die(dieformat(416));
  }

  if (($ckptStatus eq CKPTSUC) || is_dev_env())
  {
    my $oldHome;
    my $oldVersion;

    # Get old CRS home
    $oldHome = getCkptPropertyValue($ckptName, "OLD_CRS_HOME", "-global");
    trace("Old CRS home from ckpt property is [$oldHome]");
    $oldHome = trim($oldHome);

    if ($oldHome)
    {
      $CFG->oldconfig('ORA_CRS_HOME', $oldHome);
      $CFG->oldcrshome($oldHome);
    }
    else
    {
      trace("Failed to retrieve old Grid Infrastructure home location");
      die(dieformat(416));
    }

    # Get old CRS version
    $oldVersion = getCkptPropertyValue($ckptName, "OLD_CRS_VERSION", "-global");
    trace("Old crs version from ckpt property is [$oldVersion]");
    my @oldVer = split(/\./, trim($oldVersion));
    $CFG->oldconfig('ORA_CRS_VERSION', \@oldVer);
    $CFG->oldcrsver(trim($oldVersion));
  }
  else
  {
    trace("The global checkpoint status of ROOTCRS_OLDHOMEINFO is $ckptStatus");
    die(dieformat(416));
  }
}

sub reinstallACFSRoot
{  
  my $acfsroot;
  my $ret = SUCCESS;
  my $unset_home = $ENV{'ORACLE_HOME'};
   
  trace("Re-installing ACFS drivers from older home ...");
  if ($CFG->platform_family eq 'windows')
  {
    $acfsroot = catfile($CFG->OLD_CRS_HOME, 'bin', 'acfsroot.bat');
  }
  else
  {
    $acfsroot = catfile($CFG->OLD_CRS_HOME, 'bin', 'acfsroot');
  }  

  if (-e $acfsroot)
  {
    my $cmd = "$acfsroot install";
    if (! isOldVersionLT11204())
    {
      $cmd .= " -t2";
    }

    #  Set ORACLE_HOME, This value will be unset later.
    $ENV{'ORACLE_HOME'} = $CFG->OLD_CRS_HOME;
    $ENV{'ORA_CRS_HOME'} = $CFG->OLD_CRS_HOME;

    trace("Executing '$cmd'");
    my @output = system_cmd_capture($cmd);
    my $status = shift(@output);

    # Unset
    $ENV{'ORACLE_HOME'} = $unset_home;

    if (0 == $status)
    {
      trace("Successfully execute 'acfsroot install' from older home");
      $ret = SUCCESS;
    }
    elsif (2 != $status)
    {
      if (scalar(grep(/09394/, @output)) > 0)
      {
        trace("'acfsroot install' succeeded, but need a system reboot");
        $ret = WARNING;
      }
      else
      {
        trace("Failed to execute 'acfsroot install' from older home");
        print_error(196);
        return FAILED;
      }
    }
  }

  return $ret;
}


sub reinstallAFDRoot
{
  my $ret = SUCCESS;

  #
  # TODO
  #
  # For post 12.1.0.2.0, afd.conf, AFD init scripts need to be created
  # for lower home if AFD is supported.
  #
  trace("AFD is not reinstalled since AFD does not exist in older home.");

  return $ret;
}


sub post_tasks 
{   
  trace("Restore init files");
  s_restoreInitScripts($CFG->OLD_CRS_HOME);
      
  downgradeOratab();

  restoreGPNPProfile();
 
  remove_checkpoints();
  if ($CFG->LASTNODE || $CFG->ONLINE)
  {
    remove_config_params_file();
  }
      
  # Only way to downgrade TFA is to remove from the current home and set up again 
  # from the oldhome

  # Uninstall TFA
  remove_tfa();
  # Setup TFA again from old home
  setup_tfa("downgrade");
} 

#-------------------------------------------------------------------------------
# Function: Scan /etc/oratab and remove all entries that have higher version
#           of CRS home
# Args    : 0
#-------------------------------------------------------------------------------
sub downgradeOratab
{
  my $crshome = $CFG->ORA_CRS_HOME;
  my $host    = $CFG->HOST;

  if ($CFG->platform_family ne 'unix') { return; }
  my $oratab = catfile("/etc", "oratab");
  if (! -e $oratab)
  {
    trace("The $oratab file doesn't exist");
    return;
  }

  trace("Remove all new version related stuff from $oratab");
  my $oratabNew = catfile("/etc", "oratab.new.$host");

  my @oratabLines = read_file($oratab);
  open(ORATABNEW, ">${oratabNew}") or die(dieformat(207, $oratabNew, $!));
  my $changed = 0;
  foreach my $line (@oratabLines)
  {
    if ($line =~ /${crshome}/)
    {
      $changed = 1;
      next;
    }
    print ORATABNEW "$line";
  }
  close(ORATABNEW);

  if ($changed)
  {
    trace("Copying file $oratabNew to $oratab");
    copy_file($oratabNew, $oratab);
  }

  trace("Removing file $oratabNew");
  s_remove_file($oratabNew);
}


sub isLastNodeToDowngrade
{
  my $localNode = tolower_host();

  # Leaf node can never be the last node to downgrade.
  if (! isOldVersionLT121())
  {
    # The query for node role may fail if the node role is not set
    # yet in OLR of the new home
    my $node_role = getNodeConfigRole();
    $node_role = getNodeConfigRole($localNode, $CFG->oldcrshome)
                                                 unless (defined $node_role);
    if(NODE_ROLE_RIM eq $node_role)
    {
      trace("Current node is a leaf node, which cannot be the last node to ".
            "downgrade.");
      return FALSE;
    }
  }

  # No matter OCR is on ASM or NAS, all other nodes need to be downgraded
  # before considering the current node as the lastnode to downgrade.
  
  # Get the active node list in the cluster
  my @nodeList = split(/,/, $CFG->params('NODE_NAME_LIST'));
  my $remoteCrsHome;

  foreach my $node (@nodeList)
  {
    if (($localNode ne $node) && isNodeAlive($node))
    {
      $remoteCrsHome = s_getRemoteCrsHome($node);
      trace("CRS home of remote node $node is $remoteCrsHome");
      if ($remoteCrsHome eq $CFG->ORA_CRS_HOME)
      {
        trace("CRS home of remote node $node is empty or equal to the CRS ".
              "home of current node $localNode: ". $CFG->ORA_CRS_HOME);
        trace("$node has not been downgraded.");
        trace("Current node: $localNode should not be the last node to ".
              "downgrade");
        print_info(521,$node) if ($CFG->LASTNODE);
        return FALSE;
      }
      else
      {
        trace("CRS home of remote node $node is not equal to the CRS ".
              "home of current node $localNode: ". $CFG->ORA_CRS_HOME);
        trace("$node has been downgraded.");
      }
    }
  }
  trace("Current node: $localNode should be the last node to downgrade.");
  return TRUE;
}


sub isASMVersionCompatible
{
  my @cmd;
  my @out;
  my $asmrc = 1;
  my $compatibleVersion;

  my ($rc, $asm_mode);
  trace("Try to read ASM mode from the global stage profile");
  ($rc, $asm_mode) = gpnp_get_asm_mode(get_peer_profile_file(FALSE));
  if (0 != $rc)
  {
    trace("Try to read ASM mode from the node-specific profile");
    ($rc, $asm_mode) = gpnp_get_asm_mode(get_peer_profile_file(TRUE));
  }

  if (0 != $rc)
  {
    trace("Unable to get ASM mode from Oracle Clusterware GPnP profile");
    return FALSE;
  }

  trace("ASM mode = $asm_mode");
  setOraHomeSID($asm_mode);

  my @oldCrsVer = @{$CFG->oldconfig('ORA_CRS_VERSION')};
  my $oldVersion = join('.', @oldCrsVer);

  my $asmcmd =  catfile($CFG->ORA_CRS_HOME, 'bin', 'asmcmd');

  my $crshome;
  my $rc_kfod = 1;
  my @diskgroups;
  my $asm_upgrade_succeeded = isCkptSuccess("ROOTCRS_UPGRADEASM");
  if ($asm_upgrade_succeeded)
  {
    $crshome = $CFG->ORA_CRS_HOME;
    ($rc_kfod, @diskgroups) = kfodListDiskgroups($crshome);
  }

  if ($rc_kfod != 0)
  {
    trace("Calling 'kfod' from the lower version GI home");
    $crshome = $CFG->OLD_CRS_HOME;
    ($rc_kfod, @diskgroups) = kfodListDiskgroups($crshome); 
  }

  if ($rc_kfod != 0)
  {
    trace("Failed to get disk groups.");
    return FALSE;
  }
  
  if (scalar(@diskgroups) <= 0)
  {
    trace("No vote disk or OCR diskgroup found");
    return FALSE;
  }

  foreach my $diskgroup (@diskgroups)
  {
    if ($asm_upgrade_succeeded)
    {
      @cmd = ($asmcmd, "lsattr", "-G", $diskgroup, "-l", "compatible.asm");
      
      # Use backticks instead of open() to execute external command 'asmcmd'
      # due to bug 18493777
      @out = run_cmd_as_usr_with_backticks(\@cmd, $CFG->params('ORACLE_OWNER'));
      $asmrc = shift(@out);
    }
    
    if ($asmrc != 0)
    {
      trace("Calling 'asmca' from the lower version GI home"); 
      $asmcmd = catfile($CFG->OLD_CRS_HOME, 'bin', 'asmcmd');
      @cmd = ($asmcmd, "lsattr", "-G", $diskgroup, "-l", "compatible.asm");
      @out = run_cmd_as_usr_with_backticks(\@cmd, $CFG->params('ORACLE_OWNER'));
      $asmrc = shift(@out);
    }

    if ($asmrc == 0)
    {
      foreach my $line (@out)
      {
        chomp($line);
        if($line =~ /compatible\.asm/)
        {
          my @word = split(/\s+/, $line);
          $compatibleVersion = $word[1];
          last;
        }
      }
    }

    trace("Old version: $oldVersion; " .
          "ASM compatibility version: $compatibleVersion");

    if (! $compatibleVersion)
    {
      trace("Unable to get ASM compatibility version");
      return FALSE;
    }

    if (versionComparison($oldVersion, $compatibleVersion) == -1)
    {
      trace("The ASM compatibility version of diskgroup '$diskgroup' " .
            "has been advanced");
      return FALSE;
    }
  }

  trace("Check for ASM compatibility version passed");
  return TRUE;
}

# This subroutine will be called in DowngradeValidate stage, which is at the 
# beginning of downgrade.
#
# This subroutine is to: 
# 1) determine whether the last-node operations need to be performed based on 
#    whether all other upgraded nodes have been successfully downgraded; 
# 2) verify whether user provided the '-online' option correctly during 
#    downgrade:
#    '-online' option should be provided on all upgraded node of a partial 
#    upgraded cluster, when user wants to do an online downgrade. 
# 3) make sure leaf node will not be the last node to downgrade

sub lastnodeCheck
{
  # lastnode operations will be skipped if OCR has not been changed
  my $ckptName = "ROOTCRS_OCRSTATUS";
  if((isCkptexist($ckptName)) &&
     !(isCkptPropertyExists($ckptName, "OCR_CHANGED")))
  {
    # if upgrade is aborted on first node before OCR is changed,
    # no need to do the last-node operations during downgrade
    trace("OCR has not been changed.");
    $CFG->LASTNODE(FALSE);
    return SUCCESS;
  }

  if (! isOldVersionLT121())
  {
    # If the cluster is previously upgraded from legacy asm,
    # there's no need to check for leaf nodes.
    my $asm_mode;
    my $checkLeafNodes = FALSE;
    my $globalCkptName = "ROOTCRS_OLDHOMEINFO";
    if (isCkptexist($globalCkptName, "-global") 
        && isCkptSuccess($globalCkptName,, "-global"))
    {
      $asm_mode = trim(getCkptPropertyValue($globalCkptName, "ASM_MODE", "-global"));
      trace("lower version asm mode: $asm_mode");
      if ($asm_mode ne ASM_MODE_LEGACY)
      {
        $checkLeafNodes = TRUE;
      }
      else
      {
        $checkLeafNodes = FALSE;
      }
    }
    else
    {
      # The first node upgrade was not completed, there's no need to check leaf nodes.
      $checkLeafNodes = FALSE;
    }

    if ($checkLeafNodes)
    {
      # the following check makes sure a leaf node will not be left as the 
      # last node to downgrade
      checkLeafNodes();
    }
  }

  my $localNode = tolower_host();
  $CFG->LASTNODE(isLastNodeToDowngrade());

  my $lastnode_upgrade_status = getCkptStatus("ROOTCRS_LASTNODE", "-global");
  trace("Global ckpt 'ROOTCRS_LASTNODE' state: $lastnode_upgrade_status");
  if ($lastnode_upgrade_status ne CKPTSUC)
  {
    trace("Downgrading before the cluster is fully upgraded.");
    if (($CFG->LASTNODE) && ($CFG->ONLINE))
    {
      trace ("Doing an online downgrade");
      $CFG->ASMCADOWNGRADE(TRUE);
      # online downgrade will skip all the last-node operations except 
      # ASM downgrade
      $CFG->LASTNODE(FALSE);
    }
  }
  else # cluster is fully upgraded.
  {
    if ($CFG->ONLINE)
    {
      # should not pass '-online'
      die(dieformat(554));
    }
  }
  return SUCCESS;
}

sub checkRemoteStacks
{
  # After this function returns successfully,
  # the stack is running in CRS exclusive mode. 
  trace("Check if the stack is down on all remote nodes.");
  trace("Stop the current CRS stack");
  my $crsHome = getCrsHome();
  stopFullStack("force", $crsHome) || die(dieformat(349));
  trace("Starting CRS in exclusive mode");
  my $localNode = tolower_host();
  my $CRSCTL = crs_exec_path('crsctl', $crsHome);
  my @output = system_cmd_capture($CRSCTL, 'start', 'crs', '-excl');
  my $rc     = shift @output;

  # if the start fails, check if it is because stack is running on another node
  my @node_up = grep(/CRS\-4402/, @output);
  if (scalar(@node_up) > 0)
  {
    print_trace_lines(@node_up);
    trace("The GI stack on at least one other cluster node is still up.");
    print_error(549, $localNode);
    stopFullStack("force", $crsHome) || die(dieformat(349));
    exit(1);
  }
  elsif (0 == $rc)
  {
    trace("The GI stack of all remote nodes are down.");
    return SUCCESS;
  }
  else
  {
    print_lines(@output);
    trace("\"$CRSCTL start crs -excl\" failed with status $rc");
    trace("Local node $localNode failed to check the status of the Grid " . 
          "Infrastructure stack of other cluster nodes by starting CSS in " .
          "exclusive mode.");
    die(dieformat(119));
  }
}


sub isUpgradedNodeByCkpt
{
  return FALSE if (! isCkptFileExists());

  my $crsrelver = getcrsrelver1();
  my $ckptcrsver = getCkptPropertyValue("ROOTCRS_STACK", "VERSION");
  if (isVersionMatch($ckptcrsver, $crsrelver))
  {
    return TRUE;
  }
  else
  {
    return FALSE;
  }
}


sub dropMgmtdbDowngrade
{
  trace("Starting to drop MGMT DB.");

  my $savedPfile =
     catfile($CFG->oldcrshome, 'dbs', 'init-dropmgmtdbSAVED.ora');
  s_remove_file($savedPfile);

  my $crshome = $CFG->ORA_CRS_HOME;

  startFullStack($crshome) || die(dieformat(117));

  trace("Attempt to stop Mgmt DB before dropping it");
  my ($rc, @output);
  $rc = srvctl_capture(TRUE, \@output, "stop mgmtdb -f", $crshome);
  if(0 != $rc)
  {
    if (scalar(grep(/PRCR-1001/i, @output)) > 0)
    {
      # PRCR-1001 : Resource ora.mgmtdb does not exist 
      trace("Resource ora.mgmtdb does not exist, no need to stop it.");
    }
    elsif (2 != $rc) # rc==2 means the mgmtdb was already stopped
    {
      print_lines(@output);
      trace("\"srvctl stop mgmtdb -f\" failed with status $rc, output: @output");
      print_error(180, "srvctl stop mgmtdb -f");
    }
  }

  my $spFile = getMgmtdbSPfile($crshome);
  if ($spFile)
  {
    trace("Dropping the MGMT DB ...");
    # Get and create new pfile.
    my $pfile = createMgmtdbPfile($crshome, $spFile);

    # Drop MGMT database.
    deleteMGMTDB($crshome) || die(dieformat(503));
    trace("Dropping MGMT DB successfully.");
  }

  # Stop full stack.
  stopFullStack("force") || die(dieformat(349));
}

sub isFirstNodeUpgradeSuc
{
  my $ckpt_exist = isCkptexist("ROOTCRS_FIRSTNODE", "-global");
  if ($ckpt_exist && (getCkptStatus("ROOTCRS_FIRSTNODE", "-global") eq CKPTSUC))
  {
    trace("Firstnode has been upgraded.");
    return TRUE;    
  }
  else
  {
    trace("Firstnode has not been upgraded.");
    return FALSE;
  }
}

sub checkIfOCRonSharedStorage
{
  # check whether the OCR backup file has been sotred on shared storage
  my $ocrback_ckpt = "ROOTCRS_OCRBACKUP";
   if (isCkptPropertyExists($ocrback_ckpt,"MANUAL_BACKUP_NODE_NAME","-global")
       && (getCkptPropertyValue($ocrback_ckpt, 
                      "OCR_BACKUP_ON_SHARED_STORAGE", "-global") ne "TRUE"))
   {
     my $ocrBackupNodeName = trim(getCkptPropertyValue($ocrback_ckpt,
                            "MANUAL_BACKUP_NODE_NAME", "-global"));
     my $ocrBackupFileName = getCkptPropertyValue($ocrback_ckpt,
                            "MANUAL_BACKUP_FILE_NAME", "-global");
     trace("OCR backup file is generated on OCR master node: " .
           "$ocrBackupNodeName, but the OCR backup file has not been copied ".
           "to shared storage.");

     # get old CRS version
     my @oldCrsVer = @{$CFG->oldconfig('ORA_CRS_VERSION')};
     my $crsversion = join('.', @oldCrsVer);

     my $clustername = $CFG->params('CLUSTER_NAME');

     # get the OCR location
     my $oraocr = $CFG->compOCR;
     my @output = $oraocr->getOCRLoc();
     my $ocrType = shift @output;
     my $value = shift @output;
     my $localNodeName = tolower_host();

     if ($ocrType == 1)
     {
       my $filenameOnNAS = "$clustername"."_backup"."$crsversion".".ocr";
       $filenameOnNAS = catfile ($value, $filenameOnNAS);
       die(dieformat(629,$localNodeName,$ocrBackupFileName,$filenameOnNAS));
     }
     elsif ($ocrType == 2)
     {
       my $cmd;
       my $filenameOnDG = "$clustername"."_backup"."$crsversion".".ocr";
       $filenameOnDG = lc($filenameOnDG);
       $value = "+" . $value . ":";
       $filenameOnDG = catfile ($value, $filenameOnDG);
       my $ocrConfig = catfile($CFG->ORA_CRS_HOME, 'bin', 'ocrconfig');
       $cmd = "$ocrConfig -copy $ocrBackupFileName $filenameOnDG";
       die(dieformat(630, $localNodeName, $cmd, $ocrBackupNodeName));
     }
   }
   else
   {
     trace("OCR backup file is already on shared storage.");
     return SUCCESS;
   }
}

1;