#!/usr/local/bin/perl
# 
# $Header: has/install/crsconfig/oraacfs.pm /st_has_12.2.0.1.0ocwpsu/2 2017/08/07 21:10:42 apfwkr Exp $
#
# oraacfs.pm
# 
# Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
#
#    NAME
#      oraacfs.pm - Library module for ACFS root install functions.
#
#    DESCRIPTION
#      oraacfs.pm - Contains initial installation and deinstallation
#                   routines for ACFS.
#
#    NOTES
#      Required variables from the config object $CFG:
#      $CFG->platform_family
#      $CFG->DOWNGRADE
#      $CFG->HOST
#      $CFG->OLD_CRS_HOME
#      $CFG->ORA_CRS_HOME
#      $CFG->SIHA
#      $CFG->SUPERUSER
#      $CFG->oldconfig('ORA_CRS_VERSION')
#      $CFG->params('ORA_ASM_GROUP')
#      $CFG->params('ORACLE_OWNER')
#      $CFG->wipCkptName("ROOTCRS_ACFSINST")
#      $CFG->wipCkptName("ROOTCRS_ACFSUNINST")
#
#
#    MODIFIED   (MM/DD/YY)
#    apfwkr      08/07/17 - Backport madoming_bug-25518447 from main
#    apfwkr      08/03/17 - Backport
#                           loguzman_ci_backport_24953946_12.2.0.1.170117opcocwbp
#                           from st_has_12.2.0.1.0
#    luoli       12/26/16 - XbranchMerge luoli_bug-25067205 from main
#    loguzman    11/10/16 - Backport loguzman_bug-24953946 from main
#    luoli       11/30/16 - Fix bug 25067205
#    loguzman    10/27/16 - upgrade_acfs acfsutil registry output 
#    loguzman    11/10/16 - Backport loguzman_bug-24953946 from main
#    loguzman    10/27/16 - upgrade_acfs acfsutil registry output 
#    xyuan       07/06/16 - Fix bug 23747422
#    muhe        06/14/16 - Fix bug 23192386
#    madoming    05/16/16 - Fix bug 23256587
#    jorhuert    04/29/16 - Fix bug 23171392
#    lcarvall    04/29/16 - Fix Bug 22952033 - failure to upgrade based on
#                           nonexistent volume
#    bbeelamk    04/14/16 - Fix bug 23095140
#    madoming    04/12/16 - Fix bug 23092311: Remove unnecessary output
#    madoming    04/01/16 - Fix bug 23035076: Add trace messages for chkpoints
#    muhe        03/09/16 - Fix bug 22894801: Cache and reuse the result of 
#                           'acfsdriverstate supported'
#    bbeelamk    03/07/16 - Fix srvctl error
#    jorhuert    03/03/16 - Add ACFS Remote credentials.
#    agraves     02/26/16 - Move the disableADVMProxy call to the EXIT, so it
#                           fires if there are no volumes.
#    mperezh     02/24/16 - Consider null user when setting actions for the
#                           driver
#    xyuan       02/14/16 - Fix bug 22613675
#    madoming    02/22/15 - Fix bug 22749692 - Use srvctl and srvctl_capture
#    mperezh     02/11/16 - Add actions to ora.drivers.acfs
#    agraves     02/08/16 - Instead of removing ASM Proxy, disable it and stop
#                           it if no volumes.
#    madoming    12/18/15 - Fix bug 22389068
#    madoming    11/27/15 - Bug 22273934: Moved getACFSVolumeDevice from crsutils
#    madoming    11/18/15 - Fix bug 21239350 - Run acfshanfs command.
#    bbeelamk    10/07/15 - Fix bug 21878673
#    madoming    09/03/15 - Fix bug 21693990
#    agraves     08/20/15 - Remove the code that removes the proxy, now that we
#                           are making the decision to leave it there all the
#                           time.
#    madoming    08/19/15 - RTI 18357764 - Add validation to isACFSPath
#    madoming    08/11/15 - Bug 21572384 - Add trace to acfsroot install
#    muhe        08/09/15 - Fix bug 21571757
#    madoming    07/27/15 - Bug 21506962 - Quote path parameter
#    madoming    06/08/15 - Bug 21213911 - Change parameter to installUSMDriver
#    lcarvall    06/01/15 - Bug 20004511 - Check wheter a PID is from ACFS path
#    madoming    04/06/15 - Move isACFSSupported, isACFSInstalled and 
#                           isACFSLoaded functions from crsutils.pm
#    madoming    02/19/15 - Changes for new framework
#    madoming    02/12/15 - Bug 20417861 - root script changes for oda\usm
#    luoli       02/02/15 - Fix bug 20454220
#    xyuan       09/02/14 - rsc modeling
#    lcarvall    07/20/14 - Bug 13800615 - Use mnemonic string instead numeric values
#    madoming    06/30/14 - Fix bug 19068333 - Reset ACFS wrapper scripts
#    xyuan       05/06/14 - Fix bug 18493777
#    tmagana     03/05/14 - bug 18319117
#    madoming    02/13/14 - Bug 18242449 - Fix typo error.
#    madoming    01/17/14 - Bug 18093508 - Check whether volume exists.
#    madoming    01/13/14 - Bug 17794235 - Start all volumes when coming from
#                           11.2 to 12.1
#    madoming    01/03/14 - Bug 17941243 add start_filesystem_resources 
#                           function.
#    madoming    12/17/13 - Add silent option to isACFSInstalled and 
#                           isACFSLoaded Functions.
#    madoming    12/16/13 - Add method for removing ADVMProxy Resource.
#    loguzman    12/09/13 - Bug 16300127 add all volumes to CRS and add
#                           check_volume_resources code to upgrade_all_volumes
#    madoming    11/27/13 - Add validation for running acfsutil command.
#    bamorale    11/15/13 - Bug17054779 add check_volume_resources
#    agraves     11/14/13 - Add -unsupported to calls that need it.
#    madoming    10/28/13 - Add method for checking if the path on an acfs 
#                           mountpoint.
#    ifarfan     09/23/13 - Not install ACFS when running on an ASM client.
#    xyuan       09/09/13 - Fix bug 17412464
#    xyuan       08/22/13 - Remove compilation warnings
#    madoming    04/25/13 - Starting volume before to register the file system.
#    madoming    04/19/13 - Fixing bug 16677971 - Moving code up.
#    madoming    04/03/13 - Handle error PRCA-1052, when adding a volume
#    madoming    02/07/13 - Ensure that the drivers resource is stopped before
#                           trying to install or uninstall drivers using
#                           acfsroot.
#    gsanders    01/08/13 - XbranchMerge gsanders_bug-16007983 from
#                           st_has_12.1.0.1
#    gsanders    12/21/12 - Bug 16007983 register with volume disabled
#    gsanders    11/26/12 - upgrade_acfs_registry return if ACFS not supported.
#    madoming    10/31/12 - Function actionACFSDriversResource should return
#                           SUCCESS when ACFS Drivers are not supported.
#    madoming    10/22/12 - Return SUCCESS explicitly in an development 
#                           environment.
#    madoming    09/20/12 - If we can not remove ACFS Drivers, we'll turn off
#                           the ACFS persistent log.
#    madoming    09/18/12 - Add a error message when fail to uninstall acfs.
#    madoming    08/28/12 - Bug 14476864 - Failed to stop current oracle 
#                           clusterware stack during upgrade, add is_dev_env
#                           check.
#    madoming    08/20/12 - Bug 14323082 - During delete resource 
#                           ora.drivers.acfs, add by default -f attribute.
#    ifarfan     08/15/12 - Change '-' for '/' while executing acfsutil on Windows.
#    agraves     05/11/12 - Remove LASTNODE check in deleteACFSRegistry, as the
#                           function upgrade_acfs_registry is only called
#                           during last node processing.
#    anjiwaji    05/08/12 - Rename createACFSDriversResource to
#                           actionACFSDriversResource and add resource
#                           delete action
#    gsanders    03/24/12 - upgrade_acfs_registry retcode on -l V11 failure
#    anjiwaji    03/09/12 - Fix SIHA check
#    gsanders    03/08/12 - upgrade_acfs_registry return code LRG 6804883
#    anjiwaji    02/29/12 - Handle the case for Windows in drivers resource
#                           creation, Bug 13785935
#    anjiwaji    02/15/12 - Add reboot logic to removeACFSRoot
#    gsanders    01/30/12 - remove removeACFSRegistry()
#    gsanders    01/25/12 - Upgrade support 11.x to 12c
#    anjiwaji    12/14/11 - Add function to create/start the ACFS drivers.
#    anjiwaji    01/19/12 - Fix message thrown by installUSMDriver
#    anjiwaji    01/17/12 - Remove debug lines
#    agraves     11/30/11 - Change reboot necessary to be a hard failure and
#                           disable CRS.
#    gsanders    12/14/11 - ACFS registry OCR to CRS upgrade
#    gsanders    10/27/11 - Fix bug 13247676
#    agraves     11/09/11 - Remove leading 0 from 9427 message check.
#    gsanders    11/05/11 - Fix bug 13263435, look for msg ACFS-9427
#    xyuan       08/02/11 - Incorporate changes from agraves_bug-12788726
#    xyuan       07/27/11 - XbranchMerge xyuan_bug-12701521 from
#                           st_has_11.2.0.3.0
#    anjiwaji    06/14/11 - Bug12644077 Update registry ACL attrib if it
#                           already exists.
#    dpham       04/20/11 - Move acfs' related functions from crsdeconfig
#    agraves     04/07/11 - Update the checking of return codes from crsctl, to
#                           use output. crsctl returns 0 usually.
#    agraves     03/07/11 - Initial Creation
# 

=head1 NAME

  oraacfs.pm  Oracle clusterware ACFS component configuration/startup package

=head1 DESCRIPTION

   This package contains functions required for initial configuration
   and startup of the ACFS component of Oracle clusterware

=cut

package oraClusterwareComp::oraacfs;

use parent 'oraClusterwareComp';

use strict;
use Carp;
use Cwd;
use English;
use File::Temp qw/ tempfile /;
use File::Spec::Functions;
use File::Find ();
use Term::ANSIColor;
use crsutils;
use Config;

use Exporter;

# Steps to configure ACFS on current node
use constant CONFIGURE_ACFS            => 'acfs_ConfigCurrentNode_step_1';
use constant INSTALL_ACFS              => 'acfs_ConfigCurrentNode_step_2';
use constant CREATE_ACFS_RESOURCE      => 'acfs_ConfigCurrentNode_step_3';
use constant REMOVE_ACFS_RESOURCE      => 'acfs_ConfigCurrentNode_step_4';

#Steps to upgrade ACFS on current node
use constant UPGRADE_ACFS              => 'acfs_UpgradeCurrentNode_step_1';
use constant UPGRADE_ADVM_REGISTRY     => 'acfs_UpgradeCurrentNode_step_2';
use constant UPGRADE_ACFS_REGISTRY     => 'acfs_UpgradeCurrentNode_step_3';

my (@ACFS_FILTER_TYPES) = ("ora.acfs.type",
                           "ora.acfs_cluster.type",
                           "ora.fs.type");

my (@ADVM_FILTER_TYPES) = ("ora.volume.type");

sub new
{
  my $class = shift;
  # Pass the component name into the constructor 
  my $componentName = @_;

  my $self = $class->SUPER::new(@_);
  $self->_initialize();

  return $self;
}

# Initialization
sub _initialize
{
  my $self = shift;

  my $compName = $self->compName; 
  trace("Perform initialization tasks before configuring $compName");
}

# Each specific component, which inherits from oraClusterwareComp, can 
# reimplement (override) the following methods:

# Is the component supported based on platform and user input
sub isSupported
{
  my $self = shift;

  # Supported on all platforms
  return isACFSSupported();
}

# Which component does this depend on
sub dependsOn 
{ 
  # No dependency
  return undef;
}

# Has the component already been configured
sub isConfigured
{
  my $self = shift;

  return FALSE;
}

#
# Methods for install
#

# Configure actions on first node
sub configureFirstNode 
{
  my $self = shift;
  my $compName = $self->compName;
  my $stepIndicator = shift;

  trace("Executing the step [$stepIndicator] to configure $compName ".
        "on the first node");

  configACFSSteps($stepIndicator);
}

# Configure actions on every node other than first node
sub configureNonFirstNode 
{
  my $self = shift;
  my $compName = $self->compName;
  my $stepIndicator = shift;

  trace("Executing the step [$stepIndicator] to configure $compName ".
        "on the current node");

  configACFSSteps($stepIndicator);
}

# Configure actions on SIHA node
sub configureSIHA 
{
  my $self = shift;
  my $compName = $self->compName;
  my $stepIndicator = shift;

  trace("Executing the step [$stepIndicator] to configure $compName ".
        "on the SIHA node");

  configACFSSteps($stepIndicator);
}


# Configure action on first node after the configured stack
# has been started
sub postConfigFirstNode
{
  my $self = shift;
  my $compName = $self->compName;

  trace ("Setting up ACFS Credential store.");
  ACFS_MemDomSetup();
  return SUCCESS;
}

# Configure action on other nodes than the first node after
# the configured stack has been started
sub postConfigNonFirstNode
{
  my $self = shift;
  my $compName = $self->compName;

  if (isAddNode($CFG->HOST) &&
      (isHANFSSupported()))
  {   
     my ($volume, $isrunningFS, $existsFS) = getNetStorageService();
     # If the resource exists. 
     # We are going to configure HANFS
     if ($existsFS &&
         defined($volume))
     {
        trace("Installing HANFS Configuration.");
        my $acfshanfs = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfshanfs');
        my $cmd = "$acfshanfs " . "addnode -nfsv4lock " . 
                  "-volume $volume -force";
        trace("Running $cmd");
        my ($status, @out) = system_cmd_capture($cmd);
        if ( scalar(grep(/ACFS-9203/, @out) ) )
        {
           trace("HANFS Configuration was added to node $CFG->HOST.");
        }
        else
        {
           trace("HANFS Configuration can not be add to node $CFG->HOST.");
        }
     }
  }
  trace ("Setting up ACFS Credential store.");
  ACFS_MemDomSetup();
  return SUCCESS;
}

# Configure action on SIHA node after the configured stack
# has been started
sub postConfigureSIHA
{
  my $self = shift;
  my $compName = $self->compName;

  return SUCCESS;
}

#
# Methods for upgrade
#

# The method for global checks related to ACFS component before upgrade
sub preUpgradeCheck
{
  my $self = shift;
  my $compName = $self->compName;
  trace("Doing prerequisite checks related to component $compName for upgrading");
  return $self->checkPath();
}

# The method for local checks related to ACFS component before upgrading the first node
sub upgradeCheckFirstNode
{
  return SUCCESS;
}

# The method for local checks related to ACFS component before upgrading the middle node
sub upgradeCheckMiddleNode
{
  return SUCCESS;
}

# The method for local checks related to ACFS component before upgrading the last node
sub upgradeCheckLastNode
{
  return SUCCESS;
}

# The method for local checks related to ACFS component before upgrading the SIHA node
sub upgradeCheckSIHA
{
  return SUCCESS;
}

# Upgrade action on first node
sub upgradeFirstNode
{
  my $self = shift;
  my $compName = $self->compName;
  my $stepIndicator = shift;

  trace("Executing the step [$stepIndicator] to upgrade $compName ".
        "on the first node");

  upgradeACFSSteps($stepIndicator);
}

# Upgrade action on every node other than first and last node
sub upgradeMiddleNode 
{
  my $self = shift;
  my $compName = $self->compName;
  my $stepIndicator = shift;
  my $chkpoints = USECHKPOINTS;

  trace("Executing the step [$stepIndicator] to upgrade $compName ".
        "on the current node");

  if (UPGRADE_ACFS eq $stepIndicator)
  {
    perform_installUSMDriver($chkpoints);
    return SUCCESS;
  }
  elsif (UPGRADE_ADVM_REGISTRY eq $stepIndicator)
  {
    return SUCCESS;
  }
  elsif (UPGRADE_ACFS_REGISTRY eq $stepIndicator)
  {
    acfsrepl_updateResources();
    return SUCCESS;
  }
  else
  {
    # Should not reach here
    croak "Step indicator out of bounds";
  }
}

# Upgrade action on last node
sub upgradeLastNode
{ 
  my $self = shift;
  my $compName = $self->compName;
  my $stepIndicator = shift;
  my $chkpoints = USECHKPOINTS;

  trace("Executing the step [$stepIndicator] to upgrade $compName ".
        "on the last node");

  if (UPGRADE_ACFS eq $stepIndicator)
  {
    perform_installUSMDriver($chkpoints);
    return SUCCESS;
  }
  elsif (UPGRADE_ADVM_REGISTRY eq $stepIndicator)
  {
    # Add all volumes to CRS and do a check
    # to bring them online
    my $success = upgrade_all_volumes();
    if (!$success) {
      trace ("Failed to add volumes to CRS");
    }

    $success = upgrade_acfs_registry();
    if (!$success) {
      trace ("Failed to upgrade acfs registry");
      #TODO: Check if needs to return false or die
      die(dieformat(422));
    }
    return SUCCESS;
  }
  elsif (UPGRADE_ACFS_REGISTRY eq $stepIndicator)
  {
    upgrade_acfs_registry();
    acfsrepl_lastNode();
    return SUCCESS;
  }
  else
  {
    # Should not reach here
    croak "Step indicator out of bounds";
  }
}

# Upgrade action on SIHA node
sub upgradeSIHA
{
  my $self = shift;
  my $compName = $self->compName;
  my $stepIndicator = shift;

  trace("Executing the step [$stepIndicator] to upgrade $compName ".
        "on the SIHA node");

  upgradeACFSSteps($stepIndicator);
}

# Upgrade action on the first node after the higher version stack
# has been started
sub postUpgradeFirstNode
{
  my $self = shift;

  return SUCCESS;
}

# Upgrade action on every node other than first and last node
# after the higher version stack has been started
sub postUpgradeMiddleNode
{
  my $self = shift;

  # Start filesystems resources but only when the node is joining and
  # it's coming from 11.2
  # In 11.2 only had clusterwide acfs resources, They should be online on all
  # nodes. They had AUTOSTART=always, or AUTOSTART=restore. However, we had
  # incorrect return values for the OFFLINE states, which resulted in CRS not
  # always starting them even though they were online previously.
  # The resource should get pulled up by a database resource, but in some cases
  # this didn't happen. However, the intent was to always have the FS online.
  # This isn't true in 12.1.0.1.
  if (($CFG->JOIN) && (isOldVersionLT121()))
  {
    trace("Starting filesystem resources on node that ".
          "is joining the cluster ...");
    start_filesystem_resources();
  }

  return SUCCESS;
}

# Upgrade action on the last node after the higher version stack
# has been started
sub postUpgradeLastNode
{
  my $self = shift;

  clear_perXsOnACFSEnableAttr($self);
  return SUCCESS;
}

# Upgrade action on the SIHA node after the higher version stack
# has been started
sub postUpgradeSIHA
{
  my $self = shift;

  return SUCCESS;
}

#
# Methods for downgrade
#

# downgrade actions on nodes other than the last node
sub downgradeNonLastNode
{
  my $self = shift;
  my $compName = $self->compName;

  trace("Executing step downgrade $compName ".
        "on the current node");

  removeACFSSteps();
  return SUCCESS;
}

# downgrade actions on the last node
sub downgradeLastNode
{
  my $self = shift;
  my $compName = $self->compName;

  trace("Executing step downgrade $compName ".
        "on the last node");

  removeACFSSteps();
  return SUCCESS;
}

#
# Methods for deconfiguration
#

# Deconfiguration action on nodes other than the last node
sub deconfigureNonLastNode
{
  my $self = shift;
  my $compName = $self->compName;

  if (isHANFSInstalled())
  {
    trace("Removing HANFS Configuration.");
    my $acfshanfs = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfshanfs');
    my $cmd = "$acfshanfs " . "uninstall -nfsv4lock";
    trace("Running $cmd");
    my ($status, @out) = system_cmd_capture($cmd);
    if ( scalar(grep(/ACFS-9203/, @out) ) )
    {
      trace("HANFS Configuration was removed from node $CFG->HOST.");
    }
    else
    {
      trace("HANFS Configuration can not be removed from node $CFG->HOST.");
    }
  }
  
  trace("Executing step deconfiguration $compName ".
        "on the current node");

  removeACFSSteps();
  return SUCCESS;
}

# Deconfiguration action on the last node
sub deconfigureLastNode
{
  my $self = shift;
  my $compName = $self->compName;

  if (isHANFSInstalled())
  {
    trace("Removing HANFS Configuration.");
    my $acfshanfs = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfshanfs');
    my $cmd = "$acfshanfs " . "uninstall -nfsv4lock";
    trace("Running $cmd");
    my ($status, @out) = system_cmd_capture($cmd);
    if ( scalar(grep(/ACFS-9203/, @out) ) )
    {
      trace("HANFS Configuration was removed from node $CFG->HOST.");
    }
    else
    {
      trace("HANFS Configuration can not be removed from node $CFG->HOST.");
    }
  }

  trace("Executing step deconfiguration $compName ".
        "on the last node");

  removeACFSSteps();
  return SUCCESS;
}

# The method for checks before patching
sub prePatchCheck
{
  my $self = shift;
  my $compName = $self->compName;
  trace("Doing prerequisite checks related to component $compName for patching");
  return $self->checkPath();
}

# Check if it's an ACFS Path
sub checkPath
{
  my $self = shift;
  my $compName = $self->compName;
  my @pidacfs  = isPIDACFSPath();
  
  foreach my $value (@pidacfs){
    if (isACFSPath($value)) {
      die(dieformat(494));
    }
  }
  return SUCCESS;
}

# Whether or not the system reboot is needed after ACFS is configured
sub rebootRequired { return FALSE; }

# How to start the component
sub start 
{
  my $self = shift;
  return actionACFSDriversResource("start");
}

# How to stop the component
sub stop 
{ 
  my $self = shift;
  return actionACFSDriversResource("stop");
}

#
# Private methods
#

sub configACFSSteps
{
  my $stepIndicator = shift;
  my $chkpoints = USECHKPOINTS;
  my $has = "crs";

  if ( $CFG->SIHA )
  {
    $chkpoints = NOTUSECHKPOINTS;
    $has = "has";
  }

  if (CONFIGURE_ACFS eq $stepIndicator)
  {
    perform_installUSMDriver($chkpoints);
    return SUCCESS;
  }
  elsif (INSTALL_ACFS eq $stepIndicator)
  {
    return installUSMDriver($has);   
  }
  elsif (CREATE_ACFS_RESOURCE eq $stepIndicator)
  {
    return actionACFSDriversResource("add");
  }
  elsif (REMOVE_ACFS_RESOURCE eq $stepIndicator)
  {
    return actionACFSDriversResource("delete");
  }
  else
  {
    # Should not reach here
    croak "Step indicator out of bounds";
  }
}

sub upgradeACFSSteps
{
  my $stepIndicator = shift;
  my $chkpoints = USECHKPOINTS;

  if ( $CFG->SIHA )
  {
    $chkpoints = NOTUSECHKPOINTS;
  }


  if (UPGRADE_ACFS eq $stepIndicator)
  {
    perform_installUSMDriver($chkpoints);
    return SUCCESS;
  }
  elsif (UPGRADE_ADVM_REGISTRY eq $stepIndicator)
  {
    return SUCCESS;
  }
  elsif (UPGRADE_ACFS_REGISTRY eq $stepIndicator)
  {
    return SUCCESS;
  }
  else
  {
    # Should not reach here
    croak "Step indicator out of bounds";
  }
}

# When upgrading from the legacy ASM, do the global disable and re-enable
# for the ACFS resources on the last node so that per-Xs on ENABLED
# attribute go away - Fix bug#22613675.
sub clear_perXsOnACFSEnableAttr
{
  my $self = shift;

  if (isLegacyASM())
  {
    trace("Retrieve all ACFS resources that are enabled and online");
    my $crshome = $CFG->ORA_CRS_HOME;
    my $acfsResList = queryEnabledAndOnlineRes($crshome, $CFG->HOST, "acfs");
    trace("ACFS resource list: [$acfsResList]");

    my @acfsRes = split(/,/, $acfsResList);
    my %volDevices;
    foreach my $acfs (@acfsRes)
    {
      if ('ora.registry.acfs' ne $acfs)
      {
        my $vol = $self->getACFSVolumeDevice($acfs);
        if (defined $vol)
        {
          $volDevices{$acfs} = $vol;
        }
      }
    }

    foreach my $acfs (keys %volDevices)
    {
      my $volDev = $volDevices{$acfs};
      trace("Attempt to disable and re-enable $acfs for device $volDev");
      disable_enable_filesystem("disable", $volDev) ||
         die(dieformat(649, $acfs));
      disable_enable_filesystem("enable", $volDev) ||
         die(dieformat(650, $acfs));
    }
  }

  return;
}

sub removeACFSSteps
{
  my $chkpoints = USECHKPOINTS;

  if ( $CFG->SIHA )
  {
    $chkpoints = NOTUSECHKPOINTS;
  }

  # De-install current ACFS drivers
  if (isACFSSupported() && isACFSInstalled())
  {
    removeACFSRoot($chkpoints);
  }
}

# private methods called by above APIs

# OSD API definitions
# TODO: We'll need this for acfs if there are OSD sections.
#use s_oraacfs;




=head2 upgrade_acfs_registry

   Upgrades the ACFS Registry

=head3 Parameters

   None

=head3 Returns

  TRUE  - ACFS Registry successfully upgraded.
  FALSE - ACFS Registry not successfully upgraded.

=head3 Notes


=cut
sub upgrade_acfs_registry
{
    my $cmd;
    my $status;
    my $srvctl_args;
    my $acfsutil;
    if ($CFG->platform_family eq 'windows') {
        $acfsutil = catfile ($CFG->ORA_CRS_HOME, 'bin', 'acfsutil');
    }
    else {
      $acfsutil = "/sbin/acfsutil";
    }
    my $asmcmd = catfile($CFG->ORA_CRS_HOME, "bin", "asmcmd");
    my @ocrRegList;
    my @asmcmdvolinfo;

    #
    #  If USM isn't supported, return.
    #

    return SUCCESS if ( ! isACFSSupported() );

    #
    #  Get list of file systems registered in OCR pre 12.X
    #

    $cmd = "$acfsutil registry -l -V 11";
    if ($CFG->platform_family eq 'windows') {
        $cmd =~ s/\ -/\ \//g ;
    }
    ($status, @ocrRegList) = system_cmd_capture($cmd);
    if ($status != 0)
    {
      trace ("$cmd ... failed");
      print_error(422);
      return FAILED;
    }

    #
    #  For each file system registered in the OCR reregister it with CRS.
    #  Sometimes the registry can refer to a nonexistent volume, causing a 
    #  failure when trying to add it to the cluster. So we proceed to compare 
    #  the output of ASMCMD, if they are equal we can continue, otherwise we 
    #  issued a message.
    #

    foreach my $ocrReg ( @ocrRegList )
    {
        my $device       = "";
        my $path         = "";
        my $options      = "";
        my $optionsArg   = "";
        my $nodes        = "";
        my $nodeArg      = "";
        my $diskGroup    = "";
        my $volume       = "";
        my $accel        = "";
        my @out;
        my $run_as_owner = FALSE; # FALSE is run_as_root
 
        chomp $ocrReg;

        #
        #  Format hint:
        #       Device : /dev/asm/crsdg1vol5-326 : Mount Point :
        #       /tkfv_mounts/crsdg1vol5 : Options : none : Nodes : all :
        #       Disk Group : CRSDG1 : Primary Volume : CRSDG1VOL5 :
        #       Accelerator Volumes : ACCELVOL
        #  We could (and are capable of) writing much more robust
        #  regular expressions here but we're not going to.
        #

        $device = $ocrReg;
        $device =~ s/^Device : (.*) : Mount Point : .*/$1/;

        $path = $ocrReg;
        $path =~ s/.* : Mount Point : (.*) : Options : .*/$1/;

        $options = $ocrReg;
        $options =~ s/.* : Options : (.*) : Nodes : .*/$1/;
        $optionsArg = "-o $options" if ( $options ne "none" );

        $nodes = $ocrReg;
        $nodes =~ s/.* : Nodes : (.*) : Disk Group: .*/$1/;
        $nodeArg = "-n $nodes" if ( $nodes ne "all" );

        $diskGroup = $ocrReg;
        $diskGroup =~ s/.* : Disk Group: (.*) : Primary Volume : .*/$1/;

        $volume = $ocrReg;
        $volume =~ s/.* : Primary Volume : (.*) : Accelerator Volumes : .*/$1/;

        $accel = $ocrReg;
        $accel =~ s/.* : Accelerator Volumes : (.*)/$1/;

        #
        #  Check the Volume device name, it has to include "\\.\" for Windows
        #

        if (($CFG->platform_family eq "windows") && ($device !~ /^\\\\.\\/))
        {
          $device = "\\\\.\\" . $device;
        } 

        #  We have all the attrs from this registry necessary to create
        #  the new CRS Resource based registration so create it.
        #
        #  But first a work around. As of 12.1 ASM volume devices, on
        #  which file systems reside, have a corresponding CRS
        #  resource.  Normally this would be created when the volume
        #  was enabled.  But since the volume was enabled in a
        #  pre-12.1 stack the resource didn't get created. Create it
        #  here and now.  Note that if the upgrade is being rerun the
        #  volume might already exist.

        $srvctl_args = "status volume -g $diskGroup -volume $volume";
        $status = srvctl_capture($run_as_owner, \@out, $srvctl_args);
        my $isEnabled = FALSE;
        my $isRunning = FALSE;
        if ( scalar(grep(/PRCA-1051/, @out)) )
        {
          #
          #  The volume doesn't exist. Create it, otherwise issued a message 
          #  and move on to the next loop element.
          #  Bug 22952033 Could be the case when the returned value 
          #  using acfsutil registry -l -V11 is for a nonexistent diskgroups or 
          #  volumes. Check using asmcmd if true.
          #
          
          trace("Invoking asmcmd volinfo -G \"$diskGroup $volume\"");
          @asmcmdvolinfo = `$asmcmd volinfo -G $diskGroup $volume`;
          if ( scalar(grep(/ASMCMD-8001/, @asmcmdvolinfo)) )
          {
            trace ("Disk Group: $diskGroup does not exist.");          
            next;
          }
          elsif( scalar(grep(/not found in diskgroup/, @asmcmdvolinfo)) ){
            trace ("Volume: $volume does not exist in diskgroup $diskGroup.");
            next;
          }

          $srvctl_args = "add volume -g $diskGroup -d $device -volume $volume";
          trace("Invoking srvctl \"$srvctl_args\"");
          $status = srvctl_capture($run_as_owner, \@out, $srvctl_args);
          if ( $status == 0 || scalar(grep(/PRCA-1052/, @out)) )
          {
            # We ignore the message, 
            # PRCA-1052 : When attempting to create a volume resource in CRS, 
            # volume resource already exists for disk group $diskGroup 
            # and volume $volume
            
            trace ("Volume $volume was added successfully or already exists.");
            
            # After add a volume resource, it has to be enabled but not running
            $isEnabled = TRUE;
            $isRunning = FALSE;
          }
          else {
            # Error creating the volume
            print_lines(@out);
            print_error(418, $device, $path );
            return FAILED;
          }
        }
        elsif ( $status != 0 )
        {
          # Some error occurred attempting to run srvctl
          print_lines(@out);
          print_error(419);
          return FAILED;
        }
        else
        {
          # No errors detected in srvctl output from volume status
          # command.  That must mean the volume already exists for some
          # reason.  Probably the upgrade is being rerun.
          trace ("$device already exists (OK)");
          # Check whether volume is enable and running
          $isEnabled = scalar(grep(/is enabled/, @out));
          $isRunning = scalar(grep(/is running/, @out));
        }

        # Before to call registry. We need to be sure the volume is enable
        if (!$isEnabled)
        {
          trace ("Enabling volume $volume");
          $srvctl_args =  "enable volume -g $diskGroup -volume $volume";
          srvctl($run_as_owner, $srvctl_args);
        }

        # The volume could be not started. 
        # We need to be sure the volume is started
        if (!$isRunning)
        {
          trace ("Starting volume $volume");
          $srvctl_args = "start volume -g $diskGroup -volume $volume";
          srvctl($run_as_owner, $srvctl_args);
        }

        # OK. Let's register the file system in CRS if it doesn't exist
        # already. (It might already exist if the upgrade is being rerun.)

        $srvctl_args = "status filesystem -d $device";
        $status = srvctl_capture($run_as_owner, \@out, $srvctl_args);
        if ( scalar(grep(/PRCA-1070/, @out)) )
        {
          $cmd = "$acfsutil registry -v -a " .
                 "$nodeArg $optionsArg " .
                 "$path $device";
          if ($CFG->platform_family eq 'windows') {
              $cmd =~ s/\ -/\ \//g ;
          }
          ($status, @out) = system_cmd_capture($cmd);
          if ( $status != 0 )
          {
            if ( scalar(grep(/PRCA-1089/, @out)) )
            {
              # Bug 16007983 acfsutil calls srvctl to register. srvctl
              # couldn't get the volume info.  Maybe volume is disabled.
              # Forge onward.
              print_error(420, $device, $path );
            } 
            else
            {
              # fatal error registering volume with srvctl.
              print_error(420, $device, $path );
              return FAILED;
            }
          }
        }
        elsif ( $status != 0 )
        {
          # Some error occurred attempting to run srvctl
          print_lines(@out);
          print_error(421);
          return FAILED;
        }
        else
        {
          # No errors detected in srvctl output from volume status
          # command.  That must mean the volume already esists for some
          # reason.  Probably the upgrade is being rerun.
          trace ("$device already registered (OK)");
        }
    } 

    #
    #  Delete the old registry resource, ora.registry.acfs. It is no
    #  longer used to manage the registered file system.
    #

    if ( deleteACFSRegistry() != SUCCESS )
    {
      return FAILED;
    }
    return SUCCESS;
}

=head2 installUSMDriver

   Installs ACFS, ADVM, OKS - kernel and user components via acfsroot.

=head3 Parameters

   None

=head3 Returns

  TRUE  - Configuration successful
  FALSE - Configuration failed

=head3 Notes


=cut
sub installUSMDriver
{
   my $acfsroot;
   my $ret = SUCCESS;
   my $crsctl = crs_exec_path('crsctl');
   my ($has) = @_;

   # if we are running in development mode, then limit support to only when
   # the appropriate env variables are set

   if (is_dev_env())
   {
      my $acfsInstall = uc($ENV{'USM_ENABLE_ACFS_INSTALL'});

      # if this ENV is not set then we give up early
      if ( $acfsInstall ne "TRUE" )
      {
         trace("ADVM/ACFS disabled because of ENV in test mode");
         return $ret;
      }
   }

 # bug 18319117: if ACFS is not supported in this platform (either outright
 # or because the OS patch level is not proper) just return success.
   if (!isACFSSupported())
   {
     trace("Skipping ACFS install: It is not supported on this OS/patch level");
     return $ret;
   }

   if ($CFG->platform_family eq 'windows') {
      $acfsroot = catfile ($CFG->ORA_CRS_HOME, 'bin', 'acfsroot.bat');
   }
   else {
      $acfsroot = catfile ($CFG->ORA_CRS_HOME, 'bin', 'acfsroot');
   }

   if (-e $acfsroot) {
      my $cmd = "$acfsroot install -t2";
     
      trace("Stopping ora.drivers.acfs if it exists, so that it doesn't race.");
      # Don't care about the return code of this.  If it doesn't exist,
      #  we'll get a success.  This is to handle a case where OHASD has 
      #  bounced, and the drivers resource is now unintentionally online.
      # If it does exist, we'll get a success, unless there's something wrong
      #  with CRS, as the drivers resource 'stop' action doesn't do anything
      #  anymore - we just want to ensure that CRS doesn't monitor the 
      #  resource anymore, as a load or unload would be caught by CRS
      #  causing the monitor to request a restart of the resource,
      #  potentially racing with acfsroot to do its install.  (acfsroot install
      #  does an uninstall, which a running resource would pick up on and
      #  go offline.)
      # If CRS is not up, and we get an error, that's okay as well - we'll
      #  continue on, as that's effectively what we want.
      actionACFSDriversResource("stop");

      # Fix bug 25067205
      if ($OSNAME eq "aix" )
      {
        if(defined $ENV{'ODMDIR'})
        {
          trace("Getting 'ODMDIR' from system environment:");
        }
        else 
        {
          my $found = FALSE;
          my $conf_loc = "/etc/environment";
          open(CONF, "<", $conf_loc) or return FAILED;
          while (<CONF>)
          {
            if (/^ODMDIR=(\S+)/)
            {
              trace("Getting 'ODMDIR' from '/etc/environment':");
              my $value = $1;
              $value = trim($value);
              $ENV{'ODMDIR'} = $value;
              $found = TRUE;
              last;
            }
          }
          close(CONF);
          die(dieformat(676)) if (!$found);
        }
        trace("ODMDIR=$ENV{'ODMDIR'}");
      }
          
      trace("Executing '$cmd'");
      my @output = system_cmd_capture($cmd);
      my $rc = shift @output;

      if ($rc == 0) {
        my $enable = "$crsctl enable $has";
        my @output = system_cmd_capture($enable);
        my $rc = shift @output;
        trace ("$cmd ... success");

        if ($rc != 0) {
          trace("$enable .... unable to enable CRS.");
          print_error(412);
          $ret = FAILED;
        }
        else {
          trace("$enable .... success.");
        }
      }
      elsif ($rc != 2) {
         # 9427 covers unload failures.
         # 9428 covers install and load failures.
         #   (driver currently running, driver in use can't copy file,
         #    driver unable to load because Windows thinks its running
         #    but its not.)
         if ((scalar(grep(/9427/, @output))) || 
             (scalar(grep(/9428/, @output))) > 0)
         {
           #We'll keep going if we need to reboot but we are in a development
           #environment.
           if (is_dev_env())
           {
              trace ("Skipping reboot in development env ... success");
              return SUCCESS;
           }

           $ret = REBOOT;
           my $disable = "$crsctl disable $has";
           trace("$disable ... disabling CRS in preparation for reboot.");
           my @output = system_cmd_capture($disable);
           my $rc = shift @output;
           if ($rc != 0)
           {
             trace("$disable ... unable to disable CRS for reboot.");
             # Don't fail here - the user should correct
             # the disable, by running it by hand, and reboot.
             print_error(413);
           }
           else
           {
             trace("$disable ... CRS disabled, ready for reboot.");
             # At this point, CRS is disabled.  The user will need
             # to reboot, rerun the operation.  This will rely on 
             # OPatch and OUI to print the appropriate error message.
           }
         }
        else
         {
           $ret = FAILED;
           trace ("$cmd ... failed");
         }
      }
   }
   else {
      trace("$acfsroot not found");
      if (isACFSSupported())
      {
        # if acfsroot not found and usm supported, we have a problem
        # some required files are not here.
        trace("ACFS is supported on this platform, but install files are missing.");
        $ret = FAILED;
      }
      else
      {
        # If acfsroot not found and acfs not supported,
        # then assume everything is okay.
        trace("ACFS is not supported on this platform.");
        $ret = SUCCESS;
      }
   }

  trace("USM driver install status is $ret");
  return $ret;
}


# Add all volumes to CRS
sub upgrade_all_volumes{
 my @cmd;
 my $srvctl_args;
 my $crsctlcmd;
 my @volume_names;
 my $vol;
 my $dg;
 my $dev;
 my $val;
 my $txt;
 my @volumes;
 my @diskgroups;
 my @devices;
 my $i = 0;
 my @output1;
 my $rc;
 my $exit_rc = 0;
 my $resName = "";
 my $crsctl = catfile($CFG->ORA_CRS_HOME, "bin", "crsctl");
 my $unset_home = $ENV{'ORACLE_HOME'};
 my $unset_sid = $ENV{'ORACLE_SID'};
 my $asmcmd = catfile($CFG->ORA_CRS_HOME, "bin", "asmcmd");

 #
 # We need to get the old crs version to decide what to do next:
 # >> If old crs version is < 12.1 then we need to create a resource 
 # for each volume. 
 # >> Otherwise we only need to bring the existing volume resources 
 # online (this includes flex upgrades too). 
 #
 if ( ! isOldVersionLT121() )
 {
   # crs version is not lower than 12.1 
   goto check_volume_resources;  
 }

 # Get ORACLE_SID value
 @cmd = ($crsctl, "stat", "res", "ora.asm", "-init", "-p");
 open STATOUT, "@cmd |";
 my @out = <STATOUT>;
 close (STATOUT);
 @cmd = grep (/GEN_USR_ORA_INST_NAME/, @out);
 ($txt, $val) = split(/=/, $cmd[0]);
 chomp($val);
 
 #
 #  Set ORACLE_SID, ORACLE_HOME and ORACLE_OWNER.
 #  These values will be unset later.
 #
 $ENV{'ORACLE_SID'} = $val;
 my $usr = $CFG->params('ORACLE_OWNER');
 $ENV{'ORACLE_HOME'} = $CFG->ORA_CRS_HOME; 

 # Execute asmcmd volinfo --all and get its output
 @cmd = ($asmcmd, '--nocp', 'volinfo', '--all');
 $rc = run_as_user2($usr, \@volume_names, @cmd);

 #
 #  NOTE: asmcmd volinfo --all output is displayed in English.
 #  If there are no volumes then we get a "no volumes found" string
 #  in English. Look at asmcmdvol.pm file for more information.
 #

 if ($volume_names[0] =~ /Diskgroup/)
 {
   # Get all volumes and their devices
   @volumes = grep (/Volume Name/, @volume_names);
   @devices = grep (/Volume Device/, @volume_names);

   my $arrayLen = @volumes;
   my @tmp = @volume_names;

   #
   # Get diskgroup names
   #

   $i = 0;
   my $j = 0;
   foreach my $line (@tmp)
   {
     if ($line =~ /Volume Name/)
     {
       if ($volume_names[$i-2] =~ /Diskgroup Name/)
       {
         ($val,$dg) = split(/:/, $volume_names[$i-2]);
         $dg = trim($dg);
         chomp($dg);
         $diskgroups[$j] = uc($dg);
         ++$j;
       }
       else
       {
         $diskgroups[$j] = uc($dg);
         ++$j;
       }
     }
     ++$i;
   }
   $i = 0;

   # Create a resource for each volume
   while ($i < $arrayLen)
   {
     # Get volume and device names 
     ($val,$vol) = split(/:/, $volumes[$i]);
     $vol = trim($vol);
     chomp($vol);
     ($val,$dev) = split(/:/, $devices[$i]);
     $dev = trim($dev);
     chomp($vol);
     $dg = $diskgroups[$i];

     #First check whether volume exists
     my $status;
     my @srvout;
     $srvctl_args = "status volume -g $dg -volume $vol";
     $status = srvctl_capture(FALSE, \@srvout, $srvctl_args);
     if ( scalar(grep(/PRCA-1051/, @srvout) ) )
     {
       # Add volume
       $srvctl_args = "add volume -g $dg -d $dev -volume $vol";
       trace("$srvctl_args");
       $rc = srvctl(FALSE, $srvctl_args);
       if ($rc != TRUE)
       {
         trace("Cannot add volume $vol to CRS: rc=$rc");
       }
     }
     ++$i;
   }
 } 
 elsif (trim($volume_names[0]) eq "no volumes found")
 {
   # nothing to do here.
   trace ("No volumes found");
   goto EXIT;
 }
 elsif (trim($volume_names[1]) =~ /8162/)
 {
   trace("Connected to an idle instance: $volume_names[1]");
   $exit_rc = 1;
   goto EXIT;
 }
 else
 {
   # Trace what happened
   trace("upgrade_all_volumes failed: @volume_names");
   $exit_rc = 1;
   goto EXIT;
 }

check_volume_resources:
 # Do a check on all volume resources on the cluster
 # In 11.2 only had clusterwide volume resources. They should be online 
 # on all nodes.
 # So, If we came from 11.2 to 12.1, we'll try to start in other case
 # we only do a check
 if (isOldVersionLT121())
 {
   # Start volume resources
   trace("Trying to start all volumes");
   $crsctlcmd = "$crsctl start resource -w \"TYPE = ora.volume.type\" -unsupported";
   ($rc, @output1) = system_cmd_capture_noprint($crsctlcmd);
   # Volume Resources could be started and we can get result different to 0.
   # We'll return always 0.
   $exit_rc = 0;
 }
 else
 {
   $crsctlcmd = "$crsctl check resource -w \"TYPE = ora.volume.type\" -unsupported";
   ($rc, @output1) = system_cmd_capture($crsctlcmd);
   if ($rc != 0)
   {
     trace ("@output1");
     trace ("$crsctlcmd ... failed");
     $exit_rc = 1;
   }
 }


EXIT:

 disableADVMProxy();
 # Unset and exit
 $ENV{'ORACLE_HOME'} = $unset_home;
 $ENV{'ORACLE_SID'} = $unset_sid;


 if ($exit_rc != 0)
 {
     return FALSE;
 }
 return TRUE;
}   


sub deleteACFSRegistry
#-------------------------------------------------------------------------------
# Function: Delete ACFS Registry resource.
# Args    : 0
#-------------------------------------------------------------------------------
{
   my $node   = $CFG->HOST;
   my $crsctl = crs_exec_path('crsctl');
   my $res    = 'ora.registry.acfs';
   my @out;
   my $status;
   my @cmd;

   # check if acfs registry status
   @cmd = ($crsctl, 'stat', 'res', $res );
   @out      = system_cmd_capture(@cmd);
   if ( "@out" =~ "CRS-2613" ) { # Could not find resource 'ora.registry.acfs'
      trace ("ACFS registry resource does not exist. (OK)");
      return SUCCESS;
   }

   # delete acfs registry only called if lastnode. 
   # Note we don't stop it.  Stopping
   # it would unmount the file systems and in a upgrade to 12.1 we don't
   # want to unmount them.  The -f (force) option allows us to delete
   # the registry resource even though it is ONLINE.
   @cmd = ($crsctl, 'delete', 'res', $res, '-f' , '-unsupported');
   ($status, @out) = system_cmd_capture(@cmd);
   if ($status == 0) {
      trace ("@cmd ... success");
   } else {
      trace ("@cmd ... failed");
      return FAILED;
   }
   # delete the registry type
   @cmd = ($crsctl, 'delete', 'type', 'ora.registry.acfs.type', '-unsupported');
   ($status, @out) = system_cmd_capture(@cmd);
   if ($status == 0) {
      trace ("@cmd ... success");
   } else {
      trace ("@cmd ... failed (benign)");
      # harmless enough. Let's not abort an upgrade for this.
      $status = SUCCESS;
   }
   
   return SUCCESS;
}

sub disableACFSDriver
#-------------------------------------------------------------------------------
# Function: disable the ACFS drivers
# Args    : 0
#-------------------------------------------------------------------------------
{
   my $crsctl = crs_exec_path('crsctl');
   my $res    = 'ora.drivers.acfs';

   # disable acfs drivers
   my @cmd    = ($crsctl, 'modify', 'resource', $res,
                 '-attr', "\"ENABLED=0\"", '-init');
   my $status = system_cmd(@cmd);

   if ($status == 0) {
      trace ("@cmd ... success");
   } else {
      trace ("@cmd ... failed");
   }
}

sub deleteACFSDriver
#-------------------------------------------------------------------------------
# Function: Delete ACFS resource 
# Args    : 0
#-------------------------------------------------------------------------------
{
   my $crsctl = crs_exec_path('crsctl');
   my $res = 'ora.drivers.acfs';

   # delete acfs drivers
   my @cmd = ($crsctl, 'delete', 'res', $res, '-init', '-f');

   trace ("@cmd");
   my $status = system_cmd(@cmd);
   if ($status == 0) {
      trace ("@cmd ... success");
   } else {
      trace ("@cmd ... failed");
   }
}

sub removeACFSRoot
{
   my $chkpoints =  $_;
   my $acfsroot;
   my $acfsutil;
   my $has       =  "has";
   my $ckptName = "ROOTCRS_ACFSUNINST";
   my $crsctl    =  catfile ($CFG->ORA_CRS_HOME, "bin", "crsctl");

   if ($CFG->platform_family eq 'windows') {
      $acfsroot = catfile ($CFG->ORA_CRS_HOME, 'bin', 'acfsroot.bat');
   }
   else {
      $acfsroot = catfile ($CFG->ORA_CRS_HOME, 'bin', 'acfsroot');
   }
   
   if ($CFG->platform_family eq 'windows') {
      $acfsutil = catfile ($CFG->ORA_CRS_HOME, 'bin', 'acfsutil');
   }
   else {
      $acfsutil = "/sbin/acfsutil";
   }

   if (! (-e $acfsroot)) {
      trace ("ADVM/ACFS is not configured");
      return;
   }

   if ( !$CFG->SIHA )
   {
      $has = "crs"; 
   }
   if ( $chkpoints == USECHKPOINTS )
   {  
      if (!isCkptexist($ckptName)) 
      {
         trace("Writing checkpoint for USM driver uninstall");
         writeCkpt($ckptName, CKPTSTART);
         $CFG->wipCkptName($ckptName);
      }
      else {
         my $ckptStatus = getCkptStatus($ckptName);
         trace("'$ckptName' state is '$ckptStatus'");

         if (isCkptSuccess($ckptName))
         {
            trace("ACFS is already uninstalled");
         }
      }
   }
   else {
      trace("Not using checkpoint for USM driver uninstall");
   }

   if (($chkpoints == NOTUSECHKPOINTS) || (!isCkptSuccess($ckptName)) )
   {
      $CFG->wipCkptName($ckptName);

      trace("Stopping ora.drivers.acfs if it exists, so that it doesn't race.");
      # Don't care about the return code of this.  If it doesn't exist,
      #  we'll get a success.  This is to handle a case where OHASD has 
      #  bounced, and the drivers resource is now unintentionally online.
      # If it does exist, we'll get a success, unless there's something wrong
      #  with CRS, as the drivers resource 'stop' action doesn't do anything
      #  anymore - we just want to ensure that CRS doesn't monitor the 
      #  resource anymore, as a load or unload would be caught by CRS
      #  causing the monitor to request a restart of the resource,
      #  potentially racing with acfsroot to do its install.  (acfsroot install
      #  does an uninstall, which a running resource would pick up on and
      #  go offline.)
      # If CRS isn't up and we get an error, that's okay as well - its
      #  effectively what we want.
      actionACFSDriversResource("stop");

      my @cmd    = ($acfsroot, 'uninstall','-t2');
      trace ("Executing @cmd");
      my @output = system_cmd_capture(@cmd);
      my $rc = shift @output;

      if ( $rc == 0 ) {
         trace ("@cmd ... success");
         trace("ACFS drivers uninstall completed");
                 
         # Only try a re-enable if we had previously failed.
         # OR if we have no chkpoints. AND we are in the DOWNGRADE process
         # This is because we only require a reboot in the downgrade process. 
         # See below. So we disable crs before reboot and enable it here.

         if ( (($chkpoints == NOTUSECHKPOINTS) ||
               ((getCkptStatus($ckptName) == CKPTFAIL))) &&
              ($CFG->DOWNGRADE) )
         { 
            my $enable = "$crsctl enable $has";
            @output = system_cmd_capture($enable);
            $rc = shift @output;
            trace ("@cmd ... success");
            
            if ($rc != 0) {
               print_lines(@output);
               trace("$enable .... unable to enable CRS.");
               print_error(435);
               trace("\"$enable\" failed with status $rc");
               exit 1;
            }
            else {
               trace("$enable .... success.");
            }
         }

         if ($chkpoints == USECHKPOINTS)
         {
            writeCkpt($ckptName, CKPTSUC);
         }

      }
      elsif ( $rc != 2 )
      {
         #We can not unload and remove the drivers,
         #We'll try to turn off the ACFS persistent log
         if (-e $acfsutil) {
            my $cmd = "$acfsutil plogconfig -t";
            if ($CFG->platform_family eq 'windows') {
               $cmd =~ s/\ -/\ \//g ;
            }
            #We'll keep going, so we won't check the output
            system_cmd_capture($cmd);
         }
          
         if ((scalar(grep(/9118/, @output))) || 
             (scalar(grep(/9119/, @output))) ||
             (scalar(grep(/9348/, @output))) > 0)
         {
            #We'll keep going if we need to reboot but we are in a development
            #environment.
            if (is_dev_env())
            {
               trace ("Skipping reboot in development env ... success");
               trace("ACFS drivers uninstall completed");
               writeCkpt($ckptName, CKPTSUC);
               return SUCCESS;
            }
            
            #
            # We couldn't unload the old drivers or load the new ones.
            trace("ACFS drivers unable to be uninstalled.");
            if ($CFG->DOWNGRADE)
            {    
               print color 'bold';
               print_error(434);
               print color 'reset';
               
               my $disable = "$crsctl disable $has";
               trace("$disable ... disabling CRS in preparation for reboot.");
               @output = system_cmd_capture($disable);
               $rc = shift @output;

               if ($rc != 0)
               {
                   trace("$disable ... unable to disable CRS for reboot.");
                   # Don't fail here - the user should correct
                   # the disable, by running it by hand, and reboot.
                   print_error(436);
               }
               else
               {
                   trace("$disable ... CRS disabled, ready for reboot.");
                   # At this point, CRS is disabled.  The user will need
                   # to reboot, rerun the operation.  This will rely on 
                   # OPatch and OUI to print the appropriate error message.
               }
            }
            elsif ($CFG->DECONFIG)
            {
               # In the deconfig process, before uninstalling
               # ACFS/ADVM Drivers the stack should be stopped.
               # But if there is a failure to stop the stack the process 
               # continues, and waits for it to shutdown after deconfiguration
               # If this happens, the ACFS/Drivers should be uninstalled
               # manually, when the stack is stopped, so we only send a message
               # And continue with the process
               trace ("Skipping uninstall failure in deconfig process.");
               trace("ACFS drivers uninstall failed");
               writeCkpt($ckptName, CKPTSUC);
               print_error(205);
               return SUCCESS;
            }

            print_error(205);
            if ($chkpoints == USECHKPOINTS)
            {
               writeCkpt($ckptName, CKPTFAIL);
            }
            exit 1;
         }   
      }
      else 
      {
         trace ("@cmd ... failed");
         print_error(205);
         if ($chkpoints == USECHKPOINTS)
         {
            writeCkpt($ckptName, CKPTFAIL);
         }
         exit 1;
      }
   }
   else
   {
      trace("USM driver uninstall skipped.");
   }
}

=head2 acfsrepl_updateResources

   Updates the replication resources to run at the upgraded version.
   Currently this upgrades from 11.2.0.2 on Linux. For this we do the
   following:
   - Stop all the replication resources.
   - Add the attributes PRIMARY_DEVICE and MOUNTPATH to the resources.

=head3 Parameters

   None

=head3 Returns

  None 

=head3 Notes
  
  Runs only on Linux

=cut
sub acfsrepl_updateResources
{
   my $crsctlbin = catfile ($CFG->ORA_CRS_HOME, "bin", "crsctl");
   my @repltypes = ('ora.acfsrepltransport.type', 'ora.acfsreplapply.type', 
                    'ora.acfsreplmain.type', 'ora.acfsreplinit.type');
   my @replres;
   my %repldevhash;
   my $type;
   my $cmd;
   my $status;

   if ($^O !~ /linux/i) 
   {
      # Replication is only supported starting with Linux 11.2.0.2
      return;
   }

   my @oldcrs_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if (!($oldcrs_ver[0] eq '11' && $oldcrs_ver[1] eq '2' && 
       $oldcrs_ver[2] eq '0'  && $oldcrs_ver[3] eq '2') ) 
   {
     # We aren't running on 11.2.0.2
     return;
   }

   # Find all of the replication resources on this node and put them in an array.
   foreach $type (@repltypes)
   {
      # Add the resource to the list
      acfsrepl_findRes($type, \@replres, \%repldevhash);
   }

   if(@replres == 0)
   {
     # Don't do anything if we don't have any resources
     return;
   }

   foreach $type (@repltypes)
   {
      # Add the PRIMARY_DEVICE attribute to the resource type
      $cmd = "$crsctlbin modify type $type -attr \"ATTRIBUTE=PRIMARY_DEVICE\",\"TYPE=string\",\"DEFAULT_VALUE=\",\"FLAGS=CONFIG\"";
      trace("Invoking \"$cmd\"");
      $status = system_cmd ("$cmd");
   }

   # Stop the resources
   trace("Stopping replication resources");
   foreach(0..$#replres)
   {
      trace("Stopping $replres[$_]");
      $cmd = "$crsctlbin stop resource $replres[$_] -unsupported";
      trace("Invoking \"$cmd\"");
      $status = system_cmd ("$cmd");
   }

   # Update the resources
   foreach(0..$#replres)
   {
      my $res = $replres[$_];
      my $dev = $repldevhash{$res};
      
      trace("Updating resource $res");
      if($dev ne "")
      {
         # Add the PRIMARY_DEVICE attribute to the resource
         $cmd = "$crsctlbin modify resource $res " .
                 "-attr \"PRIMARY_DEVICE=$dev\" -unsupported";
         trace ("Invoking \"$cmd\"");
         $status = system_cmd ("$cmd");
      }
   }
}

=head2 acfsrepl_findRes

   Finds all the replication resources of a given type.
   Adds the resources to an array
   Adds the resources primary device to a hash list.

=head3 Parameters

   type     - Resource type to search for.
   arrayref - Reference to array that holds the resource names found.
   hashref  - Reference to hash table that hold the primary devices found.

=head3 Returns

  None 

=head3 Notes

 - Only runs on Linux.
 
=cut
sub acfsrepl_findRes
{
  my $type     = shift;
  my $arrayref = shift;
  my $hashref  = shift;

  my $crsctlbin = catfile ($CFG->ORA_CRS_HOME, "bin", "crsctl");
  my $cmd = "$crsctlbin stat res -f -w \'TYPE = $type\'";
  my $devcmd = "/sbin/acfsutil info fs -o primaryvolume"; 

  my $line;
  my $res;

  open (OUT, "$cmd | ") or trace("Failed:$cmd($!)");
  while (<OUT>)
  {
    $line = $_;
    if($line =~ /^NAME=/)
    {
      $res = $';
      chomp($res);
      push(@$arrayref, $res);
    }
    if($line =~ /^MOUNTPOINT=/)
    {
      my $dev = `$devcmd $' 2>&1`;
      chomp($dev);
      ${$hashref}{ $res } = $dev;
    }
  }

}

=head2 acfsrepl_lastNode

   Does the replication upgrade processing on the last node.
   This is run AFTER the active version has been switched.
   Only runs on Linux upgrading from 11.2.0.2

=head3 Parameters

  Does the following:
  - Deletes the 'main' resource
  - Adds the new preapply and monitor types.
  - Gets all the replication resources and starts them

=head3 Returns

  None 

=head3 Notes

 - Only runs on Linux.
 
=cut

sub acfsrepl_lastNode
{
   my $crsctlbin        = catfile ($CFG->ORA_CRS_HOME, "bin", "crsctl");

   my $replpreapplytype = "ora.acfsreplpreapply.type";
   my $replmonitortype  = "ora.acfsreplmonitor.type";

   my @replres;
   my @repltypes = ('ora.acfsrepltransport.type', 'ora.acfsreplapply.type', 
                    'ora.acfsreplmain.type', 'ora.acfsreplinit.type');

   my %repldevhash;
   my $res;
   my $cmd;
   my $status;
   my $type;

   my $rc;
   my @lsnodes;
   my $node;
   
   if ($^O !~ /linux/i) 
   {
      # Replication is only support on Linux 11.2.0.2
      return;
   }

   my @oldcrs_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')};
   if (!($oldcrs_ver[0] eq '11' && $oldcrs_ver[1] eq '2' && 
       $oldcrs_ver[2] eq '0'  && $oldcrs_ver[3] eq '2') ) 
   {
     # We aren't running on 11.2.0.2
     return;
   }

   # Get the node list in the cluster
   ($rc, @lsnodes) = get_olsnodes_info($CFG->OLD_CRS_HOME); 

   # Find all of the replication resources on this node and put them in an array.
   foreach $type (@repltypes)
   {
      # Add the resource to the list
      acfsrepl_findRes($type, \@replres, \%repldevhash);
   }

   if(@replres == 0)
   {
     # Don't do anything if we don't have any resources
     return;
   }

   # Add the new resource types. monitor and preapply.
   my $addmoncmd = "$crsctlbin add type $replmonitortype -basetype cluster_resource -attr \"ATTRIBUTE=MOUNTPOINT\",\"TYPE=string\",\"DEFAULT_VALUE=\",\"ATTRIBUTE=PRIMARY_DEVICE\",\"TYPE=string\", \"DEFAULT_VALUE=\",\"ATTRIBUTE=AGENT_FILENAME\",\"TYPE=string\",\"DEFAULT_VALUE=%CRS_HOME%/bin/oraagent%CRS_EXE_SUFFIX%\",\"ATTRIBUTE=AUTO_START\",\"TYPE=string\",\"DEFAULT_VALUE=never\",\"ATTRIBUTE=PLACEMENT\",\"TYPE=string\",\"DEFAULT_VALUE=restricted\",\"ATTRIBUTE=SERVER_POOLS\",\"TYPE=string\", \"DEFAULT_VALUE=*\",\"ATTRIBUTE=SITE\",\"TYPE=string\", \"DEFAULT_VALUE=\",\"ATTRIBUTE=START_TIMEOUT\",\"TYPE=int\", \"DEFAULT_VALUE=120\"";
   trace ("Invoking \"$addmoncmd\"");
   $status = system_cmd ("$addmoncmd");


   my $addpreappcmd = "$crsctlbin add type $replpreapplytype -basetype cluster_resource -attr \"ATTRIBUTE=MOUNTPOINT\",\"TYPE=string\",\"DEFAULT_VALUE=\",\"ATTRIBUTE=PRIMARY_DEVICE\",\"TYPE=string\", \"DEFAULT_VALUE=\",\"ATTRIBUTE=AGENT_FILENAME\",\"TYPE=string\",\"DEFAULT_VALUE=%CRS_HOME%/bin/oraagent%CRS_EXE_SUFFIX%\",\"ATTRIBUTE=AUTO_START\",\"TYPE=string\",\"DEFAULT_VALUE=never\",\"ATTRIBUTE=PLACEMENT\",\"TYPE=string\",\"DEFAULT_VALUE=restricted\",\"ATTRIBUTE=SERVER_POOLS\",\"TYPE=string\", \"DEFAULT_VALUE=*\",\"ATTRIBUTE=SITE\",\"TYPE=string\", \"DEFAULT_VALUE=\",\"ATTRIBUTE=START_TIMEOUT\",\"TYPE=int\", \"DEFAULT_VALUE=120\"";
   trace ("Invoking \"$addpreappcmd\"");
   $status = system_cmd ("$addpreappcmd");

   for(0..$#replres)
   {
      my $index = $_;
      $res = $replres[$index];

      if($res =~ /ora.repl.main./)
      {
         my $ressuffix = $';
         my $isStandby = 0;
         my $mntpt;
         my $pridev;
         $cmd = "$crsctlbin stat res $res -f";

         trace("Invoking \"$cmd\"");
         open (OUT, "$cmd | ") or die "Cannot run command:$cmd($!)\n";
         while (<OUT>)
         {
           if($_ =~ /^SITE=standby/)
           {
             $isStandby = 1;
           }
           if($_ =~ /^MOUNTPOINT=/)
           {
             $mntpt = $';
             chomp($mntpt);
           }
           if($_ =~ /PRIMARY_DEVICE=/)
           {
             $pridev = $';
             chomp($pridev);
           }
         }

         # Delete the main resource from the array.
         delete $replres[$index];

         # Add the new resources
         if($isStandby)
         {
           acfsrepl_addResource("ora.repl.preapply." . $ressuffix, $replpreapplytype, $mntpt, $pridev, $isStandby);
           push(@replres, "ora.repl.preapply." . $ressuffix);
         }
         acfsrepl_addResource("ora.repl.monitor." . $ressuffix,  $replmonitortype, $mntpt, $pridev, $isStandby);
         push(@replres, "ora.repl.monitor." . $ressuffix);

         # Delete the main resource
         $cmd = "$crsctlbin delete resource $res -unsupported";
         trace("Invoking \"$cmd\"");
         $status = system_cmd ("$cmd");
    }

  }

  # Start all of the resources in the list
  foreach(0..$#replres)
  {
    my $res = $replres[$_];
    if(defined($res))
    {
      # Start the transport daemon on each node.
      if($res =~ /ora.repl.transport./)
      {
         foreach $node (@lsnodes)
         {
           $cmd = "$crsctlbin start resource $res -n $node -unsupported";
           trace("Invoking \"$cmd\"");
           $status = system_cmd ("$cmd");
         }
      }
      else
      {
        $cmd = "$crsctlbin start resource $res -unsupported";
        trace("Invoking \"$cmd\"");
        $status = system_cmd ("$cmd");
      }
    }
  }
 
}

sub acfsrepl_addResource
{
  my $resname   = shift;
  my $type      = shift;
  my $mntpt     = shift;
  my $pridev    = shift;
  my $isStandby = shift;
  my $site;
  my $cmd;
  my $status;
  my $crsctlbin = catfile ($CFG->ORA_CRS_HOME, "bin", "crsctl");

  if($isStandby == 1)
  {
    $site = "standby";
  }
  else
  {
    $site = "primary";
  }

  # Add the resource
  $cmd = "$crsctlbin add resource $resname -type $type -attr \"MOUNTPOINT=$mntpt\",\"PRIMARY_DEVICE=$pridev\",\"SITE=$site\" -unsupported";
  trace("Invoking \"$cmd\"");
  $status = system_cmd ("$cmd");
}

=head2 actionACFSDriversResource

   Create/Start/Delete the ACFS Drivers Resource. 

=head3 Parameters

   $action - "stop"/"add"/"start"/"delete" actions for the drivers resource

=head3 Returns

  TRUE  - Creation/Start/Delete successful
  FALSE - Creation/Start/Delete failed

=head3 Notes


=cut
sub actionACFSDriversResource
{
  my $action  = shift;
  my $crsctl  = catfile ($CFG->ORA_CRS_HOME, "bin", "crsctl");
  my $ret     = FAILED;

  if (isACFSSupported()) 
  { 
    my @out  = system_cmd_capture($crsctl, "stat", "res", "ora.drivers.acfs",
                                  "-init");
    my $notExists = grep (/CRS-2613/i, @out);
    my $status    = grep (/TARGET/i, @out);

    if (scalar($notExists) > 0 && 
        ($action eq "add" || $action eq "start"))
    {
      my $user          = $CFG->params('ORACLE_OWNER');
      my $asmgrp        = $CFG->params('ORA_ASM_GROUP');
      my $asmowner      = $user;
      
      if ($CFG->platform_family ne "windows")
      {
        $asmowner = $CFG->SUPERUSER;
      }
      
      # Handle Windows case
      if ($CFG->platform_family eq "windows") 
      {
        my $NT_AUTHORITY = '';
        if (is_dev_env()) 
        {
          $NT_AUTHORITY = $CFG->params('ORACLE_OWNER');
        }
        
        if (! $NT_AUTHORITY ) 
        {
          $asmowner =  '';  
          $user     =  '';
          $asmgrp   =  '';
        }
        else
        {
          $asmowner =  $NT_AUTHORITY;  
          $user     =  $NT_AUTHORITY;
        }
      }

      my @acfs_attr = 
          ("ACL='owner:$asmowner:rwx,pgrp:$asmgrp:r-x,other::r--," .
           "user:$user:r-x'");

      if ($CFG->platform_family eq 'windows')
      {
        push @acfs_attr,
             "ACTIONS='mc_refresh ".
                      "mc_rescan ".
                      "mc_aggregate'";
      }
      else
      {
        push @acfs_attr,
             "ACTIONS='mc_refresh,user:$user "."mc_rescan,user:$user ".
             "mc_aggregate,user:$user'";
      }

      $ret = crsctlResource("add", 
                            "ora.drivers.acfs", 
                            "ora.drivers.acfs.type", \@acfs_attr);
    }
    elsif (scalar($status) > 0  && $action eq "delete")
    {
      $ret = crsctlResource($action, "ora.drivers.acfs");
    }
    elsif (scalar($status) > 0  && $action eq "stop")
    {
      # If the resource doesn't exist, then the catch all 'else' will
      # ensure that we return success.  Otherwise, we try and stop it.
      system_cmd_capture($crsctl, "stop", "res", "ora.drivers.acfs", "-init");
      $ret = SUCCESS;
    }
    else
    {
      # Make sure 'crsctl stat res' command ran properly
      if (scalar($status) > 0 || scalar($notExists) > 0) 
      {
        # The resource already exists or was already deleted  
        $ret = SUCCESS; 
      }
      else
      { 
        # There was an error running the command
        $ret = FAILED; 
      }
    }

    if ($action eq "start")
    {
      # Get the status again just to be sure  
      @out  = system_cmd_capture($crsctl, "stat", "res", "ora.drivers.acfs",
                                 "-init");
      my $statusOnline  = grep (/TARGET=ONLINE/i,  @out);
      my $statusOffline = grep (/TARGET=OFFLINE/i, @out);
      
      if (scalar($statusOnline) > 0) 
      {
        # The resource has been started already  
        $ret = SUCCESS; 
      }
      elsif (scalar($statusOffline) > 0 )
      { 
        # The resource has not been started 
        $ret = crsctlResource($action, "ora.drivers.acfs");
      }
      else
      {
        # The command failed or resource was not created
        $ret = FAILED;
      }
    }
  }
  else {
     # if ACFS Drivers are not supported, nothing to do
     # We return SUCCESS to continue.
     $ret = SUCCESS;
  }
  return $ret;
}

=head2 isACFSPath

   Check if the path is on an ACFS Mountpoint. 

=head3 Parameters

   $path - Path to check

=head3 Returns

  TRUE  - If $path is on an ACFS Mountpoint.
  FALSE - If not

=head3 Notes


=cut
sub isACFSPath
{
    my $path = shift;
    my $cmd;
    my $status;
    my $result;
    my $acfsutil;
    if ($CFG->platform_family eq 'windows') {
        $acfsutil = catfile ($CFG->ORA_CRS_HOME, 'bin', 'acfsutil');
    }
    else {
      $acfsutil = "/sbin/acfsutil";
    }

    # If path is not defined, empty or contains only white-space characters, 
    # return FALSE
    if ((not defined($path)) ||
        ($path =~ /^\s*$/))
    {
       return FALSE;
    }

    $cmd = "$acfsutil info fs \"$path\" -o mountpoint";
    if ($CFG->platform_family eq 'windows') {
        $cmd =~ s/\ -/\ \//g ;
    }

    if (!isACFSInstalled(1) || 
        !isACFSLoaded(1))
    {
       # Check if ACFS is installed and loaded. 
       # If not installed or loaded, ACFS file systems 
       # cannot be mounted, hence there is no ACFS path.
       return FALSE;
    }

    # check if acfsutil path is available
    if (! (-e $acfsutil)) {
       trace ("$acfsutil not found");
       return FALSE;
    }

    my @output;
    ($status, @output) = system_cmd_capture($cmd);
    if ($status != 0)
    {
      # We can fail for many different things:
      # * acfsutil info fs: ACFS-03036: no mounted ACFS file systems and
      #   we got $status equal to 1
      # * acfsutil info fs: CLSU-00101: operating system error message: 
      #   No such file or directory. We got $status equal to 1
      # Maybe acfsutil doesn't exist. It could happen if ACFS is not installed
      # or supported.
      # Any case, we return FALSE, because we can not determine if it's part of
      # an acfs MountPoint
      return FALSE;
    }
    #If $status is equal to 0. We should get the mountpoint in the first line
    $result = shift @output;
    #Just for validation. $result should be equal or a substring in the path
    if ($path =~ /\Q$result\E/)
    {
      return TRUE;
    }
    return FALSE;
}

=head2 isPIDACFSPath
 
 Get paths for all related PIDs, where a shell was started.
  
=head3 Parameters
      
=head3 Returns
        
 Array with all related paths for actual shell.
           
=head3 Notes
             
=cut
sub isPIDACFSPath
{
  my @pspwdx    = "";
  my @psunique  = "";
  my @pstree    = "";
  my @pstreeval = "";
  my @psreturn  = "";
  my %psseen    = "";
  my $pwd       = $ENV{'PWD'}?$ENV{'PWD'}:getcwd();
  my $OS        = $Config{osname};
  my $ps        = "ps | sort | egrep -v \"ps|egrep|sort|PID\" |";

  if ($OS eq "linux" || $OS eq "solaris" || $OS eq "aix"){
    open (PS, "$ps") or die(dieformat(180, $ps));
    @pstree = <PS>;
    foreach my $value (@pstree) {
      $value =~ s/^\s+//g;
      @pstreeval = split(/ /, $value);

      if ($OS eq "aix")
      {
        my $temp = system("procwdx $pstreeval[0] >/dev/null 2>&1 ");
        if ($temp eq "0"){
           @pspwdx = split(/:/, `procwdx $pstreeval[0]`);
         }
      }
      else
      {
        my $temp = system("pwdx $pstreeval[0] >/dev/null 2>&1 ");
        if ($temp eq "0"){
          @pspwdx = split(/:/, `pwdx $pstreeval[0]`);
        }
      }
      $pspwdx[1] =~ s/\s//g;

      if ($pspwdx[1] =~ /\//){
        push (@psreturn, $pspwdx[1]);
      }
    }
    @psunique = grep { !$psseen{$_}++ } @psreturn; # No repetitions
    return (@psunique);
  }
  push (@psreturn, $pwd);
  return (@psreturn);  # Return empty value
  #  Windows doesn't have a parent-child relationship between processes
}

=head2 disableADVMProxy

   Disable and Stop resource ora.proxy_advm.  This is done only if there are
   no ADVM resources\volumes in the cluster.  In 12.2, the proxy should be
   offline and disabled if that is the case.

=head3 Parameters

=head3 Returns

=head3 Notes

=cut
sub disableADVMProxy{
  #Before disabling ADVM Proxy, need to check if there are volumes
  my $status;
  my @out;
  my $srvctl_args = "status volume";

  trace ("Checking if we should disable the ASM Proxy");
  $status = srvctl_capture(FALSE, \@out, $srvctl_args);

  if(($status != 0) && 
     ($status != 2) &&
     (!scalar(grep(/PRCA-1051/, @out))))
  {
    print_lines(@out);
    print_info(180, "srvctl $srvctl_args");
  }

  if ( scalar(grep(/PRCA-1051/, @out) ) )
  {
    #  We need to stop it before removing the resource
    $srvctl_args = "status asm -proxy";
    $status = srvctl_capture(FALSE, \@out, $srvctl_args);
    if ( scalar(grep(/PRCR-1001/, @out) ) )
    {
       # The resource doesn't exist.  
       # We need to do nothing.
       trace("ADVM Proxy resource doesn't exist.");
    }
    else
    {
      if ( scalar(grep(/is running/, @out) ) )
      {
        # We need to stop before removing 
        $srvctl_args = "stop asm -proxy";
        if (srvctl(FALSE, $srvctl_args))
        {
          # The resource was stopped.  
          trace("ADVM Proxy resource is not running.");
        }
        else
        {
          # Some error occurred attempting to run srvctl
          # But we don't stop the upgrade process
          trace("Error running srvctl $srvctl_args command");
        }
      }

      $srvctl_args = "disable asm -proxy";
      if (srvctl(FALSE, $srvctl_args))
      {
        # The resource was removed.  
        trace("ADVM Proxy resource was disabled.");
      }
      else
      {
        # Some error occurred attempting to run srvctl
        # But we don't stop the upgrade process
        trace("Error running srvctl $srvctl_args command");
      }
    }
  }
  else
  {
    # We have volumes, we can not remove the ora.proxy_advm resource
    trace("ADVM Proxy resource can not be disabled as volumes exist.");
  } 
}

=head2 start_filesystem_resources

   Start Filesystem resources. 

=head3 Parameters

   None

=head3 Returns

  None

=head3 Notes

=cut
sub start_filesystem_resources
{
    if (isACFSSupported())
    {
        my $cmd;
        my $crsctl = crs_exec_path('crsctl');
        # Start Filesystems resources on the cluster
        $cmd = "$crsctl start resource -w \"TYPE = ora.acfs.type\" -unsupported";
        trace("Running $cmd.");
        # The filesystem resources could be started in this case we can get
        # result different to 0. But it's ok because we are trying to start all 
        # the resources. This is the reason that we don't check the result.
        # Since the failures from this command execution are not fatal, the
        # command output should just be kept in the trace log in case the
        # error messages outputting to the console confuse users. 
        system_cmd_capture($cmd);
    }
}

=head1 EXPORTED FUNCTIONS       
 
=head2 getACFSVolumeDevice
 
  Get the volume device used by specified ACFS resource
 
=head3 Parameters       
 
  [0] The name of given ACFS resource
  [1] CRS home
                
=head3 Returns  
                        
  The volume device used by the given ACFS resource

=cut

sub getACFSVolumeDevice
{
  my $self = shift;
  my $acfsres = $_[0];
  my $crshome = $_[1];
  my $vol_dev;

  trace("Attempt to find the volume device used by '$acfsres'");

  my $CRSCTL;
  if ($crshome)
  {
    $CRSCTL = catfile($crshome, 'bin', 'crsctl');
  }
  else
  {
    $CRSCTL = crs_exec_path('crsctl');
  } 
  
  my @cmd = ($CRSCTL, 'stat', 'res', $acfsres, '-f');
  my @out = system_cmd_capture(@cmd);
  my $rc = shift @out;
  
  if (0 != $rc)
  {
    print_lines(@out);
    trace("crsctl stat res $acfsres -f failed with status $rc");
    die(dieformat(180, "crsctl stat res $acfsres -f"));
  }

  my @res_attr = grep(/VOLUME_DEVICE/, @out);
  if (scalar(@res_attr) > 0)
  {
    my @vols = split(/=/, $res_attr[0]);
    $vol_dev = $vols[1];
  }
  else
  {
    # We are expecting to have VOLUME_DEVICE just 
    # for ora.acfs.type or ora.fs.type, 
    # any other we'll return undef
    if ((scalar(grep(/TYPE=ora\.acfs\.type/, @out)) > 0) ||
        (scalar(grep(/TYPE=ora\.fs\.type/, @out)) > 0))
    {    
      die(dieformat(604, $acfsres));
    }
    else
    {
      trace("The ACFS Resource '$acfsres' does not use volume device");
      return undef;
    }
  }

  trace("The ACFS '$acfsres' uses the volume device [$vol_dev]");
  return $vol_dev;
}

=head2 reset_ACFS_wrapper_scripts

   When cloning the GI Home, the home is copied to a new location.  
   If the scripts are already instantiated, then they will not be 
   updated with the correct oracle home. 
   This resets the ACFS scripts so that when we later call acfsroot, 
   acfsroot updates the home correctly.

=head3 Parameters

   None

=head3 Returns

  None

=head3 Notes

=cut
sub reset_ACFS_wrapper_scripts
{
    my $self = shift;
    my $myPath = $CFG->ORA_CRS_HOME;
    my (@progs);
    my $ckptName = "ROOTCRS_ACFSINST";
    if ( !$CFG->SIHA )
    {
        if (isCkptexist($ckptName)) 
        {
            my $ckptStatus = getCkptStatus($ckptName);
            trace("'$ckptName' state is '$ckptStatus'");
            if (isCkptSuccess($ckptName))
            {
                trace("ACFS is already initialized, skipped reset_ACFS_wrapper_scripts function");
                return;
            }
        }
    }

    trace("Running reset_ACFS_wrapper_scripts function");

    if ($CFG->platform_family eq 'windows') 
    {
        (@progs) = (
                    "$myPath\\bin\\acfsdriverstate.bat",
                    "$myPath\\bin\\acfsload.bat",
                    "$myPath\\bin\\acfsregistrymount.bat",
                    "$myPath\\bin\\acfssinglefsmount.bat",
                    "$myPath\\bin\\acfsreplcrs.bat"
                  );
    }
    else
    {
        (@progs) = (
                    "$myPath/bin/acfsdriverstate",
                    "$myPath/bin/acfsload",
                    "$myPath/bin/acfsregistrymount",
                    "$myPath/bin/acfssinglefsmount",
                    "$myPath/bin/acfsreplcrs",
                    "$myPath/bin/acfsrepl_apply"
                   );
    }

    my ($prog); 
    my ($line);
    my (@buffer);

    foreach $prog (@progs)
    {
        my ($read_index, $write_index);

        $read_index = 0;
        open READ, "<$prog" or next;
        while ($line = <READ>)
        {
            if ($CFG->platform_family eq 'windows') 
            {
                if ($line =~ m/^set CRS_HOME=/)
                {
                    $line = "set CRS_HOME=\%~dp0..\n";
                }
            }
            else
            {
                if ($line =~ m/^ORA_CRS_HOME/)
                {
                    $line = "ORA_CRS_HOME=\%ORA_CRS_HOME\%\n";
                }
            }
            $buffer[$read_index++] = $line;
        }
        close (READ);
    
        $write_index = 0;
        open WRITE, ">$prog" or next;
        while($write_index < $read_index)
        {
            print WRITE "$buffer[$write_index++]";
        }
        close (WRITE);
    }
}

=head2 isACFSSupported

   Determines if this platform is an ACFS supported platform
   by calling 'acfsdriverstate supported'.

=head3 Parameters

   None

=head3 Returns

  TRUE  - ACFS Supported
  FALSE - ACFS Not Supported

=head3 Notes


=cut
sub isACFSSupported
{
   my $ACFS_supported = FALSE;
   my $acfsdriverstate;

   if (defined $CFG->ACFSSupported)
   {
      $ACFS_supported = $CFG->ACFSSupported;
      trace ("isACFSSupported: $ACFS_supported");
      return $ACFS_supported;
   }

   if ($CFG->platform_family eq 'windows') 
   {
      $acfsdriverstate = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfsdriverstate.bat');
   }
   else
   {
      $acfsdriverstate = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfsdriverstate');
   }

   # check if acfs is supported
   if (! (-e $acfsdriverstate)) 
   {
      trace ("$acfsdriverstate not found");
      goto CACHE_ACFSSUPPORTED;
   }

   my @cmd = ($acfsdriverstate, "supported");
   my @out = system_cmd_capture(@cmd);
   my $rc  = shift @out;

   if ($rc == 0) 
   {
      if (scalar(grep(/9200/, @out)) > 0) 
      {
         $ACFS_supported = TRUE;
         trace ("acfs is supported");
      }
      else
      {
         $ACFS_supported = FALSE;
         trace ("acfs is not supported");
      }
   }
   else 
   {
      # If result code is equal to 1.
      # It could be because acfs is not supported or
      # for a transient fail. We'll check for a error
      # first.
      # 9305 - ADVM/ACFS installation cannot proceed
      # 9389 - ORACLE_HOME is not set to the location 
      #        of the Grid Infrastructure home.
      if ((scalar(grep(/9305/, @out)) > 0) ||
          (scalar(grep(/9389/, @out)) > 0)) 
      {
         trace ("acfsdriverstate supported failed");
         trace (@out);
         die(dieformat(609));
      }
      else
      {
         $ACFS_supported = FALSE;
         trace ("acfs is not supported");
      }      
   }

CACHE_ACFSSUPPORTED:
   $CFG->ACFSSupported($ACFS_supported);
   return $ACFS_supported;
}

=head2 isACFSInstalled

   Determines wheter ACFS is installed
   by calling 'acfsdriverstate installed'.

=head3 Parameters

   Silent Option

=head3 Returns

  TRUE  - ACFS Installed
  FALSE - ACFS Not Installed

=head3 Notes


=cut
sub isACFSInstalled
{
   my $silent = shift;
   my $ACFS_installed = FALSE;
   my $acfsdriverstate;

   if ($CFG->platform_family eq 'windows') {
      $acfsdriverstate = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfsdriverstate.bat');
   }
   else {
      $acfsdriverstate = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfsdriverstate');
   }

   # check if acfs is installed
   if (! (-e $acfsdriverstate)) {
      trace ("$acfsdriverstate not found");
      return FALSE;
   }

   my @cmd = ($acfsdriverstate, "installed");
   if ($silent)
   {
      push @cmd, '-s';     
   }
   trace("Running @cmd");
   my @out = system_cmd_capture(@cmd);
   my $rc  = shift @out;

   if ($rc == 0) {
      $ACFS_installed = TRUE;
      trace ("acfs is installed");
   }
   else {
      $ACFS_installed = FALSE;
      trace ("acfs is not installed");
   }

  return $ACFS_installed;
}

=head2 isACFSLoaded

   Determines wheter ACFS is loaded
   by calling 'acfsdriverstate loaded'.

=head3 Parameters

   Silent Option

=head3 Returns

  TRUE  - ACFS Loaded
  FALSE - ACFS Not Loaded

=head3 Notes


=cut
sub isACFSLoaded
{
   my $silent = shift;
   my $ACFS_loaded = FALSE;
   my $acfsdriverstate;

   if ($CFG->platform_family eq 'windows') {
      $acfsdriverstate = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfsdriverstate.bat');
   }
   else {
      $acfsdriverstate = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfsdriverstate');
   }

   # check if acfs is loaded
   if (! (-e $acfsdriverstate)) {
      trace ("$acfsdriverstate not found");
      return FALSE;
   }

   my @cmd = ($acfsdriverstate, "loaded");
   if ($silent)
   {
      push @cmd, '-s';     
   }
   trace("Running @cmd");

   my @out = system_cmd_capture(@cmd);
   my $rc  = shift @out;

   if ($rc == 0) {
      $ACFS_loaded = TRUE;
      trace ("acfs is loaded");
   }
   else {
      $ACFS_loaded = FALSE;
      trace ("acfs is not loaded");
   }

  return $ACFS_loaded;
}

#   This code contains the functionality to install the USM
# driver by calling code in the oraacfs.pm module.
# 
# ARGS:
#  $chkpoints - 0 or 1 In SIHA, there is no chkpoint functionality.
#      0 - NOTUSECHKPOINTS - Don't use checkpoints
#      1 -    USECHKPOINTS - Go ahead and use checkpoints.
sub perform_installUSMDriver
{
   my $chkpoints = $_[0];
   my $has;
   my $ckptName = "ROOTCRS_ACFSINST";
   if ($chkpoints == NOTUSECHKPOINTS || $CFG->SIHA) {
     $has = "has";
     $ckptName = "ROOTHAS_ACFSINST";
   }
   else {
     $has = "crs";
   }

   if ($chkpoints == USECHKPOINTS)
   {
       if (!isCkptexist($ckptName)) {
          trace("Writing checkpoint for USM driver install");
          writeCkpt($ckptName, CKPTSTART);
          $CFG->wipCkptName($ckptName);
       }
       else {
          my $ckptStatus = getCkptStatus($ckptName);
          trace("'$ckptName' state is '$ckptStatus'");

          if (isCkptSuccess($ckptName))
          {
             trace("ACFS is already initialized");
          }
       }
   }
   else {
       trace("Not using checkpoint for USM driver install");
   }

   if (($chkpoints != USECHKPOINTS) ||  (!isCkptSuccess($ckptName)) )
   {
      $CFG->wipCkptName($ckptName);

      if (USECHKPOINTS == $chkpoints && !($CFG->SIHA))
      {
        # Only ohasd needs to be up before installing acfs drivers
        bounce_ohasd();
      }
   
      # install acfsroot kernel
      my $ret = installUSMDriver($has);
      if (FAILED == $ret)
      {
         # This is some failure that doesn't relate to driver load\unload.
         # For instance, this could be an issue generating symbols.
         print_error(196);
         if ($chkpoints == USECHKPOINTS)
         {
           writeCkpt($ckptName, CKPTFAIL);
         }
         exit 1;
      }
      elsif (REBOOT == $ret)
      {
         # We couldn't unload the old drivers or load the new ones.
         # CRS is now disabled, so the user can reboot and try again.
         trace("ACFS drivers unable to be installed.");
   	 set_bold();
         print_error(400);
   	 reset_bold();

         writeCkpt($ckptName, CKPTFAIL);
         exit 1;
      }
      else
      {
         # It worked!
         trace("ACFS drivers installation completed");
         if ($chkpoints == USECHKPOINTS)
         {
           writeCkpt($ckptName, CKPTSUC);
           $CFG->wipCkptName("ROOTCRS_STACK");
         }
         # In SIHA, We need to restart OHASD after
         # ACFS Install is successful
         # NOTUSECHKPOINTS remains here because it's in use in siha upgrade.
         elsif ($chkpoints == NOTUSECHKPOINTS || $CFG->SIHA)
         {
           writeCkpt($ckptName, CKPTSUC);

           trace("Restarting OHASD in SIHA");
           bounce_ohasd_SIHA();
         }
      }
   }
   else
   {
      trace("ACFS drivers installation skipped");
   }
}

=head2 isHANFSInstalled

   Determines whether HANFS is installed
   by calling 'acfshanfs installed'.

=head3 Parameters

=head3 Returns

  TRUE  - HANFS Installed
  FALSE - HANFS Not Installed

=head3 Notes


=cut
sub isHANFSInstalled
{
   my $HANFS_installed = FALSE;
   my $acfshanfs;

   # This command only run in Linux.   
   $acfshanfs = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfshanfs');

   # check if acfshanfs is installed
   if (! (-e $acfshanfs)) {
      trace ("$acfshanfs not found");
      return FALSE;
   }

   my $cmd = "$acfshanfs " . "installed -nfsv4lock";
   trace("Running $cmd");
   my ($status, @out) = system_cmd_capture($cmd);
   if ( scalar(grep(/ACFS-9203/, @out) ) )
   {
      $HANFS_installed = TRUE;
      trace("acfshanfs is installed");
   }
   else
   {
      $HANFS_installed = FALSE;
      trace("acfshanfs is not installed");
   }
   
   return $HANFS_installed;
}

=head2 isHANFSSupported

   Determines whether HANFS is supported
   by calling 'acfshanfs supported'.

=head3 Parameters

=head3 Returns

  TRUE  - HANFS Supported
  FALSE - HANFS Not Supported

=head3 Notes


=cut
sub isHANFSSupported
{
   my $HANFS_supported = FALSE;
   my $acfshanfs;

   $acfshanfs = catfile($CFG->ORA_CRS_HOME, 'bin', 'acfshanfs');
   
   # check if acfshanfs is supported
   if (! (-e $acfshanfs)) {
      trace ("$acfshanfs not found");
      return FALSE;
   }
   
   my $cmd = "$acfshanfs " . "supported -nfsv4lock";
   trace("Running $cmd");
   my ($status, @out) = system_cmd_capture($cmd);
   if ( scalar(grep(/ACFS-9200/, @out) ) )
   {
      $HANFS_supported = TRUE;
      trace("acfshanfs is supported");
   }
   else
   {
      $HANFS_supported = FALSE;
      trace("acfshanfs is not supported");
   }
   
   return $HANFS_supported; 
}
  
=head2 getNetStorageService

   Determines whether ora.netstorageservice exists
   by calling 'srvctl status netstorageservice',
   if the resource is running and the volume device

=head3 Parameters

=head3 Returns

  VOLUME DEVICE 

  RESOURCE is running
  
  RESOURCE exists.

=head3 Notes


=cut
sub getNetStorageService
{
   my $isrunningFS = FALSE;
   my $existsFS = FALSE;
   my $volume = "";

   my $crsctlBin = catfile ($CFG->ORA_CRS_HOME, "bin", "crsctl");

   # check if oresource exists
   my $status;
   my @out;
   my $srvctl_args = "status netstorageservice";
   $status = srvctl_capture(FALSE, \@out, $srvctl_args);
   if (scalar(grep(/PRCR-1001/, @out)))
   {
      # The resource doesn't exist.
      trace( "Resource ora.netstorageservice does not exist.");
   }
   else
   {
      # The resource exists.
      trace( "Resource ora.netstorageservice exists.");
      my $volumeResourceDependency = "ora.netstorageservice";
      # Check for filesystem resource
      open CRSCTL, "$crsctlBin stat res $volumeResourceDependency -f |";
      while (<CRSCTL>)
      {
         if (/CRS-4047/)
         {
            # "No Oracle Clusterware components configured.". We can not 
            # get this error, because srvctl run with no issues.
            # Just a validation.
            trace("We can not proceed because CRS stack is not up.");
            last;
         }
         elsif (/CRS-2613/)
         {
            # "Could not find resource". We can not get this error,
            # because srvctl run with no issues.
            # Just a validation.
            $existsFS = FALSE;
            last;
         }
         elsif (/STATE=OFFLINE/)
         {
            $existsFS = TRUE;
         }
         elsif (/STATE=ONLINE/)
         {
            $existsFS = TRUE;
            $isrunningFS = TRUE;
         }
         elsif (/STABLE_STORAGE/)
         {
            $volume = ($_);
            chomp($volume);
            $volume = substr($volume, index($volume, '=') + 1);
         }
      }
      close CRSCTL;
   }
   return ($volume, $isrunningFS, $existsFS);
}


=head2 

    Check if we are installing any of the Member Cluster / Domain Services
    Cluster classes/types.

    To determine the necessary information, we use following functions: 

    1) Local ASM Member Cluster - Default case. Do nothing.
    2) Local ASM Cluster        - Default case. Do nothing.
    3) App Member Cluster       - $isMemberClass
    4) DB Member Cluster        - $isMemberClass
    5) ODA domU                 - isODADomu()
    6) ODA dom1                 - isODA()
    7) OPC domU                 - isOPCDomu()
    8) OPC dom0                 - isOPCDom0() 
    9) Domain Cluster           - $isDomainClass

    In any DomU mode or Member Class, we need to enable the CCM/B using srvctl.

=cut
sub ACFS_MemDomSetup
{

    my $set_up_acfsremote = FALSE;
    my $set_up_ccmb = FALSE;
    my $run_as_owner = FALSE;
    my $cluster_class = crsutils::getClusterClass_crsctl();
    my $isDomainClass = $cluster_class eq CLUSTER_CLASS_DOMAINSERVICES ? 
                        TRUE : FALSE;
    my $isMemberClass = $cluster_class eq CLUSTER_CLASS_MEMBER ? 
                        TRUE : FALSE;

    my $acfsutil;
    if ($CFG->platform_family eq 'windows') {
        $acfsutil = catfile ($CFG->ORA_CRS_HOME, 'bin', 'acfsutil');
    }
    else {
      $acfsutil = "/sbin/acfsutil";
    }

    if($isMemberClass  ||        # App | DB Member Cluster
       $isDomainClass  ||        # Domain Cluster
       isODA()         ||        # ODA
       isODADomu()     ||        # ODA DomU
       isOPCDomu()     ||        # OPC DomU
       isOPCDom0()     ||        # OPC Dom0
       isFarASM()   
      )
    {
        $set_up_acfsremote = TRUE;
    }

    trace("setup acfsremote = $set_up_acfsremote ");

    if($set_up_acfsremote == TRUE)
    {
        my $user = $CFG->params('ORACLE_OWNER');
        # Execute this command so the ACFS OCR key is created.
        my $cred_cmd = "$acfsutil cluster credential -s $user"; 
        trace("Executing '$cred_cmd'");
        my @cred_output = system_cmd_capture($cred_cmd);

        # There really shouldn't be any problem...
        my $cred_rc = shift @cred_output;    
        # PROCR_SUCCESS(0) should be returned
        if($cred_rc != 0)
        {
            trace ("Unable to set up ACFS Remote credentials domain.");
            print_lines("@cred_output");
        }
    }

    # For now, this only runs on linux
    if($Config{osname} eq 'linux')
    {

        if($isMemberClass  ||        # App | DB Member Cluster
           isOPCDomu()     ||        # OPC DomU
           isODADomu()               # ODA DomU
          )               
        {
            $set_up_ccmb = TRUE;
        }

        trace("setup ccmb = $set_up_ccmb ");


        # FIXME 22735932 - REPORT ERROR SETTING UP ACFS REMOTE IN GRID INSTALLER
        if(($set_up_ccmb == TRUE) && isFarASM())
        {
            my $cred_location = $CFG->params('ASM_CREDENTIALS');
            if(!$cred_location)
            {
                my $cred_cmd = "$acfsutil cluster credential -i $cred_location";
                chomp($cred_cmd);
                my @cred_output = system_cmd_capture($cred_cmd);

                my $cred_rc = shift @cred_output;    

                if($cred_rc != 0)
                {
                    # Check why we failed.
                    if(scalar(grep(/ACFS-09827/,@cred_output)) > 0)
                    {
                        trace("ACFS Remote credentials were not found in the provided Cluster Manifest File.");
                        print_lines("@cred_output");
                    }
                    else
                    {
                        trace("Failed to import ACFS Remote credentials.");
                        print_lines("@cred_output");
                    }

                }
            }

            if(defined $ENV{USM_ENABLE_CCMB_INSTALL})
            {
                my @ccmb_output = srvctl($run_as_owner,"add ccmb");
                my $ccmb_ret = shift @ccmb_output;
                if ($ccmb_ret != 0)
                {
                    trace ("Unable to add ccmb.");
                    print_lines("@ccmb_output");
                }
                @ccmb_output = srvctl($run_as_owner,"start ccmb");
                $ccmb_ret = shift @ccmb_output;
                if ($ccmb_ret != 0)
                {
                    trace ("Unable to start ccmb.");
                    print_lines("@ccmb_output");
                }
            }
            else
            {
                trace("Skipping CCMB Install");
            }
        }
    }
}


=head2 enableAndStartADVMACFSonNode
   
    This subroutine will enable on the $CFG->NODE node 
      advm and acfs resources

=head3 Parameters

=head3 Returns

=head3 Notes


=cut
sub enableAndStartADVMACFSonNode
{
   my $acfs_res_list = $CFG->ACFS_ADVM_RESOURCES_LIST;
   my $node = $CFG->HOST;

   my @volDevices;

   trace("ACFS/ADVM resources to be enabled and started: [$acfs_res_list]");
   my @acfsres = split(/,/, $acfs_res_list);

   my $CRSCTL = crs_exec_path('crsctl');

   foreach my $acfs (@acfsres)
   {
     trace("Enabling '$acfs' ...");
     my $attr = "\"ENABLED\@SERVERNAME($node)=1\"";
     my @cmd  = ($CRSCTL, 'modify', 'resource', $acfs, '-attr', $attr,
                  '-unsupported');
     my @out  = system_cmd_capture(@cmd);
     my $rc = shift @out;
     if (0 != $rc)
     {
       print_lines(@out);
       my $cmdstr = join(' ', @cmd);
       trace("$cmdstr failed with status $rc");
       die(dieformat(180, $cmdstr));
     }

     # Matching the volume resource whose name starts with 'ora'
     # and ends with 'advm'.
     if ($acfs =~ /^ora\..+\.advm$/)
     {
       trace("Attempting to start the volume resource [$acfs] on the" .
             " first node $node");
       my @volRes = split(/\./, $acfs);
       trace("Operating on the resource [@volRes]");
       # We are expecting 4 fields where the second field is the DG name,
       # and the third field is volume name, e.g., ora.dgname.vol.advm
       if (4 == scalar(@volRes))
       {
         my $dg = $volRes[1];
         my $vol = $volRes[2];
         trace("Starting the volume [$vol] created on [$dg]");
         my $run_as_owner = TRUE;
         my $cmdStr = "start volume -volume $vol -diskgroup $dg -node $node";
         srvctl($run_as_owner, $cmdStr) || exit(1);
       }
     }

     if (('ora.registry.acfs' ne $acfs) &&
         ($acfs !~ /^ora\..+\.advm$/))
     {
         #my $vol = ($CFG->compACFS)->getACFSVolumeDevice($acfs);
       my $vol = getACFSVolumeDevice($acfs);
       if (defined $vol)
       {
         push(@volDevices, $vol);
       }
     }
   }

   trace("Device names of ACFS volumes to be started: [@volDevices]");
   if (scalar(@volDevices) > 0)
   {
     # Send the start of the resources in batches.
     my $numberOfVolArrays = ceil(scalar(@volDevices)/10);
     my @tmpVolDevices;
     my @arrayrefs;

     # Create an array of references to the pieces of the original
     while (@volDevices) {
       foreach    (0..$numberOfVolArrays-1){
         if (@volDevices) {
            push @{$arrayrefs[$_]}, shift @volDevices;
         }
       }
     }

     for (my $i = 0; $i < $numberOfVolArrays; $i++)
     {
       @tmpVolDevices = @{$arrayrefs[$i]};
       trace("Starting acfs resources in this batch: @tmpVolDevices");
       enable_start_acfs("start", $node, \@tmpVolDevices);
       sleep(1);
     }
   }
}

=head2 getAllACFSandADVMRes 
 
  Retrieve all ACFS and volume resources that are enabled and being online
  on the current node
 
=head3 Parameters       
 
  [0] CRS home 
                
=head3 Returns  
                        
  A comma separated list including all qualified ACFS and volume resources

=cut

sub getAllACFSandADVMRes 
{
  my $self = shift;
  my $crshome = $_[0];
  my $allRes;

  trace("Retrieve all ACFS & ADVM resources that are enabled and online");
  my $acfsRes = queryEnabledAndOnlineRes($crshome, $CFG->HOST, "acfs", \@ACFS_FILTER_TYPES);
  my $advmRes = queryEnabledAndOnlineRes($crshome, $CFG->HOST, "advm", \@ADVM_FILTER_TYPES);

  trace("ACFS resource list: [$acfsRes]");
  trace("ADVM resource list: [$advmRes]");

  if ($acfsRes && $advmRes)
  {
    $allRes = $acfsRes . "," . $advmRes;
  }
  elsif ($acfsRes)
  {
    $allRes = $acfsRes;
  }
  elsif ($advmRes)
  {
    $allRes = $advmRes;
  }

  trace("ACFS + ADVM resource list: [$allRes]");
  return $allRes;
}

=head2 getAllACFSandADVMResInCluster
 
  Retrieve all ACFS and volume resources in any state, in the cluster
 
=head3 Parameters       
 
  [0] CRS home 
                
=head3 Returns  
                        
  A comma separated list including all qualified ACFS and volume resources

=cut

sub getAllACFSandADVMResInCluster
{
  my $self = shift;
  my $crshome = $_[0];
  my $allRes;

  trace("Retrieve all ACFS & ADVM resources in the cluster");
  my $acfsRes = queryRes($crshome, "acfs", \@ACFS_FILTER_TYPES);
  my $advmRes = queryRes($crshome, "advm", \@ADVM_FILTER_TYPES);

  trace("ACFS resource list: [$acfsRes]");
  trace("ADVM resource list: [$advmRes]");

  if ($acfsRes && $advmRes)
  {
    $allRes = $acfsRes . "," . $advmRes;
  }
  elsif ($acfsRes)
  {
    $allRes = $acfsRes;
  }
  elsif ($advmRes)
  {
    $allRes = $advmRes;
  }

  trace("ACFS + ADVM resource list: [$allRes]");
  return $allRes;
}

1;