#!/usr/bin/env perl
#
# $Header: has/src/utl/clsdiaglsnr/diagsnap_setup.pl /st_has_12.2.0.1.0ocwpsu/1 2017/07/31 21:49:52 shullur Exp $
#
# diagsnap_setup.pl
#
# Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
#
#    NAME
#      diagsnap_setup.pl - <one-line expansion of the name>
#
#    DESCRIPTION
#      <short description of component this file declares/defines>
#
#    NOTES
#      <other useful comments, qualifications, etc.>
#
#    MODIFIED   (MM/DD/YY)
#    shullur     07/16/17 - Backport shullur_bug-25785073 from main
#    shullur     11/01/16 - Backport shullur_bug-24567625 from main
#    shullur     09/13/16 - For enabling diagsnap in dev env
#    gnagiah     10/28/16 - Backport gnagiah_bug-24672000 from main
#    gnagiah     10/28/16 - Backport gnagiah_bug-24693374 from main
#    gnagiah     10/28/16 - Backport gnagiah_bug-24542813 from
#                           st_has_12.2.0.1.0
#    gnagiah     09/20/16 - Bug 24672000. ensure the traces are not world
#                           readable
#    gnagiah     09/20/16 - Bug 24693374. Fix the top command arguments
#    gnagiah     09/20/16 - Bug 24546078. Fix orphanded remote shells
#    gnagiah     08/27/16 - Bug 24542813. Typo in mpstat command argument for
#                           Linux
#    shullur     07/13/16 - For fixing bug 23566517
#    shullur     06/23/16 - For fixing bug 23597298
#    shullur     05/10/16 - For fixing the crsd dependency
#    gnagiah     07/23/15 - Bug 21441744, autostart on leaf nodes
#    gnagiah     07/23/15 - Bug 21469457, too fast to call check followed by clean
#    gnagiah     05/05/15 - Bug 21032314, fixes crsd initial startup
#    gnagiah     04/28/15 - Bug 20927693, fixes perl shebang and perl version
#    gnagiah     03/30/15 - Script to collect OS level diagnostic on-demand
#    gnagiah     03/30/15 - Creation
#

use v5.6;

our $opt = '';
our $reg = 0;

if ( defined $ARGV[0] ) {
    $opt = shift;
}

$reg = 1 if ( $opt eq 'lastnode' );

setup();

sub setup {
    use strict;
    use warnings;
    use English;
    use File::Basename;
    use File::Spec::Functions;
    use Cwd 'realpath';

    my %params;
    my ( $sFile, $pFile );

    my ( $sName, $sDir, $sSuf ) = fileparse( $0, qr/\.[^.]*/ );
    $sName =~ s/\_[^.]*//;

    if ($opt ne "test")
    {
      $sFile = catfile( $sDir, $sName . $sSuf );
      $pFile = catfile( $sDir, '..', 'crs', 'install', 'crsconfig_params' );
    } else {
      $sFile = catfile( $sDir, '..', '..', '..', '..', 'oracle', 'bin', $sName . $sSuf );
      $pFile = catfile( $sDir, '..', '..', '..', '..', 'oracle', 'has_work_global', 
                        'crsconfig_params');
      print("params file $pFile\n");
    }

    die "Unable to find the config parmeters file" if ( !defined $pFile );

    open( my $pF, '<', $pFile )
        or die "Unable to open parameter file $pFile : $!";
    while (<$pF>) {
        chomp;
        s/#.*//;
        s/^\s+//;
        s/\s+$//;
        next unless length;
        if ( $_ =~ /^ORACLE_HOME|^ORACLE_BASE|^ORACLE_OWNER|^ORA_DBA_GROUP/ )
        {
            my ( $var, $value ) = split( /\s*=\s*/, $_, 2 );
            $params{$var} = $value;
        }
    }
    close $pF;

    die "Unable to retrieve ORACLE_HOME"   if !$params{ORACLE_HOME};
    die "Unable to retrieve ORACLE_BASE"   if !$params{ORACLE_BASE};
    die "Unable to retrieve ORACLE_OWNER"  if !$params{ORACLE_OWNER};
    die "Unable to retrieve ORA_DBA_GROUP" if !$params{ORA_DBA_GROUP};

    open( my $sF, ">", $sFile )
        or die "Unable to open script file $sFile : $!";

    open( my $mF, "<", $0 );

    while ( my $line = <$mF> ) {
        chomp $line;
        if ( $line =~ /^#!/ ) {
            print $sF "#!$params{ORACLE_HOME}/perl/bin/perl\n";
        }
        elsif ( $line =~ /^#/ ) {
            print $sF "$line\n";
        }
        else {
            last;
        }
    }

    close $mF;

    while (<DATA>) {
        chomp;
        s/%%ORACLE_HOME%%/$params{ORACLE_HOME}/;
        s/%%ORACLE_BASE%%/$params{ORACLE_BASE}/;
        s/%%ORACLE_OWNER%%/$params{ORACLE_OWNER}/;
        s/%%ORA_DBA_GROUP%%/$params{ORA_DBA_GROUP}/;
        print $sF "$_\n";
    }
    close $sF;

    my $oid = getpwnam $params{ORACLE_OWNER};
    my $gid = getgrnam $params{ORA_DBA_GROUP};

    if ( !defined $oid || !defined $gid ) {
        die "Unable to resolve the id/group for user";
    }

    if ( -e $sFile ) {
        chmod 0755, $sFile;
        chown $oid, $gid, $sFile;
        #unlink($0);
    }

    my $pout = "Script $sName.pl instantiated ";

    if ($reg) {
        my $cmd = qq($sFile register);
        system($cmd);
        $pout .= "and registered as CRS resource ";
    }

    $pout .= "successfully.\n";

    print $pout;

    exit 0;
}

__DATA__

=head1 NAME

  diagsnap.pl - DIAGnostic SNAPshot data collector

=head1 SYNOPSIS

  diagsnap.pl   start
                stop
                check
                snap
                register
                deregister
                help

  Options:

    start          Start listening to diag events
    stop           Stop listening to diag events
    check          Checks if this is running
    snap           Trigger a collection manually
    register       Register as cluster resource 
    deregister     Deregister as cluster resource
    help           Prints this help

=head1 DESCRIPTION

  This script listens to diag events posted by the clusterware components 
  and collects OS level diagnostic information to help debug the issues.

=cut

#=====
$| = 1;
use strict;
use warnings;
use English;
use Pod::Usage;

use File::Copy;
use File::Path;
use File::Basename;
use File::Path qw(make_path remove_tree);
use File::Spec::Functions;
use Cwd 'realpath';

use POSIX;
use POSIX ":sys_wait_h";
use POSIX qw(tzset strftime setsid);
use Time::HiRes qw(gettimeofday tv_interval sleep);

use IPC::Open3;
use IO::Select;
use Symbol 'gensym';

use threads;
use threads::shared;
use Thread::Queue;

tzset;
initSignals();

# ========INSTANTIATION VARIABLES========
our $DS_ORACLE_BASE   = '%%ORACLE_BASE%%';
our $DS_ORACLE_HOME   = '%%ORACLE_HOME%%';
our $DS_ORACLE_OWNER  = '%%ORACLE_OWNER%%';
our $DS_ORA_DBA_GROUP = '%%ORA_DBA_GROUP%%';

# ======================
our $DEBUG    = 0;
our $SUCCESS  = 0;
our $ERROR    = 1;
our $silent   = 0;
our $exitcode = $ERROR;

our $evtlsnrthrd;
our $heatbeatthrd;
our $evtlsnrpid : shared;
our $heartbeat : shared;
our %allpids : shared;

# ======================

our $opt = '';

if ( defined $ARGV[0] ) {
    $opt = shift;
}

our $ctx    = ctx->new();
our $eventq = Thread::Queue->new();

# ======================

validateCtx();

if ( $opt eq 'start' ) {
    $exitcode = $SUCCESS;
    &diagsnapStart();
}
elsif ( $opt eq 'stop' ) {
    $exitcode = $SUCCESS;
    &diagsnapStop();
}
elsif ( $opt eq 'check' ) {
    $exitcode = $SUCCESS;
    &diagsnapCheck();
}
elsif ( $opt eq 'clean' ) {
    $exitcode = $SUCCESS;
    &diagsnapClean();
}
elsif ( $opt eq 'snap' ) {
    $exitcode = $SUCCESS;
    &diagsnapSnap();
}
elsif ( $opt eq 'register' ) {
    $exitcode = $SUCCESS;
    &diagsnapRegisterRes();
}
elsif ( $opt eq 'deregister' ) {
    $exitcode = $SUCCESS;
    &diagsnapDeRegisterRes();
}
elsif ( $opt eq 'help' ) {
    $exitcode = $SUCCESS;
    chdir( $ctx->{SCRIPT_BASE} );
    pod2usage($SUCCESS);
}
else {
    $exitcode = $SUCCESS;
    chdir( $ctx->{SCRIPT_BASE} );
    pod2usage(
        -msg     => "\nUnknown option : $opt",
        -exitval => $ERROR
    );
}

exit($exitcode);

#=======COMMANDS ========

sub initCmds() {
    initCmdFile();
    mkCmdPing();
    mkCmdIfconfig();
    mkCmdNetstat();
    mkCmdPstack();
    mkCmdLsof();
    mkCmdPs();
    mkCmdTop();
    mkCmdVmStat();
    mkCmdMpStat();
    mkCmdArp();

    if ( isRoot() ) {
        mkCmdTracert();
        mkCmdTcpDump();
    }
    else {
        msg("Non-root user, skipping traceroute commands");
        msg("Non-root user, skipping tcpdump commands");
    }

    if ( $ctx->{CUR_INCDIR_PFIX} ne 'ref' ) {
        mkCmdArchives();
    }
    else {
        removeCmd('archive');
    }
}

sub initCmdFile {
    my $cmdfile = $ctx->{CMD_FILE};
    my %usrCmds;
    local *CMDFHDL;
    if ( !-e $cmdfile ) {
        msg("User Commands file not found, creating one ...");
        open CMDFHDL, "> $cmdfile" or abortme("Command file open failed");
        print CMDFHDL printCmdFileTempl();
        close CMDFHDL;
        chmod 0600, $cmdfile;
    }
    msg("Reading User Commands/Config, will sync if any commands found.");
    open CMDFHDL, "< $cmdfile" or abortme("Command file open failed");
    while (<CMDFHDL>) {
        chomp;
        s/#.*//;
        s/^\s+//;
        s/\s+$//;
        next unless length;
        my ( $param, $val ) = split( /\s*~~\s*/, $_, 2 );
        next if ( $val eq "" );
        if ( $param =~ m/^p_(\S+)/ ) {
            my $pm = $1;
            if (   $ctx->{$pm} != $val
                && $ctx->{CONF_PARAMS}->{$pm}
                && $val =~ /^\d+$/ )
            {
                msg("\tUpdating parameter $pm from $ctx->{$pm} to $val");
                $ctx->{$pm} = $val + 0;
            }
        }
        else {
            $usrCmds{ 'usr_' . $param } = qq($val);
            msg("\tUpdating user command ($param) : $val");
        }
    }
    close CMDFHDL;
    updateCmds(%usrCmds);
}

sub mkCmdPing {

    my %pCmd;
    my $fCmd = '';
    my ( $cmd, $ip, $args, $pfix, $cmdName );
    my @iflists = @{ $ctx->{ALLINFINFO} };
    my @nodes   = @{ $ctx->{NODELST} };
    my @eadas   = @{ $ctx->{EFFADA} };

    $pfix = '';

    foreach my $if (@iflists) {
        my ( $key, $key2, $node, $inf, $ip, $mtu ) = split( '~~', $if );
        foreach my $eada (@eadas) {
            if ( $inf =~ m/$eada/ ) {
                for ( my $pf = $ctx->{PLATFORM} ) {
                    if (/darwin/) {
                        $cmd  = "/sbin/ping";
                        $args = "-n -t 10";
                    }
                    elsif (/linux/) {
                        $cmd  = "/bin/ping";
                        $mtu  = $mtu - 8;
                        $args = "-n -w 15 -s $mtu -I $inf";
                    }
                    elsif (/solaris/) {
                        $cmd  = "/usr/sbin/ping";
                        $mtu  = $mtu - 8;
                        $pfix = "-s";
                        $inf  = substr( $inf, 0, -1 );
                        $args = "$mtu 15 -i $inf";
                    }
                    elsif (/aix/) {
                        $cmd  = "/usr/bin/ping";
                        $mtu  = $mtu - 8;
                        $pfix = "-s $mtu -c 15 -w 5 -o $inf";
                        $args = "";
                    }
                    elsif (/hpux/) {
                        $cmd  = "/usr/sbin/ping";
                        $mtu  = $mtu - 8;
                        $args = "$mtu -n 15";
                    }
                    else {
                        wrn("mkCmdPing - Invalid platform ($pf)- skipping commands"
                        );
                        return 0;
                    }
                    unless ( -e $cmd ) {
                        wrn("$cmd not found, - skipping ping commands");
                        return 0;
                    }
                }
                $fCmd    = qq($cmd $pfix $ip $args);
                $cmdName = "ping" . "_" . fmtName($key) . "_" . fmtName($ip);
                $pCmd{qq($cmdName)} = qq($fCmd);
            }
        }
    }
    updateCmds(%pCmd);
}

sub mkCmdIfconfig {
    my ( %pCmd, $cmd, $args, $spare );
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            $cmd  = "/sbin/ifconfig";
            $args = "-a";
        }
        elsif (/linux/) {
            $cmd  = "/sbin/ifconfig";
            $args = "-a";
        }
        elsif (/solaris/) {
            $cmd  = "/usr/sbin/ifconfig";
            $args = "-a";
        }
        elsif (/aix/) {
            $cmd  = "/etc/ifconfig";
            $args = "-a";
        }
        elsif (/hpux/) {
            $cmd = "/usr/sbin/ifconfig";
            $spare
                = qq(for i in `netstat -in|egrep -v "Name|Mtu"|awk '{print \$1}'`;do $cmd \$i;done);
        }
        else {
            wrn("mkCmdIfconfig - Invalid platform ($pf)- skipping commands");
            return 0;
        }
    }
    unless ( -e $cmd ) {
        wrn("$cmd not found, - skipping ifconfig commands");
        return 0;
    }
    my $cmdName = 'ifconfig';

    my $fCmd;

    if ($spare) {
        $fCmd = $spare;
    }
    else {
        $fCmd = qq($cmd $args);
    }

    $pCmd{qq($cmdName)} = qq($fCmd);

    updateCmds(%pCmd);
}

sub mkCmdNetstat {
    my ( %pCmd, $cmd, @args, $cmdName, $fCmd );
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            wrn("mkCmdNetstat - Invalid platform ($pf)- skipping commands");
            return 0;
        }
        elsif (/linux/) {
            $cmd     = "/bin/netstat";
            $args[0] = "-ap";
            $args[1] = "-i";
            $args[2] = "-r";
            $args[3] = "-s";
            $args[4] = "";

        }
        elsif (/solaris/) {
            $cmd     = "/bin/netstat";
            $args[0] = "-ap";
            $args[1] = "-i";
            $args[2] = "-r";
            $args[3] = "-s";
            $args[4] = "";

        }
        elsif (/aix/) {
            $cmd     = "/usr/bin/netstat";
            $args[0] = "-ao";
            $args[1] = "-i";
            $args[2] = "-r";
            $args[3] = "-s";
            $args[4] = "";
            $args[5] = "-D";
        }
        elsif (/hpux/) {
            $cmd     = "/usr/bin/netstat";
            $args[0] = "-in";
            $args[1] = "-r";
            $args[2] = "-s";
            $args[3] = "-w";
            $args[4] = "";
        }
        else {
            wrn("mkCmdNetstat - Invalid platform ($pf)- skipping commands");
            return 0;
        }
    }
    unless ( -e $cmd ) {
        wrn("$cmd not found, - skipping netstat commands");
        return 0;
    }
    foreach my $arg (@args) {
        $cmdName            = fmtName( "netstat" . $arg );
        $fCmd               = qq($cmd $arg);
        $pCmd{qq($cmdName)} = qq($fCmd);
    }
    updateCmds(%pCmd);
}

sub mkCmdPstack {
    my ( %pCmd, $cmd, $args, $cmdName, $fCmd );
    my %vpm  = %{ $ctx->{VITAL_PROCESS} };
    my @pids = values %vpm;
    my %vp   = map { $vpm{$_} => $_ } keys(%vpm);

    foreach my $pid (@pids) {

        next if $pid == 0;

        if ( kill( 0, $pid ) ) {
            my $p = $vp{$pid};
            for ( my $pf = $ctx->{PLATFORM} ) {
                if (/darwin/) {
                    wrn("mkCmdPstack - Invalid platform ($pf)- skipping commands"
                    );
                    return 0;
                }
                elsif (/linux/) {
                    $cmd  = "/usr/bin/pstack";
                    $fCmd = "for i in {1..3};";
                    $fCmd
                        .= " do printf \"zzz \"; date; $cmd $pid; sleep 5; ";
                    $fCmd .= " done";
                }
                elsif (/solaris/) {
                    $cmd  = "/bin/pstack";
                    $fCmd = "/usr/bin/bash -c \" for i in 1 2 3;";
                    $fCmd
                        .= " do printf \\\"zzz \\\$i \\\"; date; $cmd $pid;";
                    $fCmd .= " if [ \\\$i != 3 ]; then sleep 5; fi;";
                    $fCmd .= " done \"";
                }
                elsif (/aix/) {
                    $cmd  = "/usr/bin/procstack";
                    $fCmd = "/usr/bin/bash -c \" for i in 1 2 3;";
                    $fCmd
                        .= " do printf \\\"zzz \\\$i \\\"; date; $cmd $pid;";
                    $fCmd .= " if [ \\\$i != 3 ]; then sleep 5; fi;";
                    $fCmd .= " done \"";
                }
                elsif (/hpux/) {
                    $cmd  = "/usr/ccs/bin/pstack";
                    $fCmd = "/usr/bin/bash -c \" for i in 1 2 3;";
                    $fCmd
                        .= " do printf \\\"zzz \\\$i \\\"; date; $cmd $pid;";
                    $fCmd .= " if [ \\\$i != 3 ]; then sleep 5; fi;";
                    $fCmd .= " done \"";
                }
                else {
                    wrn("mkCmdPstack - Invalid platform ($pf)- skipping commands"
                    );
                    return 0;
                }
                unless ( -e $cmd ) {
                    wrn("$cmd not found, - skipping pstack commands");
                    return 0;
                }
            }
            $cmdName = "pstack_$pid\_" . fmtName($p);
            $pCmd{qq($cmdName)} = qq($fCmd);
        }
    }
    updateCmds(%pCmd);
}

sub mkCmdLsof {
    my ( %pCmd, $cmd, $args, $cmdName, $fCmd );
    my %vpm  = %{ $ctx->{VITAL_PROCESS} };
    my @pids = values %vpm;
    my %vp   = map { $vpm{$_} => $_ } keys(%vpm);

    foreach my $pid (@pids) {

        next if $pid == 0;

        if ( kill( 0, $pid ) ) {
            my $p = $vp{$pid};
            for ( my $pf = $ctx->{PLATFORM} ) {
                if (/darwin/) {
                    wrn("mkCmdLsof - Invalid platform ($pf)- skipping commands"
                    );
                    return 0;
                }
                elsif (/linux/) {
                    $cmd  = "/usr/sbin/lsof";
                    $args = "-p $pid";
                }
                elsif (/solaris/) {
                    $cmd  = "/usr/bin/pfiles";
                    $args = "$pid";
                }
                elsif (/aix/) {
                    $cmd  = "/usr/sbin/lsof";
                    $args = "-p $pid";
                }
                elsif (/hpux/) {
                    $cmd  = "/usr/local/bin/lsof";
                    $args = "-p $pid";
                }
                else {
                    wrn("mkCmdLsof - Invalid platform ($pf)- skipping commands"
                    );
                    return 0;
                }
                unless ( -e $cmd ) {
                    wrn("$cmd not found, - skipping lsof commands");
                    return 0;
                }
            }
            $cmdName = "lsof_$pid\_" . fmtName($p);
            $pCmd{qq($cmdName)} = qq($cmd $args);
        }
    }
    updateCmds(%pCmd);
}

sub mkCmdPs {
    my ( %pCmd, $cmd, @args, $cmdName, $fCmd );
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            wrn("mkCmdPs - Invalid platform ($pf)- skipping commands");
            return 0;
        }
        elsif (/linux/) {
            $cmd = "/bin/ps";
            $args[0] = "-elLf";
        }
        elsif (/solaris/) {
            $cmd = "/usr/bin/ps";
            $args[0] = "-elLf";
        }
        elsif (/aix/) {
            $cmd = "/usr/bin/ps";
            $args[0] = "-elf -mo THREAD";
        }
        elsif (/hpux/) {
            $cmd = "/usr/bin/ps";
            $args[0] = "-elf";
        }
        else {
            wrn("mkCmdPs - Invalid platform ($pf)- skipping commands");
            return 0;
        }
    }
    unless ( -e $cmd ) {
        wrn("$cmd not found, - skipping ps commands");
        return 0;
    }
    foreach my $arg (@args) {
        $cmdName            = fmtName( "ps" . $arg );
        $fCmd               = qq($cmd $arg);
        $pCmd{qq($cmdName)} = qq($fCmd);
    }
    updateCmds(%pCmd);
}

sub mkCmdTracert {
    my %pCmd;
    my $fCmd    = '';
    my @iflists = @{ $ctx->{ALLINFINFO} };
    my @nodes   = @{ $ctx->{NODELST} };
    my @eadas   = @{ $ctx->{EFFADA} };
    my ( $cmd, $ip, $args, $cmdName );
    foreach my $if (@iflists) {
        my ( $key, $key2, $node, $inf, $ip, $mtu ) = split( '~~', $if );
        foreach my $eada (@eadas) {
            if ( $inf =~ m/$eada/ ) {
                for ( my $pf = $ctx->{PLATFORM} ) {
                    if (/darwin/) {
                        $cmd  = "/usr/sbin/traceroute";
                        $args = "-n -t 10";
                    }
                    elsif (/linux/) {
                        $cmd  = "/bin/traceroute";
                        $args = "-i $inf -m 5";
                    }
                    elsif (/solaris/) {
                        $inf  = substr( $inf, 0, -1 );
                        $cmd  = "/usr/sbin/traceroute";
                        $args = "-i $inf -m 5";
                    }
                    elsif (/aix/) {
                        $inf  = substr( $inf, 0, -1 );
                        $cmd  = "/usr/bin/traceroute";
                        $args = "-i $inf -m 5";
                    }
                    elsif (/hpux/) {
                        $cmd  = "/usr/contrib/bin/traceroute";
                        $args = "-i $inf -m 5";
                    }
                    else {
                        wrn("mkCmdTracert - Invalid platform ($pf)- skipping commands"
                        );
                        return 0;
                    }
                }
                unless ( -e $cmd ) {
                    wrn("$cmd not found, - skipping traceroute commands");
                    return 0;
                }
                $fCmd = qq($cmd $args $ip);
                $cmdName
                    = "traceroute" . "_" . fmtName($key) . "_" . fmtName($ip);
                $pCmd{qq($cmdName)} = qq($fCmd);
            }
        }
    }
    updateCmds(%pCmd);
}

sub mkCmdTop {
    my ( %pCmd, $cmd, @args, $cmdName, $fCmd );
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            wrn("mkCmdPs - Invalid platform ($pf)- skipping commands");
            return 0;
        }
        elsif (/linux/) {
            $cmd = "/usr/bin/top";
            $args[0] = "-b -d 5 -n 3";
        }
        elsif (/solaris/) {
            $cmd     = "/usr/bin/prstat";
            $args[0] = "-T 1 1";
            $args[0] = "-J 1 1";
        }
        elsif (/aix/) {
            wrn("mkCmdTop - top not available for $pf - skipping commands");
            return 0;
        }
        elsif (/hpux/) {
            $cmd = "/usr/bin/top";
            $args[0] = "-d 1";
        }
        else {
            wrn("mkCmdTop - Invalid platform ($pf)- skipping commands");
            return 0;
        }
    }
    unless ( -e $cmd ) {
        wrn("$cmd not found, - skipping top commands");
        return 0;
    }
    foreach my $arg (@args) {
        $cmdName            = fmtName( "top_" . $ctx->{HOSTNAME} );
        $fCmd               = qq($cmd $arg);
        $pCmd{qq($cmdName)} = qq($fCmd);
    }
    updateCmds(%pCmd);
}

sub mkCmdVmStat {
    my ( %pCmd, $cmd, @args, $cmdName, $fCmd );
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            wrn("mkCmdVmStat - Invalid platform ($pf)- skipping commands");
            return 0;
        }
        elsif (/linux/) {
            $cmd     = "/usr/bin/vmstat";
            $args[0] = "-a -S M 1 15";
            $args[1] = "-S M 1 15";
            $args[2] = "-d";
            $args[3] = "-D";
            $args[4] = "-s";
            $args[5] = "-m";
        }
        elsif (/solaris/) {
            $cmd     = "/usr/bin/vmstat";
            $args[0] = "-i";
            $args[1] = "-p";
            $args[2] = "1 15";
        }
        elsif (/aix/) {
            $cmd     = "/usr/bin/vmstat";
            $args[0] = "-v";
            $args[1] = "-s";
            $args[2] = "-Iwt 1 15";
        }
        elsif (/hpux/) {
            $cmd     = "/usr/bin/vmstat";
            $args[0] = "1 15";
            $args[1] = "-n 1 15";
            $args[2] = "-d 1 15";
        }
        else {
            wrn("mkCmdVmStat - Invalid platform ($pf)- skipping commands");
            return 0;
        }
    }
    unless ( -e $cmd ) {
        wrn("$cmd not found, - skipping vmstat commands");
        return 0;
    }
    foreach my $arg (@args) {
        $cmdName            = fmtName( "vmstat" . $arg );
        $fCmd               = qq($cmd $arg);
        $pCmd{qq($cmdName)} = qq($fCmd);
    }
    updateCmds(%pCmd);
}

sub mkCmdMpStat {
    my ( %pCmd, $cmd, @args, $cmdName, $fCmd );
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            wrn("mkCmdMpStat - Invalid platform ($pf)- skipping commands");
            return 0;
        }
        elsif (/linux/) {
            $cmd = "/usr/bin/mpstat";
            $args[0] = "-P ALL 1 15";
        }
        elsif (/solaris/) {
            $cmd = "/usr/bin/mpstat";
            $args[0] = "1 15";
        }
        elsif (/aix/) {
            $cmd = "/usr/bin/mpstat";
            $args[0] = "1 15";
        }
        elsif (/hpux/) {
            wrn("mpstat - Not available for $pf - skipping commands");
            return 0;
        }
        else {
            wrn("mkCmdMpStat - Invalid platform ($pf)- skipping commands");
            return 0;
        }
    }
    unless ( -e $cmd ) {
        wrn("$cmd not found, - skipping mpstat commands");
        return 0;
    }
    foreach my $arg (@args) {
        $cmdName            = fmtName( "mpstat" . $arg );
        $fCmd               = qq($cmd $arg);
        $pCmd{qq($cmdName)} = qq($fCmd);
    }
    updateCmds(%pCmd);
}

sub mkCmdArp {
    my ( %pCmd, $cmd, @args, $cmdName, $fCmd );
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            wrn("mkCmdArp - Invalid platform ($pf)- skipping commands");
            return 0;
        }
        elsif (/linux/) {
            $cmd = "/sbin/arp";
            $args[0] = "-an";
        }
        elsif (/solaris/) {
            $cmd = "/usr/sbin/arp";
            $args[0] = "-an";
        }
        elsif (/aix/) {
            $cmd = "/etc/arp";
            $args[0] = "-an";
        }
        elsif (/hpux/) {
            $cmd = "/usr/sbin/arp";
            $args[0] = "-An";
        }
        else {
            wrn("mkCmdArp - Invalid platform ($pf)- skipping commands");
            return 0;
        }
    }
    unless ( -e $cmd ) {
        wrn("$cmd not found, - skipping arp commands");
        return 0;
    }
    foreach my $arg (@args) {
        $cmdName            = fmtName( "arp" . $arg );
        $fCmd               = qq($cmd $arg);
        $pCmd{qq($cmdName)} = qq($fCmd);
    }
    updateCmds(%pCmd);
}

sub mkCmdTcpDump {
    my ( %pCmd, $cmd, $args, $cmdName, $fCmd );
    my @adaps = @{ $ctx->{EFFADA} };
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            wrn("mkCmdTcpDump - Invalid platform ($pf)- skipping commands");
            return 0;
        }
        elsif (/linux/) {
            $cmd  = "/usr/sbin/tcpdump";
            $args = "-c 200 -i";
        }
        elsif (/solaris/) {
            wrn("mkCmdTcpDump - tcpdump Not available for $pf - skipping commands"
            );
            return 0;
        }
        elsif (/aix/) {
            $cmd  = "/usr/sbin/tcpdump";
            $args = "-c 200 -i";
        }
        elsif (/hpux/) {
            $cmd  = "/usr/local/bin/tcpdump";
            $args = "-c 200 -i";
        }
        else {
            wrn("mkCmdTcpDump - Not available for ($pf)- skipping tcpdump commands"
            );
            return 0;
        }
    }
    unless ( -e $cmd ) {
        wrn("$cmd not found, - skipping tcpdump commands");
        return 0;
    }
    foreach my $adap (@adaps) {
        $cmdName            = fmtName( "tcpdump_" . $adap );
        $fCmd               = qq($cmd $args $adap);
        $pCmd{qq($cmdName)} = qq($fCmd);
    }
    updateCmds(%pCmd);
}

sub mkCmdArchives {

    my $date = gettimeofday;
    my $archfile
        = catfile( '.', 'ref_collect_arch_' . fmtTrTime($date) . '.zip' );
    my $chdir = qq(cd $ctx->{CUR_INCDIR};);
    my $refdirs = catdir( '..', 'ref_*' );

    my ( %pCmd, $cmd, $arg, $cmdName, $fCmd, $altCmd, $su, $sucmd, $gzip );
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            wrn("mkCmdArchives - Invalid platform ($pf)- skipping commands");
            return 0;
        }
        elsif (/linux/) {
            $su  = "/bin/su -l";
            $cmd = "/usr/bin/zip";
            $arg = "-r";
        }
        elsif (/solaris/) {
            $su  = "/bin/su";
            $cmd = "/usr/bin/zip";
            $arg = "-r";
        }
        elsif (/aix/) {
            $su     = "/bin/su";
            $cmd    = "/usr/bin/tar";
            $gzip   = "/usr/bin/gzip";
            $altCmd = "$chdir $cmd cvf - $refdirs | $gzip > $archfile";
        }
        elsif (/hpux/) {
            $su  = "/bin/su";
            $cmd = "/usr/local/bin/zip";
            $arg = "-r";
        }
        else {
            wrn("mkCmdArchives - Invalid platform ($pf)- skipping commands");
            return 0;
        }
    }

    unless ( -e $cmd | ( defined $gzip && -e $gzip ) ) {
        wrn("$cmd not found, - skipping archive commands");
        return 0;
    }

    $cmdName = fmtName("archive");

    if ($altCmd) {
        $fCmd = $altCmd;
    }
    else {
        $fCmd = qq($chdir $cmd $arg $archfile $refdirs);
    }

    if ( isRoot() ) {
        $fCmd = qq($su $ctx->{ORACLE_OWNER} -c "$fCmd");
    }

    $pCmd{qq($cmdName)} = qq($fCmd);

    updateCmds(%pCmd);

}

#====== START =========

sub diagsnapStart {

    printLogHdr();

    my $rm = $ctx->{RUN_MODE};

    msgs(
        "$ctx->{SCRIPT_NAME} invoked with $opt option in $ctx->{RUN_MODE_DESC}{$rm} mode"
    );

    startCheck();

    #daemonize();

    initNodeDetails();

    if ( $rm != 3 ) {
        $evtlsnrthrd = threads->create( \&startEvtLsnr );
        sleep(.5);
    }
    else {
        wrn("Running as a $ctx->{RUN_MODE_DESC}{$rm} plugin, event listener will not be spawned."
        );
    }

    $heatbeatthrd = threads->create( \&checkHeartBeat );
    sleep(.5);

    msg("Started successfully, Listening for events...");
    setCheckCount();

    my $lc = 0;

    while (1) {
        $lc = $lc + $ctx->{LOOP_INT};

        $ctx->{SKIP_COLLECT}    = 0;
        $ctx->{CUR_INCIDENT}    = '';
        $ctx->{CUR_INCDIR_PFIX} = '';
        $heartbeat              = $ctx->{HEART_BEAT};

        if ( $lc >= $ctx->{CHECK_INTERVAL} ) {
            msg("Initiating reference collection at $lc sec interval");
            refCheck();
            sleep(.5);
            $lc = 0;
        }

        if ( $rm == 3 ) {
            plgCheck();
        }
        else {
            evtCheck();
        }

        if ( checkTrigger() ) {
            msg("An event occured while running in $ctx->{RUN_MODE_DESC}{$rm} mode"
            );
            checkInciFreq();
            setInciDir() if ( $ctx->{SKIP_COLLECT} == 0 );
            printEvt();

            if ( $ctx->{SKIP_COLLECT} == 0 ) {
                initCmds();
                execCmds();
            }
        }

        killTimeouts();

        if ( $rm != 3 ) {
            if ( $evtlsnrthrd->is_running() ) {
                sleep( $ctx->{LOOP_INT} );
            }
            else {
                abortme("Event Listener Thread Vanished!!!, Exiting");
            }
        }
        else {
            sleep( $ctx->{LOOP_INT} );
        }

    }
}

sub startCheck {

    my $rm = $ctx->{RUN_MODE};

    platformCheck();
=pod
    if ( $rm > 1 ) {
        if ( !isGridUp() ) {
            say("Start failed : Clusterware not running");
            abortme(
                "$ctx->{SCRIPT_NAME} needs Clusterware running to proceed further"
            );
        }
    }
=cut

    if ( isRegResource() && ( $ctx->{CRS_ENABLED} > 1 ) ) {
        say("$ctx->{SCRIPT_NAME} registered as CRS resource");
        say("Starting using \"crsctl start res $ctx->{CRS_RES_NAME}\" ");
        if ( startCrsResource() == 1 ) {
            say("Start issued successfully");
        }
        else {
            say("Start failed using crsctl, check crsd scriptagent logs");
        }
        exit $exitcode;
    }

    pidCheck();
}

sub startEvtLsnr {

    my $evtlsnr = catfile( $ctx->{OH_BIN}, 'clsdiaglsnr' );
    my $fh;

    if ( -e $evtlsnr ) {

        $evtlsnrpid = open( $fh, "$evtlsnr |" )
            || abortme("Unable to start listener");

        if ($evtlsnrpid) {
            while ( my $line = <$fh> ) {
                chomp($line);

                if ( $line =~ /^clsdiaglsnr/ ) {
                    msg($line);
                }
                else {
                    $eventq->enqueue($line);
                }
            }
        }
    }
    else {
        abortme("startEvtLsnr: Event listener not found");
    }
}

sub checkInciFreq {

    my $to = eval( $ctx->{EXEC_TIMEOUT} + 3 );

    $ctx->{CUR_ITIME} = gettimeofday;

    my $int = int tv_interval [ $ctx->{PRE_ITIME} ], [ $ctx->{CUR_ITIME} ];

    if ( $int < $to ) {
        msg("Event received too soon $int/$to seconds");
        msg("Skipping diag collection for this event");
        $ctx->{SKIP_COLLECT} = 1;
    }
}

sub evtCheck {
    if ( $eventq->pending() ) {
        $ctx->{CUR_INCIDENT}    = $eventq->dequeue();
        $ctx->{CUR_INCDIR_PFIX} = 'evt';
        setTrigger();
    }
}

sub plgCheck {
    my $evnt = checkPlgTrigger();
    if ( $evnt ne '0' ) {
        $ctx->{CUR_INCDIR_PFIX} = 'evt';
        $ctx->{CUR_INCIDENT}    = $evnt;
        setTrigger();
    }
}

sub refCheck {
    my $time = gettimeofday;
    initNodeDetails();
    setTrigger();
    $ctx->{CUR_INCDIR_PFIX} = 'ref';
    $ctx->{CUR_INCIDENT}
        = "t_type=reference~~d_comment=Reference collection~~d_TIMESTAMP="
        . fmtTime($time);
}

sub printEvt {
    my $time = gettimeofday;
    if ( !$ctx->{CUR_INCIDENT} ) {
        $ctx->{CUR_INCIDENT}
            = "t_type=manual~~d_comment=Data collection triggered manually~~d_TIMESTAMP="
            . fmtTime($time);
    }
    my @evt = split( '~~', $ctx->{CUR_INCIDENT} );
    my $p = "Following Event received\n";
    $p .= '-' x 79 . "\n";
    foreach my $e (@evt) {
        my $o = "  > ";
        my ( $k, $kv ) = split( '=', $e );
        my ( $t, $tv ) = split( '_', $k );
        if ( $t eq 't' ) {
            $o .= "TYPE      < ";
        }
        elsif ( $t eq 'p' ) {
            $o .= "PARAMETER < ";
        }
        else {
            $o .= "DATA      < ";
        }
        $o .= uc($tv) . " : $kv\n";
        $p .= $o;
    }
    $p .= '-' x 79;
    msg($p);

    my $etf = $ctx->{CUR_INCTRC};
    if ($etf) {
        my $date = gettimeofday;
        $date = fmtTime($date);
        open( my $etfh, ">>$etf" )
            or abortme("printEvt: Can't open $etf for append: $!");
        print $etfh "$date: [$$] $p\n";
        close($etfh);
        chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID}, $etf;
    }
}

sub pidCheck {
    my $pidf  = $ctx->{PID_FILE};
    my $mypid = $ctx->{PROG_PID};

    if ( -e $pidf ) {
        my $fpid = getPid();
        if ( $mypid ne $fpid ) {
            if ( kill( 0, $fpid ) ) {
                msgs("$ctx->{PROG_NAME}[0] with pid $fpid running already.");
                abortme("pidCheck: Only one of me could exists... ");
            }
        }
    }
}

sub platformCheck {
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            msg("$ctx->{SCRIPT_NAME} invoked on OS X/Darwin");
        }
        elsif (/linux/) {
            msg("$ctx->{SCRIPT_NAME} invoked on Linux");
        }
        elsif (/solaris/) {
            msg("$ctx->{SCRIPT_NAME} invoked on Solaris");
        }
        elsif (/aix/) {
            msg("$ctx->{SCRIPT_NAME} invoked on AIX");
        }
        else {
            say("Invalid platform ($pf). Halting execution");
            abortme(
                "platformCheck: Invalid platform ($pf). Halting execution");
        }
    }
}

sub initNodeDetails {
    queryVitalPids();
    updateNodeDetails();
    updateInfDetails();
    initCmdFile();
}

sub killTimeouts {
    if (%allpids) {
        my $evt_time = $ctx->{PRE_ITIME};
        my $cur_time = gettimeofday;
        my $timeout  = $ctx->{EXEC_TIMEOUT};

        # my $timeout  = 10;
        my $elapsed = int tv_interval [$evt_time], [$cur_time];
        if ( $elapsed >= $timeout ) {
            foreach my $fpid ( keys %allpids ) {
                if ( kill( 0, $fpid ) ) {
                    wrn("Terminating timed out ($timeout secs) process : $fpid"
                    );
                    kill -1, $fpid;
                }
                else {
                    delete $allpids{$fpid};
                }
            }
        }
    }
}

#====== STOP ===========

sub diagsnapStop {

    my $rm = $ctx->{RUN_MODE};

    msgs(
        "$ctx->{SCRIPT_NAME} invoked with $opt option in $ctx->{RUN_MODE_DESC}{$rm} mode"
    );

    if ( isRegResource() && ( $ctx->{CRS_ENABLED} > 1 ) ) {
        say("$ctx->{SCRIPT_NAME} registered as CRS resource");
        say( "Stopping using \"crsctl stop res $ctx->{CRS_RES_NAME}\" to stop"
        );
        if ( stopCrsResource() == 1 ) {
            msg("Stop issued successfully");
            say("Stop issued successfully.") if ( $rm > 1 );
        }
        else {
            wrn("Failed stopping the resource");
            say("Failed stopping the resource") if ( $rm > 1 );
        }
        return $SUCCESS;
    }

    my $pidf = $ctx->{PID_FILE};

    if ( -e $pidf ) {
        my $killpid  = getPid();
        my $orgOwner = getOwnerForPid($killpid);
        if ( $ctx->{OWNER} ne $orgOwner && !( isRoot() ) ) {
            wrn("Trying to stop a process owned by '$orgOwner' as '$ctx->{OWNER}'"
            );
            say("Stop Failed. Can not stop a process started by root.")
                if ( $rm > 1 );
            return $ERROR;
        }
        if ( kill( 0, $killpid ) ) {
            msg("[stop] Stopping the deamon ($killpid), as per request.");
            kill INT => $killpid;
            sleep 1;
            diagsnapClean() if ( $rm > 1 );
            say("Stopped successfully.") if ( $rm > 1 );
            resetCheckCount();
        }
        else {
            msg("[stop] The deamon ($killpid) is not running.");
            say("Stopped successfully.") if ( $rm > 1 );
        }
    }
    else {
        msgs(
            "[Stop] Looks like the deamon is never started or may be cleaned..."
        );
    }
    return $SUCCESS;
}

#====== CHECK ===========

sub diagsnapCheck {

    my $rm = $ctx->{RUN_MODE};

    msgs(
        "$ctx->{SCRIPT_NAME} invoked with $opt option in $ctx->{RUN_MODE_DESC}{$rm} mode"
    );

    if ( isRegResource() && ( $rm > 1 ) ) {
        say("$ctx->{SCRIPT_NAME} registered as CRS resource");
        say("Use \"crsctl stat res $ctx->{CRS_RES_NAME}\" for the status");
        $exitcode = $ERROR;
        return $exitcode;
    }

    my $pidf = $ctx->{PID_FILE};

    if ( -e $pidf ) {
        my $checkpid = getPid();

        if ( getCheckCount() == 0 ) {
            msg("[check] followed by a stop/clean, check with wait.");
            for ( my $cnt = 0; $cnt < 10; $cnt++ ) {
                if ( $checkpid && kill( 0, $checkpid ) ) {
                    sleep(1);
                }
                else {
                    last;
                }
            }
        }

        if ( $checkpid && kill( 0, $checkpid ) ) {
            msg("[check] The deamon ($checkpid), is online.");
            say("Script $ctx->{SCRIPT_NAME} on $ctx->{HOSTNAME} is ONLINE.")
                if ( $rm > 1 );
            $exitcode = $SUCCESS;
            setCheckCount();
        }
        else {
            msg("[check] The deamon ($checkpid), is offline.");
            say("Script $ctx->{SCRIPT_NAME} on $ctx->{HOSTNAME} is OFFLINE.")
                if ( $rm > 1 );
            $exitcode = $ERROR;
        }
    }
    else {
        msg("[Check] Looks like the deamon is never started or may be cleaned..."
        );
        $exitcode = $ERROR;
    }
    return $exitcode;
}

#====== CLEAN ===========

sub diagsnapClean {

    my %pids;
    my @pats = ( $ctx->{SCRIPT_NAME}, 'clsdiaglsnr.bin' );

    my $cpid = getPid();
    $pids{$cpid}++ if ($cpid);

    foreach my $pat (@pats) {
        foreach my $p ( pgrep($pat) ) {
            $pids{$p}++;
        }
    }

    delete $pids{$$};

    foreach my $pid ( sort { $a <=> $b } keys %pids ) {
        if ( kill 0, $pid ) {
            msg("diagsnapClean: Cleaning stray process $pid");
            kill INT => $pid;
        }
        delete $pids{$pid};
    }
    resetCheckCount();
}

#====== COLLECT ===========

sub diagsnapSnap {
    my $rm = $ctx->{RUN_MODE};
    msgs(
        "$ctx->{SCRIPT_NAME} invoked with $opt option in $ctx->{RUN_MODE_DESC}{$rm} mode"
    );

    my $cm = getPluginEnabled();
    if ( $cm ne $rm && $rm == 3 ) {
        msgs(
            "Can not trigger event, $ctx->{SCRIPT_NAME} is running in $ctx->{RUN_MODE_DESC}{$cm} mode"
        );
        return $SUCCESS;
    }

    if ( $rm == 3 ) {

        if ( defined $ENV{'DS_PLUGIN_TRIGGER'} ) {
            my $evt = $ENV{'DS_PLUGIN_TRIGGER'};
            if ( $evt =~ /^t_type=.*(?=\~\~d_TIMESTAMP=)/ ) {
                setPlgTrigger($evt);
            }
            else {
                msgs("Event is not in the expected format.");
            }
        }
        else {
            msgs(
                "Event is not provided, use DS_PLUGIN_TRIGGER to provide the event."
            );
        }
    }
    else {
        setTrigger();
    }

    return $SUCCESS;
}

#====== REGISTER ===========

sub diagsnapRegisterRes {

    printLogHdr();
    platformCheck();
    msg("[register] $ctx->{SCRIPT_NAME} invoked with register option");

    my $res = $ctx->{CRS_RES_NAME};

    if ( isGridUp() == 0 ) {
        msgs("Needs clusterware up and running to proceed further. Exiting");
        $exitcode = $ERROR;
        return $exitcode;
    }

    if ( isRegResource() == 1 ) {
        msgs("Resource $res already registered as CRS resource. Exiting");
        $exitcode = $SUCCESS;
    }
    else {
        msg("Will proceed to register resource $res");

        my $crsctl = $ctx->{CRSCTL};
        my $script = $ctx->{SCRIPT};

        my $cmd = qq($crsctl add res $res -type local_resource -attr ");
        $cmd .= qq(ACTION_SCRIPT=$script,);
        $cmd .= qq(DESCRIPTION=DIAG SNAPshot deamon.,);
        $cmd .= qq(RESTART_ATTEMPTS=5,);
        $cmd .= qq(AUTO_START=always,);
        $cmd .= qq(CHECK_INTERVAL=30);
        $cmd .= qq(");

        my ( $rc, @out ) = runCmd_silent($cmd);

        if ($rc) {
            msgs("$res registered to CRS successfully as $ctx->{OWNER}");

            if ( startCrsResource() == 1 ) {
                msg("Resource $res started successfully.");
                $exitcode = $SUCCESS;
            }
            else {
                wrn("Failed to start $res");
                $exitcode = $ERROR;
            }
        }

    }
    return $exitcode;
}

#====== DEREGISTER ===========

sub diagsnapDeRegisterRes() {

    printLogHdr();
    platformCheck();
    msg("$ctx->{SCRIPT_NAME} invoked with deregister option");

    my $res = $ctx->{CRS_RES_NAME};

    if ( isGridUp() == 0 ) {
        msgs("Needs clusterware up and running to proceed further. Exiting");
        $exitcode = $ERROR;
        return $exitcode;
    }

    if ( isRegResource() == 0 ) {
        msgs("$res is not registered to CRS yet. Exiting");
        $exitcode = $SUCCESS;
    }
    else {
        msg("Will proceed to deregister resource $res");

        if ( !stopCrsResource() ) {
            wrn("Stop resource failed. Exiting");
            say("Deregister $res failed, check $ctx->{TRACE_FILE}");
            $exitcode = $ERROR;
            return $exitcode;
        }

        my $crsctl = $ctx->{CRSCTL};
        my $cmd    = qq($crsctl delete res $res -f -i);
        my ( $rc, @out ) = runCmd_silent($cmd);

        foreach my $line (@out) {
            if ( $line =~ /2677/ ) {
                $rc = 1;
                last;
            }
            elsif ( $line =~ /0245/ ) {
                wrn("User '$ctx->{OWNER}' does not have privilege to delete the resource"
                );
                $rc = 0;
                last;
            }
        }

        if ( $rc == 0 ) {
            wrn("Failed to deregister resource $res - ($rc)");
            say("Deregister $res failed, check $ctx->{TRACE_FILE}");
            $exitcode = $ERROR;
        }
        else {
            msgs("Deregistered $res from CRS successfully");
            $exitcode = $SUCCESS;
        }

    }
    return $exitcode;
}

#======= Run Commands =====

sub runCmd_silent {
    $silent = 1;
    my ( $rc, @output ) = runCmd(@_);
    $silent = 0;
    return ( $rc, @output );
}

sub runCmd {
    my $rc  = 0;
    my $prc = 0;
    my $pid = 0;
    my @output;

    msg("Executing cmd: @_ ") if ( !$silent );

    if ( !( $pid = open( CMD, "@_ 2>&1 |" ) ) ) {
        $rc = -1;
    }
    else {
        msg("Successfully executed with pid $pid") if ( !$silent );
        $allpids{$pid}++;
        @output = (<CMD>);
        close CMD;
        $prc = $CHILD_ERROR >> 8;
        chomp(@output);
    }

    if ( $prc != 0 ) {
        $rc = $prc;
    }
    elsif ( $rc < 0 || $CHILD_ERROR < 0 ) {
        push @output,
            "Failure in execution (rc=$rc, $CHILD_ERROR, $!) for command @_";
    }
    elsif ( $rc & 127 ) {
        my $sig = $rc & 127;
        push @output, "Failure with signal $sig from command: @_";
    }
    elsif ($rc) { msg("Failure with return code $rc from command @_"); }
    return ( $rc, @output );
}

sub execCmds {

    my $or = $ctx->{OUT_REDIRECT};
    my $er = $ctx->{ERR_REDIRECT};

    my %Cmds = %{ $ctx->{ALLCMD} };

    msg(      "Executing "
            . ( keys %Cmds )
            . " commands for new incident at "
            . fmtTime( $ctx->{CUR_ITIME} ) );

    $ctx->{PRE_ITIME} = $ctx->{CUR_ITIME};

    my $id = $ctx->{CUR_INCDIR};

    while ( my ( $k, $v ) = each %Cmds ) {
        my $rm = $ctx->{RUN_MODE};
        if ($rm == 3)
        {
          if ($k =~ m/pstack\_/ || $k =~ m/top\_/ || $k =~ m/mpstat\_/ ||
              $k =~ m/vmstat\_/ || $k =~ m/ps\_/)
          { #Skip pstack, top, mpstat, vmstat, ps collection in case of plugin mode
            next;
          }
        }
        my $tf = catfile( $id, $k . '.trc' );
        my $cmd = qq($v $or "$tf" $er);

        my $pid = fork();

        if ($pid) {
            msg("Spawned process $pid for : $k");
            $allpids{$pid}++;
        }
        elsif ( $pid == 0 ) {
            eval {
                setpgrp( 0, 0 );
                printTrcHeader( $tf, $v );
                exec $cmd;
                exit 0;
            };
        }
        else {
            abortme("Fatal: Could not fork: $!");
        }
    }
    return 1;
}

#======= GENERIC ======

sub setInciDir {

    my $dpat   = $ctx->{CUR_INCDIR_PFIX};
    my $trbase = $ctx->{TRACE_BASE};
    my %thash;

    if ( !$dpat ) {
        $dpat = 'man';
        $ctx->{CUR_INCDIR_PFIX} = $dpat;
    }

    my $cmax = $ctx->{INCDIR_CNT}->{$dpat};

    opendir( my $DIR, $trbase );
    while ( my $entry = readdir $DIR ) {
        next unless -d $trbase . '/' . $entry;
        next if ( $entry =~ m/^\./ );
        if ( $entry =~ /^$dpat/ ) {
            my ( $pre, $cnt, $suf ) = split( '_', $entry );
            $thash{$cnt} = $entry;
        }
    }
    closedir $DIR;

    foreach my $c ( sort { $b <=> $a } keys %thash ) {
        my $nc = $c + 1;
        if ( $nc > $cmax ) {
            my ( $pre, $cnt, $suf ) = split( '_', $thash{$c} );
            my $dir = catdir( $trbase, $pre . "_" . $c . "_" . $suf );
            if ( -d $dir ) {
                remove_tree($dir) || err ("Error removing the $dir");
            }
        }
        elsif ( $nc <= $cmax ) {

            my ( $pre, $cnt, $suf ) = split( '_', $thash{$c} );
            my $odir = catdir( $trbase, $pre . "_" . $c . "_" . $suf );
            my $ndir = catdir( $trbase, $pre . "_" . $nc . "_" . $suf );
            if ( -d $odir && ( !-d $ndir ) ) {
                move( $odir, $ndir )
                    || err ("Rename directory $odir to $ndir failed");
                chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID}, $ndir;
            }
        }
    }

    my $id
        = catfile( $trbase, $dpat . "_1_" . fmtTrTime( $ctx->{CUR_ITIME} ) );
    if ( !-e $id ) {
        msg("New incident directory: $id");
        mkpath($id) or abortme("Fatal: Unable to create incident directory");
        chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID}, $id;
    }
    $ctx->{CUR_INCDIR} = $id;
    $ctx->{CUR_INCTRC} = catfile( $id, 'events.trc' );
}

sub updateCmds {
    my (%nCmd) = @_;
    my @keypat = split( '_', ( keys %nCmd )[0] ) if (%nCmd);

    if ( %{ $ctx->{ALLCMD} } ) {
        my %cmds = %{ $ctx->{ALLCMD} };

        foreach my $key ( keys %cmds ) {
            if ( $key =~ /^$keypat[0]\_/ ) {
                delete $ctx->{ALLCMD}->{$key};
            }
        }
    }

    if (%nCmd) {
        @{ $ctx->{ALLCMD} }{ keys %nCmd } = values %nCmd;
    }
}

sub removeCmd {
    my $key = shift;
    delete $ctx->{ALLCMD}->{$key};
}

sub queryVitalPids {
    my @pat = qw(ocssd.bin gipcd.bin);
    if ( isRoot() ) {
        push @pat, qw(crsd.bin ohasd.bin);
    }
    else {
        msg("Non-root user, skipping pstack of crsd.bin and ohasd.bin");
    }
    foreach my $p (@pat) {
        $ctx->{VITAL_PROCESS}{$p} = ( pgrep($p) )[0];
    }
}

sub updateNodeDetails {
    my $olsnodes = catfile( $ctx->{OH_BIN}, 'olsnodes' );
    my $cmd = qq($olsnodes -n -i -s -t);
    my ( $rc, @out ) = runCmd_silent($cmd);
    my ( @res, @nodes, @onodes );

    foreach my $line (@out) {
        if ( $line =~ /^PRCO-2/ ) {
            abortme("Node discovery failed. Clusterware not running");
        }
    }

    if ( $rc >= 0 ) {
        msg("Output from the olsnodes:");
        foreach (@out) {
            @res = split( ' ', $_ );
            push( @nodes, $res[0] );
            push( @onodes, join( '~', @res ) );
            msg("   $_");
        }
        $ctx->{NODELST}  = \@nodes;
        $ctx->{OLSNODES} = \@onodes;
    }
    else {
        abortme("Node discovery failed.");
    }

    getOifCfg();
}

sub getOifCfg {
    my $oifcfg = catfile( $ctx->{OH_BIN}, 'oifcfg' );
    my $cmd = qq($oifcfg getif -global);
    my ( $rc, @out ) = runCmd_silent($cmd);
    my ( @r, @effada, @res );

    if ( $rc >= 0 ) {
        msg("Output from the oifcfg:");
        foreach my $line (@out) {
            @r = split( ' ', $line );
            push( @effada, $r[0] );
            push( @res, join( '~', @r ) );
            msg("   $line");
        }
        $ctx->{EFFADA} = \@effada;
        $ctx->{OIFCFG} = \@res;
    }
}

sub updateInfDetails {

    my ( $su, $ssh, $sshopt, $cmd, $infPat, $inetPat, $mtuPat );
    my ( $inf, $key, $ip, $mtu, $found, @iflist );
    my ( $mtudef, $ishpux ) = 0;
    for ( my $pf = $ctx->{PLATFORM} ) {
        if (/darwin/) {
            $ssh     = '/usr/bin/ssh';
            $cmd     = '/sbin/ifconfig -a';
            $infPat  = '^(\S+)';
            $inetPat = 'inet (\S+)';
        }
        elsif (/linux/) {
            $su     = "/bin/su -l $ctx->{ORACLE_OWNER} -c";
            $ssh    = '/usr/bin/ssh';
            $sshopt = '-o StrictHostKeyChecking=no ';
            $sshopt .= '-o BatchMode=yes ';
            $sshopt .= '-o ConnectTimeout=5 ';
            $cmd     = '/sbin/ifconfig -a';
            $infPat  = '^(\S+)';
            $inetPat = 'inet addr:(\S+)';
            $mtuPat  = 'MTU:(\S+)';
        }
        elsif (/solaris/) {
            $su     = "/bin/su $ctx->{ORACLE_OWNER} -c";
            $ssh    = '/usr/bin/ssh';
            $sshopt = '-o StrictHostKeyChecking=no ';
            $sshopt .= '-o BatchMode=yes ';
            $sshopt .= '-o ConnectTimeout=5 ';
            $cmd     = '/sbin/ifconfig -a';
            $infPat  = '^(\S+)';
            $inetPat = 'inet (\S+)';
            $mtuPat  = 'mtu (\S+)';
        }
        elsif (/aix/) {
            $su     = "/bin/su $ctx->{ORACLE_OWNER} -c";
            $ssh    = '/usr/bin/ssh';
            $sshopt = '-o StrictHostKeyChecking=no ';
            $sshopt .= '-o BatchMode=yes ';
            $sshopt .= '-o ConnectTimeout=5 ';
            $cmd     = '/etc/ifconfig -a';
            $infPat  = '^(\S+):';
            $inetPat = 'inet (\S+)';
            $mtuPat  = 'mtu (\S+)';
            $mtudef  = 1;
        }
        elsif (/hpux/) {
            $su     = "/bin/su $ctx->{ORACLE_OWNER} -c";
            $ssh    = '/usr/bin/ssh';
            $sshopt = '-o StrictHostKeyChecking=no ';
            $sshopt .= '-o BatchMode=yes ';
            $sshopt .= '-o ConnectTimeout=5 ';
            $cmd     = '/usr/bin/netstat -in';
            $infPat  = '^(\S+)';
            $inetPat = 'inet (\S+)';
            $mtuPat  = 'mtu (\S+)';
            $ishpux  = 1;
        }
        else {
            abortme("getInfInfo - Invalid platform ($pf). Halting execution");
        }
    }

    $key = $inf = $ip = $mtu = '';
    foreach my $node ( @{ $ctx->{NODELST} } ) {
        my $found = 0;
        my $rCmd  = qq($ssh $sshopt $node $cmd);
        $rCmd = qq($su \"$rCmd\") if ( isRoot() );
        my ( $rc, @out ) = runCmd_silent($rCmd);
        if ( $rc >= 0 ) {
            foreach my $line (@out) {
                if ($ishpux) {
                    next if ( $line =~ m/Name/ );
                    my @r = split( ' ', $line );
                    $key = $node . '_' . $r[0];
                    $inf = $r[0];
                    $ip  = $r[3];
                    $mtu = $r[1];
                }
                else {
                    if ( $line =~ m/$infPat/ ) {
                        $key = $node . '_' . $1;
                        $inf = $1;
                    }
                    elsif ( $line =~ m/$inetPat/ && $ip eq '' ) {
                        $ip = $1;
                    }
                    elsif ( $line =~ m/$mtuPat/ && $mtu eq '' ) {
                        $mtu = $1;
                    }
                    else {
                        next;
                    }
                    $mtu = 1500 if ($mtudef);
                }

                if ( $key && $inf && $ip && $mtu ) {
                    my $r = $key;
                    $r .= '~~' . $key . "_" . $ip;
                    $r .= '~~' . $node;
                    $r .= '~~' . $inf;
                    $r .= '~~' . $ip;
                    $r .= '~~' . $mtu;

                    $ip = $mtu = '';
                    push @iflist, $r;
                    $found = 1;
                }
            }
        }
        if ( !$found ) {
            wrn("No interface information found for node $node");
            wrn("User equivalence failed - $out[0]");
            wrn("This node $node will ping tests will be ignored");
        }
    }
    upsertInf( \@iflist );
}

sub upsertInf {
    my @niflist = @{ $_[0] };
    my @oiflist = @{ $ctx->{ALLINFINFO} };
    my @uiflist = ();

    foreach my $n (@niflist) {
        my $ifound = 0;
        my ( $nkey, $nkey2, $nnode, $ninf, $nip, $nmtu ) = split( '~~', $n );

        if ( scalar(@oiflist) == 0 ) {
            push @uiflist, $n;
            msg("Adding new interface $ninf for $nnode with ip $nip");
            next;
        }

        foreach my $o (@oiflist) {
            my ( $okey, $okey2, $onode, $oinf, $oip, $omtu )
                = split( '~~', $o );
            if ( $nkey2 eq $okey2 ) {
                $ifound = 1;
                if ( $oip ne $nip ) {
                    msg("IP address updated for interface $ninf for $nnode from $oip to $nip"
                    );
                }
                elsif ( $omtu ne $nmtu ) {
                    msg("MTU updated for interface $ninf for $nnode from $omtu to $nmtu"
                    );
                }
                push @uiflist, $n;
                last;
            }
        }

        if ( !$ifound ) {
            msg("Adding new interface $ninf for $nnode with ip $nip");
            push @uiflist, $n;
        }
    }

    if ( scalar(@oiflist) > scalar(@niflist) ) {
        foreach my $o (@oiflist) {
            my $found = 0;
            my ( $okey, $okey2, $onode, $oinf, $oip, $omtu )
                = split( '~~', $o );
            foreach my $n (@niflist) {
                my ( $nkey, $nkey2, $nnode, $ninf, $nip, $nmtu )
                    = split( '~~', $n );
                if ( $okey2 eq $nkey2 ) {
                    $found = 1;
                    last;
                }
            }
            wrn("Interface $oinf not available on node $onode anymore, removing..."
            ) if !$found;
        }
    }

    $ctx->{ALLINFINFO} = \@uiflist;
}

sub validateCtx {
    if ( !isRoot() ) {
        if ( $ctx->{OWNER} ne $ctx->{ORACLE_OWNER} ) {
            say("Can be run either by 'root' or ' $ctx->{ORACLE_OWNER}' only\n"
            );
            abortme(
                "Can be run either by 'root' or ' $ctx->{ORACLE_OWNER}' only"
            );
        }
    }

    chdir( $ctx->{ORACLE_BASE} )
        or die(
        "init failed: switch to \$ORACLE_BASE failed (\'$ctx->{ORACLE_BASE}\')"
        );

    if ( !-e $ctx->{TRACE_BASE} ) {
        mkpath( $ctx->{TRACE_BASE} )
            or die(
            "init failed: Unable to create trace directory for $ctx->{TRACE_BASE}"
            );
        chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID},
            $ctx->{TRACE_BASE};
        chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID},
            $ctx->{PROD_BASE};
    }

    $ENV{'ORACLE_BASE'}     = $ctx->{ORACLE_BASE};
    $ENV{'ORACLE_HOME'}     = $ctx->{ORACLE_HOME};
    $ENV{'LD_LIBRARY_PATH'} = catfile( $ctx->{ORACLE_HOME}, 'lib' );

    checkRunMode();

    if ( $ctx->{OPT_MAP}{$opt} && $ctx->{RUN_MODE} > 1 ) {
        say("Log file : $ctx->{TRACE_FILE}");
    }

}

sub checkRunMode {

    if ( $ENV{'DS_PLUGIN_MODE'} ) {
        $ctx->{RUN_MODE} = 3;
        $ctx->{RUN_MODE_DESC}{3} = $ENV{'DS_PLUGIN_MODE'};
    }
    else {
        $ctx->{RUN_MODE} = 2;
    }

    if ( $ctx->{CRS_ENABLED} < 2 ) {
        $ctx->{RUN_MODE} = $ctx->{CRS_ENABLED};
    }

}

sub printLogHdr {
    trace( "-" x 79 );
    msg("DIAGnostic SNAPshot Collector Deamon ($ctx->{HOSTNAME} - $ctx->{PLATFORM})"
    );
    trace( "-" x 79 );
}

sub printTrcHeader {
    my ( $tf, $cmd ) = @_;
    my $hdrTime = gettimeofday;
    my $sptStr
        = "+------------------------------------------------------------------------------+\n";
    my $hdStr  = " Oracle Americas - Diagnostic SNAPshot Collection\n";
    my $msgStr = " " . fmtTime($hdrTime) . ": executing ($$): \"$cmd\"\n";
    open( FILE, ">> $tf" )
        or err ("Couldn't open $tf for writing: $!");
    print FILE $sptStr, $hdStr, $msgStr, $sptStr, "\n";
    close(FILE);
    chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID}, $tf;
    chmod 0600, $tf;
}

# ========= Internal Subs ========

sub isGridUp {
    my $crsctl = $ctx->{CRSCTL};
    my $cmd    = "$crsctl check crs";
    my @chk    = runCmd_silent($cmd);
    if ( scalar( grep( /4533/, @chk ) ) > 0 )
    {
        msg("EVMD is online");
        return 1;
    }
    else {
        msg("EVMD is down");
        return 0;
    }
}

sub isRegResource {
    my $res    = $ctx->{CRS_RES_NAME};
    my $crsctl = $ctx->{CRSCTL};
    my $rc     = 0;

    my $cmd = qq($crsctl status resource $res);
    my @chk = runCmd_silent($cmd);
    if ( ( scalar( grep( /2613/, @chk ) ) > 0 ) ) {
        $rc = 0;
    }
    else {
        $rc = 1;
    }

    return $rc;
}

sub isRoot {
    if ( $ctx->{OWNER} eq 'root' ) {
        return 1;
    }
    else {
        return 0;
    }
}

sub startCrsResource {
    my $res    = $ctx->{CRS_RES_NAME};
    my $crsctl = $ctx->{CRSCTL};
    my $rc     = 0;

    my $cmd = qq($crsctl start resource $res);
    $silent = 1;
    my ( $r, @chk ) = runCmd($cmd);
    $silent = 0;

    foreach my $line (@chk) {
        if ( $line =~ /2676/ ) {
            $rc = 1;
            last;
        }
        elsif ( $line =~ /0245/ ) {
            wrn("User '$ctx->{OWNER}' does not have privilege to start the resource, start failed"
            );
            $rc = 0;
            last;
        }
        elsif ( $line =~ /5702/ ) {
            wrn("$res is already running");
            $rc = 1;
            last;
        }
    }

    return $rc;
}

sub stopCrsResource {
    my $res    = $ctx->{CRS_RES_NAME};
    my $crsctl = $ctx->{CRSCTL};
    my $rc     = 0;

    my $cmd = qq($crsctl stop resource $res);

    $silent = 1;
    my ( $r, @chk ) = runCmd($cmd);
    $silent = 0;

    foreach my $line (@chk) {
        if ( $line =~ /2677/ ) {
            $rc = 1;
            last;
        }
        elsif ( $line =~ /0245/ ) {
            wrn("User '$ctx->{OWNER}' does not have privilege to stop the resource, stop failed"
            );
            $rc = 0;
            last;
        }
        elsif ( $line =~ /2675/ ) {
            wrn($line);
            $rc = 0;
            last;
        }
        elsif ( $line =~ /5809/ ) {
            wrn($line);
            wrn("Ignoring the above error and considering the resource is stopped"
            );
            $rc = 1;
            last;
        }
        elsif ( $line =~ /2500/ ) {
            wrn("$res is not running");
            $rc = 1;
            last;
        }
    }

    return $rc;
}

# ========== Helper Subs =========

sub fmtName {
    my $str = shift;
    $str =~ s/[\-]/\_/g;
    $str =~ s/[\.]/\_/g;
    $str =~ s/[\:]/\-/g;
    $str =~ s/[\s]/\-/g;
    return $str;
}

sub fmtTime {
    my $t = shift;
    my $date = strftime "%Y-%m-%d %H:%M:%S", localtime $t;
    $date .= sprintf ".%03d", ( $t - int($t) ) * 1000;
    return $date;
}

sub fmtTrTime {
    my $t = shift;
    $t = gettimeofday if ( !( length $t ) );
    my $date = strftime "%Y%m%d-%H%M%S", localtime $t;
    return $date;
}

sub fmtTimeDiff {
    my $t1  = [shift];
    my $t2  = [shift];
    my $res = tv_interval $t1, $t2;
    return sprintf( "%.3f", $res ) . " seconds";
}

sub trim {
    my $str = $_;
    $str = shift;
    $str =~ s/^\s+//;
    $str =~ s/\s+$//;
    return $str;
}

sub pgrep {
    my $pat = shift;
    my ( $cmd, $args, $plat );

    $args = $pat;

    for ( my $pf = $^O ) {
        if (/darwin/) {
            $cmd  = "/bin/ps";
            $args = "-n -t 10";
        }
        elsif (/linux/) {
            $cmd = "/usr/bin/pgrep";
        }
        elsif (/solaris/) {
            $cmd = "/usr/bin/pgrep";
        }
        elsif (/aix/) {
            $cmd  = "ps -ef | grep $args | grep -v grep |awk '{print \$2}'";
            $args = "";
        }
        elsif (/hpux/) {
            $cmd = "/usr/bin/pgrep";
        }
        else {
            wrn("pgrep - Invalid platform ($pf)- skipping commands");
            return 0;
        }
    }

    $cmd = qq($cmd $args);
    my ( $rc, @out ) = runCmd_silent($cmd);
    return @out;
}

sub getOwnerForPid {
    my $pid = shift;
    my $procf = catfile( '/proc', $pid );
    if ( -e $procf ) {
        return getpwuid( ( lstat($procf) )[4] );
    }
    return $ctx->{OWNER};
}

sub checkHeartBeat {

    my $timeout = 30;

    msg("Starting Heartbeat Thread");

    while (1) {
        my $cur_time = gettimeofday;
        my $hbt_time = $heartbeat ? $heartbeat : $cur_time;
        my $elapsed  = int tv_interval [$hbt_time], [$cur_time];

        if ( $elapsed >= $timeout ) {
            msg("Heartbeat Check: hb: $hbt_time ct: $cur_time el: $elapsed");
            wrn("Missing heartbeat for more than $timeout seconds");
            wrn("Will reincarnate myself, if running as CRS resource");
            diagsnapClean();
            abortme("Reincarnating myself, Not a bug ...");
        }
        sleep($timeout);
    }

}

sub setPid {
    readPFile();
    $ctx->{PROG_PID}       = $$;
    $ctx->{TRIGGER}        = 0;
    $ctx->{PLUGIN_ENABLED} = $ctx->{RUN_MODE};
    $ctx->{PLUGIN_TRIGGER} = 0;
    writePFile();
    $ctx->{PROG_ALIVE} = 1;
    chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID}, $ctx->{PID_FILE};
    chmod 0600, $ctx->{PID_FILE};
}

sub getPid {
    readPFile();
    return $ctx->{PFILE_PID};
}

sub getPluginEnabled {
    readPFile();
    return $ctx->{PLUGIN_ENABLED};
}

sub setPluginEnabled {
    readPFile();
    $ctx->{PLUGIN_ENABLED} = 1;
    $ctx->{PROG_PID}       = $ctx->{PFILE_PID};
    writePFile();
}

sub unSetPluginEnabled {
    readPFile();
    $ctx->{PLUGIN_ENABLED} = 0;
    $ctx->{PROG_PID}       = $ctx->{PFILE_PID};
    writePFile();
}

sub getStartCount {
    readPFile();
    return $ctx->{START_COUNT};
}

sub setStartCount {
    readPFile();
    if ( $ctx->{START_COUNT} >= $ctx->{MAX_START_COUNT} ) {
        $ctx->{START_COUNT} = 0;
    }
    else {
        $ctx->{START_COUNT}++;
    }
    $ctx->{PROG_PID} = $ctx->{PFILE_PID};
    writePFile();
}

sub getCheckCount {
    readPFile();
    return $ctx->{CHECK_COUNT};
}

sub resetCheckCount {
    readPFile();
    $ctx->{CHECK_COUNT} = 0;
    $ctx->{PROG_PID}    = $ctx->{PFILE_PID};
    writePFile();
}

sub setCheckCount {
    readPFile();
    if ( $ctx->{CHECK_COUNT} >= $ctx->{MAX_CHECK_COUNT} ) {
        $ctx->{CHECK_COUNT} = 1;
    }
    else {
        $ctx->{CHECK_COUNT}++;
    }
    $ctx->{PROG_PID} = $ctx->{PFILE_PID};
    writePFile();
}

sub checkTrigger {
    if ( !-e $ctx->{PID_FILE} ) {
        abortme("PID file not accessible, exiting...");
    }
    readPFile();
    my $trg = $ctx->{TRIGGER};
    $ctx->{TRIGGER}  = 0;
    $ctx->{PROG_PID} = $ctx->{PFILE_PID};
    writePFile();
    return $trg;
}

sub setTrigger {
    readPFile();
    $ctx->{TRIGGER}  = 1;
    $ctx->{PROG_PID} = $ctx->{PFILE_PID};
    writePFile();
}

sub checkPlgTrigger {
    if ( !-e $ctx->{PID_FILE} ) {
        abortme("PID file not accessible, exiting...");
    }
    readPFile();
    my $ptrg = $ctx->{PLUGIN_TRIGGER};
    $ctx->{PLUGIN_TRIGGER} = 0;
    $ctx->{PLUGIN_ENABLED} = $ctx->{RUN_MODE};
    $ctx->{PROG_PID}       = $ctx->{PFILE_PID};
    writePFile();
    return $ptrg;
}

sub setPlgTrigger {
    readPFile();
    $ctx->{PLUGIN_TRIGGER} = shift;
    $ctx->{PROG_PID}       = $ctx->{PFILE_PID};
    $ctx->{PLUGIN_ENABLED} = $ctx->{RUN_MODE};
    writePFile();
}

sub writePFile {
    my $pfile  = $ctx->{PID_FILE};
    my $myPid  = $ctx->{PROG_PID};
    my $mySCnt = $ctx->{START_COUNT};
    my $myCCnt = $ctx->{CHECK_COUNT};
    my $myTrg  = $ctx->{TRIGGER};
    my $myPEn  = $ctx->{PLUGIN_ENABLED};
    my $myPTrg = $ctx->{PLUGIN_TRIGGER};
    my $myHB   = gettimeofday;

    my $in = "$myPid||$mySCnt||$myCCnt||$myTrg||$myPEn||$myPTrg||$myHB\n";

    local *WRITEP;
    open WRITEP, "> $pfile" or abortme("Write PID failed");
    flock( WRITEP, 2 );
    print WRITEP $in;
    close WRITEP;

    return $myPid;
}

sub readPFile {
    my $pfile = $ctx->{PID_FILE};
    if ( -e $pfile ) {
        local *READP;
        open READP, "< $pfile" or abortme("Read PID failed");
        flock( READP, 1 );
        chomp( my $out = <READP> );
        close READP;
        my ( $pid, $scnt, $ccnt, $trg, $pen, $ptrg, $hb )
            = split( '\|\|', $out );
        if ( !$pid ) {
            if ( $ctx->{RUN_MODE} > 1 && $opt eq 'start' ) {
                say("Failed starting $ctx->{SCRIPT_NAME}, please retry again."
                );
            }
            unlink $pfile;
            abortme(
                "Empty PID read, cleared dat file. retry should succeed.");
        }
        $ctx->{PFILE_PID}      = $pid;
        $ctx->{START_COUNT}    = $scnt;
        $ctx->{CHECK_COUNT}    = $ccnt;
        $ctx->{TRIGGER}        = $trg;
        $ctx->{PLUGIN_ENABLED} = $pen;
        $ctx->{PLUGIN_TRIGGER} = $ptrg;
        $ctx->{HEART_BEAT}     = $hb;
    }
    else {
        $ctx->{PFILE_PID}      = $$;
        $ctx->{START_COUNT}    = 0;
        $ctx->{CHECK_COUNT}    = 1;
        $ctx->{TRIGGER}        = 0;
        $ctx->{PLUGIN_ENABLED} = 0;
        $ctx->{PLUGIN_TRIGGER} = 0;
        $ctx->{HEART_BEAT}     = gettimeofday;
    }
}

sub daemonize {

    chdir $ctx->{ORACLE_BASE};
    umask 0;
    my $pid = fork();

    if ( $pid < 0 ) {
        abortme("Failed to daemonize: $!");
    }
    elsif ($pid) {
        say("Starting daemon. Monitor logs for successful start.")
            if ( $ctx->{RUN_MODE} > 1 );
        exit 0;
    }

    ( setsid() != -1 ) || die "Can't start a new session: $!";

    msg("Switching as deamon with new pid $$");
    setPid();
    setStartCount();

    initSignals();

    open( STDIN,  "</dev/null" );
    open( STDOUT, ">/dev/null" );
    open( STDERR, ">&STDOUT" );
}

sub initSignals {

    $SIG{CHLD} = 'IGNORE';
    $SIG{INT} = $SIG{TERM} = $SIG{HUP} = $SIG{KILL} = \&ENDME;
}

sub REAPER {
    my $pid;
    while ( ( $pid = waitpid( -1, &WNOHANG ) ) > 0 ) {
        msg("Child process $pid ended.");
        delete $allpids{$pid};
    }
    $SIG{CHLD} = \&REAPER;
}

sub ENDME {
    msg("DIAGnostic SNAPshot Daemon Killed/Ended successfully");
    if ($evtlsnrpid) { kill INT => $evtlsnrpid; }
    cleanThreads();
    cleanPids();
    $SIG{INT} = $SIG{TERM} = $SIG{HUP} = $SIG{KILL} = \&ENDME;
    exit $SUCCESS;
}

sub cleanThreads {
    foreach my $thr ( threads->list ) {
        if ( $thr->is_running() || $thr->is_joinable() ) {
            $thr->detach();
        }
        else {
            $thr->kill('KILL')->detach();
        }
    }
}

sub cleanPids {
    if (%allpids) {
        kill -1, keys %allpids;
    }
}

sub msgs {
    my $rm = $ctx->{RUN_MODE};
    msg(@_);
    say(@_) if ( $rm > 1 );
}

sub msg {
    trace("[MSG] @_");
}

sub say {
    my $date = gettimeofday;
    $date = fmtTime($date);
    print "$date: [$ctx->{SCRIPT_NAME}][$$] @_\n";
}

sub wrn {
    trace("[WRN] @_");
}

sub err {
    trace("[ERR] @_");
}

sub abortme {
    trace("[ABT] !!!FATAL: @_");
    kill INT => $$;
}

sub trace {
    my $date = gettimeofday;
    $date = fmtTime($date);
    local *TRCFILE;
    if ($ctx) {
        my $TRACE_FILE = $ctx->{TRACE_FILE};
        if ($TRACE_FILE) {
            rotateLogs() if ( -e $TRACE_FILE );
            open( TRCFILE, ">>$TRACE_FILE" )
                or die "Fatal: Can't open $TRACE_FILE for append: $!";
            print TRCFILE "$date: [$$]@_\n";
            close(TRCFILE);
            chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID},
                $TRACE_FILE;
        }
    }
    else {
        print "$date: [$$]@_\n";
    }
}

sub rotateLogs {
    my $logfile = $ctx->{TRACE_FILE};
    my $trbase  = $ctx->{TRACE_BASE};
    my $logsize = -s $logfile;
    my $maxsize = 10 * 1024 * 1024;
    my $maxlogs = 5;
    my $logname = $ctx->{PROG_NAME}[0] . '_' . $ctx->{HOSTNAME};

    my $logpat = $logname . '.l\d{2}$';
    my %thash;

    if ( $logsize > $maxsize ) {
        opendir( my $DIR, $trbase );
        while ( my $entry = readdir $DIR ) {
            next if ( $entry =~ m/^\./ );
            if ( $entry =~ /^$logpat/ ) {
                my ( $fname, $fsuf ) = split( '\.l', $entry );
                $fsuf += 0;
                $thash{$fsuf} = $entry;
            }
        }
        closedir $DIR;

        foreach my $c ( sort { $b <=> $a } keys %thash ) {
            my $nc = $c + 1;
            if ( $nc > $maxlogs ) {
                my $log
                    = catfile( $trbase, $logname . sprintf( ".l%02d", $c ) );
                if ( -e $log ) {
                    unlink $log or err ("Unable to delete file $log");
                }
            }
            elsif ( $nc <= $maxlogs ) {
                my $nlog
                    = catfile( $trbase, $logname . sprintf( ".l%02d", $nc ) );
                my $olog
                    = catfile( $trbase, $logname . sprintf( ".l%02d", $c ) );
                if ( -e $olog && ( !-e $nlog ) ) {
                    move( $olog, $nlog )
                        || err ("Rename file $olog to $nlog failed");
                    chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID},
                        $nlog;
                }
            }
        }

        my $flog = catfile( $trbase, $logname . sprintf( ".l%02d", 1 ) );
        move( $logfile, $flog )
            or err ("Rename file $logfile to $flog failed");
        chown $ctx->{ORACLE_OWNER_ID}, $ctx->{ORACLE_GROUP_ID}, $flog;
        msg("Log file rotated, since the previous one exceeded 10M.");
    }

}

# ========== CTX Package =========
{

    package ctx;
    use Data::Dumper;
    use Time::HiRes qw/ time sleep gettimeofday /;
    use Carp;
    use Cwd;
    use Config;
    use Sys::Hostname;
    use POSIX;
    use File::Path;
    use File::Basename;
    use File::Spec::Functions;
    use Symbol 'gensym';

    sub new {
        my $this  = shift;
        my $class = ref($this) || $this;
        my $self  = {};
        bless $self, $class;
        $self->initialize();
        return $self;
    }

    sub initialize {
        my $self = shift;
        $self->{PROG_NAME}    = [ fileparse( $0, qr{\.[^.]*$} ) ];
        $self->{SCRIPT_NAME}  = basename($0);
        $self->{SCRIPT}       = Cwd::abs_path($0);
        $self->{PROG_PID}     = $$;
        $self->{PROG_ALIVE}   = 0;
        $self->{START_TIME}   = gettimeofday;
        $self->{HOSTNAME}     = hostname;
        $self->{PLATFORM}     = @Config{qw(osname)};
        $self->{CRS_RES_NAME} = $self->{PROG_NAME}[0];
        $self->{OWNER}        = ( getpwuid $> );

        $self->{ORACLE_BASE}  = $DS_ORACLE_BASE;
        $self->{ORACLE_HOME}  = $DS_ORACLE_HOME;
        $self->{ORACLE_OWNER} = $DS_ORACLE_OWNER;
        $self->{ORACLE_GROUP} = $DS_ORA_DBA_GROUP;

        $self->{ORACLE_OWNER_ID} = getpwnam $self->{ORACLE_OWNER};
        $self->{ORACLE_GROUP_ID} = getgrnam $self->{ORACLE_GROUP};

        $self->{MAX_START_COUNT} = 25;
        $self->{MAX_CHECK_COUNT} = 60;

        $self->{EXEC_TIMEOUT}   = 20;
        $self->{CHECK_INTERVAL} = 900;
        $self->{EVT_CNT}        = 50;
        $self->{REF_CNT}        = 6;
        $self->{MAN_CNT}        = 10;

        if ( $ENV{'_CRS_ENABLED'} ) {
            $self->{CRS_ENABLED} = $ENV{'_CRS_ENABLED'};
        }
        else {
            $self->{CRS_ENABLED} = 2;
        }

        $self->{CRS_ACTIVE_VERSION} = '';

        $self->{SCRIPT_BASE} = getcwd;
        $self->{USER} = $ENV{LOGNAME} || $ENV{USER} || getpwuid($<);

        $self->{PROD_BASE}
            = catfile( $self->{ORACLE_BASE}, $self->{PROG_NAME}[0] );

        $self->{TRACE_BASE}
            = catfile( $self->{ORACLE_BASE}, $self->{PROG_NAME}[0],
            $self->{HOSTNAME} );

        $self->{TRACE_FILE}
            = catfile( $self->{TRACE_BASE},
            $self->{PROG_NAME}[0] . '_' . $self->{HOSTNAME} . '.log' );

        $self->{CMD_FILE} = catfile( $self->{TRACE_BASE},
                  $self->{PROG_NAME}[0] . '_'
                . $self->{HOSTNAME} . '_'
                . $self->{OWNER}
                . '.cmd' );

        $self->{PID_FILE}
            = catfile( $self->{TRACE_BASE},
            '.' . $self->{PROG_NAME}[0] . '.dat' );

        $self->{OH_BIN} = catfile( $self->{ORACLE_HOME}, 'bin' );
        $self->{CRSCTL} = catfile( $self->{OH_BIN},      'crsctl' );

        $self->{OUT_REDIRECT} = '>>';
        $self->{ERR_REDIRECT} = '2>&1';

        $self->{LOOP_INT} = 0.5;

        $self->{ALLINFINFO} = [];
        $self->{NODELST}    = [];
        $self->{OLSNODES}   = [];
        $self->{EFFADA}     = [];
        $self->{OIFCFG}     = [];
        $self->{ALLCMD}     = {};

        $self->{CUR_ITIME} = '';
        $self->{PRE_ITIME} = '';

        $self->{PFILE_PID}      = '';
        $self->{START_COUNT}    = 0;
        $self->{CHECK_COUNT}    = 1;
        $self->{TRIGGER}        = 0;
        $self->{PLUGIN_ENABLED} = 0;
        $self->{PLUGIN_TRIGGER} = 0;
        $self->{HEART_BEAT}     = gettimeofday;

        $self->{RUN_MODE}      = 0;
        $self->{RUN_MODE_DESC} = '';

        $self->{SKIP_COLLECT} = 0;

        $self->{CUR_INCDIR_PFIX} = '';
        $self->{INCDIR_CNT}      = {
            'ref' => $self->{REF_CNT},
            'man' => $self->{MAN_CNT},
            'evt' => $self->{EVT_CNT}
        };

        $self->{CONF_PARAMS} = {
            'EXEC_TIMEOUT'   => 1,
            'CHECK_INTERVAL' => 1,
            'EVT_CNT'        => 1,
            'REF_CNT'        => 1,
            'MAN_CNT'        => 1
        };

        $self->{RUN_MODE_DESC} = {
            '0' => 'disabled CRS resource',
            '1' => 'CRS resource',
            '2' => 'Stand Alone',
            '3' => '<Third Party>'
        };

        $self->{VITAL_PROCESS} = {
            'ocssd.bin' => 0,
            'gipcd.bin' => 0,
            'crsd.bin'  => 0,
            'ohasd.bin' => 0
        };

        $self->{OPT_MAP} = {
            'start'      => 1,
            'stop'       => 1,
            'check'      => 1,
            'snap'       => 1,
            'register'   => 1,
            'deregister' => 1
        };

        return $self;
    }

    sub DESTROY {
        my $self = shift;
    }

}

sub printCmdFileTempl {
    return my $templ = <<EOF;
#+-----------------------------------------------------------------------------------+
#|            Clusterware diagnositics User commands/config                          |
#+-----------------------------------------------------------------------------------+
#|The commands in this file will be executed by $ctx->{PROG_NAME}[0] script.                     |
#|Enter commands in the following format. These commands will be synced              |
#|automatically next time a event is received. Please ensure the command             |
#|completes within EXEC_TIMEOUT seconds, else the commands will be terminated.       |
#|                                                                                   |
#| <command_name>~~<command line>                                                    |
#|  command_name - Command identifier. Must be unique and single word.               |
#|  ~~           - Seperator (double tilde)                                          |
#|  command line - Command to executed.                                              |
#|                                                                                   |
#|Eg:                                                                                |
#|   netstat~~netstat -r                                                             |
#|                                                                                   |
#|Also you can add config parameters starting with p_<PARAMETER>~~<value>            |
#|Following parameters can be configured.                                            |
#|   EXEC_TIMEOUT   - Time out for the commands to be executed. Default 20 secs      |
#|   CHECK_INTERVAL - How often a reference collection to be made. Default 900 secs  |
#|   EVT_CNT        - Number of event Collections. Default 50                        |
#|   REF_CNT        - Number of reference Collections. Default 6                     |
#|   MAN_CNT        - Number of manual Collections. Default 10                       |
#|                                                                                   |
#|Eg:                                                                                |
#|   p_CHECK_INTERVAL~~900                                                           |
#+-----------------------------------------------------------------------------------+
p_CHECK_INTERVAL~~900
netstat~~netstat -r
EOF
}
