#!/usr/local/bin/perl
# 
# jwcctl_lib.pm
# 
# Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
#
#    NAME
#      jwcctl_lib.pm - JWC Control Action Script Generic Library Module
#
#    DESCRIPTION
#      jwcctl_lib.pm - Generic library module for JWC Control
#
#    NOTES
#
#    MODIFIED   (MM/DD/YY)
#    gmaldona    06/28/16 - Bug-23711491, Add more traces.
#    jgrout      06/15/16 - Fix bug 23590228
#    srghosha    06/03/16 - Add JVM system parameter of 
#                           freeable cache memory percentage
#    jgrout      05/17/16 - Fix bug 23301755
#    jgrout      05/16/16 - Fix bug 23274763
#    jgrout      04/26/16 - Fix bug 22720990
#    espgarci    04/22/16 - Add jwc-security.jar for the shutter
#    jgrout      04/12/16 - Fix bug 22651447
#    jgrout      01/25/16 - Fix bug 22066785
#    gmaldona    01/20/16 - Add logger options to JWCChecker and 
#                           ShutdownContainer, get STDOUT and STDERR and remove
#                           Start event log file.
#    jgrout      01/20/16 - Remove permgen and add metaspace parameters
#    lluis       01/19/16 - Add TLS protocol support
#    srghosha    12/16/15 - add MPA log level
#    jgrout      08/05/15 - Remove active/software version check from start
#    jgrout      07/22/15 - Fix bug 21489522
#    jmunozn     06/18/15 - Replace consoleapps.jar with jwc-cred.jar
#    gmaldona    05/26/15 - The checker must take the url from jwc.properties
#                           instead of the agent script.
#    jgrout      05/14/15 - Adapt to use Credentials Manager
#    gmaldona    05/12/15 - Change start and stop arguments
#    jgrout      04/17/15 - Adapt to use JWC Checker
#    jgrout      02/27/15 - Adapt for JWC Control from OC4J Control
#    jmunozn     11/10/14 - Include JAZN jar in shutter classpath
#    jgrout      10/15/14 - Fix bug 19426384
#    jgrout      08/22/14 - Fix bug 18605562
#    jgrout      06/10/14 - Fix indentations by removing all tabs
#    jgrout      05/13/14 - Fix bug 18372004
#    jgrout      03/21/14 - Fix bug 17815202
#    jgrout      03/13/14 - Fix bug 17920033
#    jgrout      03/06/13 - Forward merge fix for bug 15968944 to main
#    dsemler     10/04/12 - fix comparison operator in stop loop
#    dsemler     10/03/12 - Bug 14619154 : set the isleep_interval default to
#                           1 second
#    ekhabie     05/16/12 - Update to new ShutdownOc4j location
#    jgrout      03/09/12 - Fix bug 13831825
#    jgrout      02/07/12 - Created

package jwcctl_lib;

use strict;
use warnings;
use File::Spec::Functions;
use Config;
use jwcctl_common;
use s_jwcctl_lib;

our $VERSION = '1';

use English;
use Exporter;

# Implementation constants
use constant {
  CHECKER_CLASS => "oracle.cluster.jwc.tomcat.client.JWCChecker" ,
  CHECKER_INITIAL => "c" ,
  JWC_CLIENT_JAR => "jwc-client.jar" ,
  JWC_SECURITY_JAR => "jwc-security.jar",
  JWC_LOGGING_JAR => "jwc-logging.jar" ,
  JWCCRED_JAR => "jwc-cred.jar" ,
  TOMCAT_LOGGING_JAR => "tomcat-juli.jar" ,
  LOGGING_MANAGER_CLASS => "org.apache.juli.ClassLoaderLogManager" ,
  SHUTTER_INITIAL => "s" ,
  SHUTTER_CLASS => "oracle.cluster.jwc.tomcat.client.ShutdownContainer" ,
  TOMCAT_INITIAL => "d" ,
  TOMCAT_BOOTSTRAP_CLASS => "org.apache.catalina.startup.Bootstrap" ,
  TOMCAT_BOOTSTRAP_JAR => "bootstrap.jar"
};

# Do NOT use constants as hash keys - Perl 5.8 and later don't support it

# USR_ORA_ENV keys

our $dbwlm_logging_level_key = "DBWLM_LOGGING_LEVEL";
our $freeable_cache_memory_pct_key = "FREEABLE_CACHE_MEMORY_PCT";
our $mpa_logging_level_key = "MPA_LOGGING_LEVEL";
our $isleep_interval_key = "ISLEEP_INTERVAL";
our $logging_config_key = "LOGGING_CONFIG";
our $logging_manager_key = "LOGGING_MANAGER";
our $min_heap_size_key = "MIN_HEAP_SIZE";
our $max_heap_size_key = "MAX_HEAP_SIZE";
our $max_metaspace_size_key = "MAX_METASPACE_SIZE";
our $script_debug_key = "SCRIPT_DEBUG";
our $server_options_key = "SERVER_OPTIONS";
our $srvm_trace_level_key = "SRVM_TRACE_LEVEL";
our $use_scan_ip_key = "use_scan_IP";

# Global variable keys

our $catalina_home_key = "CATALINA_HOME";
our $catalina_base_key = "CATALINA_BASE";
our $jlib_key = "JLIB";
our $jre_key = "JRE";
our $pid_file_key = "PID_FILE";
our $shutter_log_name_key = "SHUTTER_LOG_NAME";
our $shutter_log_num_key = "SHUTTER_LOG_NUM";
our $shutter_log_size_key = "SHUTTER_LOG_SIZE";
our $checker_log_name_key = "CHECKER_LOG_NAME";
our $checker_log_num_key = "CHECKER_LOG_NUM";
our $checker_log_size_key = "CHECKER_LOG_SIZE";


our @exp_vars;

our %envhash;
our %globhash;

our (@ISA, @EXPORT);

# Export in a BEGIN block to avoid compilation failure

BEGIN {
  require Exporter;
  @ISA = qw(Exporter);

  my @exp_const = qw(SUCC_CODE FAIL_CODE ERROR_PID NULL_PID
                     ENTRY_SUCCESS ENTRY_FAIL
                     STATE_ONLINE STATE_OFFLINE STATE_UNKNOWN STATE_FAILED
                    );

  my @exp_func = qw(init_globals jwc_start jwc_stop jwc_check jwc_clean
                   );

  my @exp = (@exp_const, @exp_func);
  @EXPORT  = @exp;
}

sub init_globals
#---------------------------------------------------------------------
# Function: Initialize globals
#
# Args    : None
#
# Returns : SUCC_CODE (operation succeeded)
#           FAIL_CODE (operation failed)
#---------------------------------------------------------------------
{

  my $oracle_home = $ENV{"ORACLE_HOME"};
  my $catalina_home = $ENV{"CATALINA_HOME"};
  my $catalina_base = $ENV{"CATALINA_BASE"};

  my $jlib = catfile($oracle_home, "jlib");

  my $temp_dir = catfile($catalina_base, "temp");
  my $pid_file = catfile($temp_dir, ".jwcpid");

  # Initialize environment hash
  %envhash = ($dbwlm_logging_level_key => undef ,
              $freeable_cache_memory_pct_key  => undef ,
              $mpa_logging_level_key  => undef ,
              $isleep_interval_key => 1 ,
              $logging_config_key => undef ,
              $logging_manager_key => undef ,
              $min_heap_size_key => "128M" ,
              $max_heap_size_key => "384M" ,
              $max_metaspace_size_key => undef ,
              $script_debug_key => undef ,
              $server_options_key => "" ,
              $srvm_trace_level_key => undef,
              $use_scan_ip_key => undef);

  # Initialize global hash
  %globhash = ($catalina_home_key => $catalina_home ,
               $catalina_base_key => $catalina_base ,
               $jlib_key => $jlib ,
               $jre_key => get_jre() ,
               $pid_file_key => $pid_file,
               $shutter_log_name_key => "jwc_shutter_stdout_err_\%g.log",
               $shutter_log_num_key => "10",
               $shutter_log_size_key => "1048576",
               $checker_log_name_key => "jwc_checker_stdout_err_\%g.log",
               $checker_log_num_key => "10",
               $checker_log_size_key => "1048576");

  # Force STDOUT and STDERR to be flushed for each I/O operation
  select(STDERR);
  $| = 1;
  select(STDOUT);
  $| = 1;

  # Parse USR_ORA_ENV into environment hash
  my @envpairlist = split(/\s+/, $ENV{"_USR_ORA_ENV"});
  foreach my $envpair (@envpairlist)
  {
    my @envlist = split(/\=/, $envpair);
    if (@envlist == 2)
    {
      my ($opt, $optval) = @envlist;
      if (exists $envhash{$opt})
      {
        $envhash{$opt} = $optval;
        next;
      }
    }
    print STDERR
      " CRS_ERROR:WARNING: Invalid data $envpair in _USR_ORA_ENV\n";
  }

  if (defined($envhash{$script_debug_key}))
  {
    set_debug_out();
  }
  else
  {
    unset_debug_out();
  }

  return s_init_globals();

}

sub get_jwc_properties
#---------------------------------------------------------------------
# Function: Return JWC configuration properties 
#
# Args    : None
#
# Returns : JWC properties
#---------------------------------------------------------------------
{
  my %props;
  my $catalina_base = $globhash{$catalina_base_key};
  my $jwcFile = catfile($catalina_base, "conf", "jwc.properties");
  if (-e $jwcFile) {
    # Open jwc.properties file
    open my $fh, $jwcFile or die " CRS_ERROR:Could not open file '$jwcFile' ";
    foreach my $line (<$fh>) {
      chomp($line);
      my @tokens = split (/=/, $line);
      if (@tokens == 2 ) {
        $props{$tokens[0]}=$tokens[1];
      }
    }
    # Close jwc.properties
    close $fh;
  } else {
    print STDERR " CRS_ERROR:Could not open file '$jwcFile' ";
  }
  
  return %props;
}

sub get_jre
#---------------------------------------------------------------------
# Function: Return JRE file path
#
# Args    : None
#
# Returns : JRE file path
#---------------------------------------------------------------------
{
  return s_get_jre();
}

sub get_jstack
#---------------------------------------------------------------------
# Function: Return jstack command file path
#
# Args    : None
#
# Returns : jstack command file path
#---------------------------------------------------------------------
{
  return s_get_jstack();
}

sub get_crskeytoolctl
#---------------------------------------------------------------------
# Function: Return crskeytoolctl command file path
#
# Args    : None
#
# Returns : crskeytoolctl command file path
#---------------------------------------------------------------------
{
  return s_get_crskeytoolctl();
}

sub async_start
#---------------------------------------------------------------------
# Function: Start command asynchronously in the background
#
# Args    : Initial (for tomcat, shutter or checker)
#           Files for STDIN, STDOUT and STDERR
#           List with program name and its arguments
#
# Returns : None
#---------------------------------------------------------------------
{
  s_async_start(@_);
}

sub sync_start
#---------------------------------------------------------------------
# Function: Start command synchronously
#
# Args    : Initial (for tomcat, shutter or checker)
#           Files for STDIN, STDOUT and STDERR
#           List with program name and its arguments
#
# Returns : Result of synchronous command
#---------------------------------------------------------------------
{
  return s_sync_start(@_);
}

sub create_lock_file
#---------------------------------------------------------------------
# Function: Create the lock file in the given path. The lock file will be
#           open if it already exists
#
# Args    : path to the lock file to create
#
# Returns : Filehandle of the lock file
#---------------------------------------------------------------------
{
  return s_create_lock_file(@_);
}

sub file_lock
#---------------------------------------------------------------------
# Function: Lock a file (LOCK_EX)
#
# Args    : Filehandle of the file to lock
#
# Returns : None
#---------------------------------------------------------------------
{
  return s_file_lock(@_);
}

sub file_unlock
#---------------------------------------------------------------------
# Function: Unlock a file
#
# Args    : Filehandle of the file to unlock
#
# Returns : None
#---------------------------------------------------------------------
{
  return s_file_unlock(@_);
}

sub get_jwc_pid_list
#---------------------------------------------------------------------
# Function: Get a list of JWC container PID(s).
#
# Args    : None
#
# Returns : List of JWC container PID(s) (if any)
#---------------------------------------------------------------------
{
  debug_out("Get JWC PIDs");
  my @jwc_pid_list = s_get_jwc_pid_list(TOMCAT_INITIAL,
				    TOMCAT_BOOTSTRAP_CLASS);

  debug_out("Done Getting JWC PIDs");
  return @jwc_pid_list;
}

sub get_jwc_checker_pid_list
#---------------------------------------------------------------------
# Function: Get a list of JWC checker PID(s).
#
# Args    : None
#
# Returns : List of JWC checker command PID(s) (if any)
#---------------------------------------------------------------------
{
  debug_out("Get JWC checker PIDs");
  my @pid_list = s_get_jwc_executable_pid_list(CHECKER_INITIAL,
					       CHECKER_CLASS);
  debug_out("Done getting JWC checker PIDs");
  return @pid_list;
}

sub get_jwc_shutter_pid_list
#---------------------------------------------------------------------
# Function: Get a list of JWC shutter PID(s).
#
# Args    : None
#
# Returns : List of JWC shutter command PID(s) (if any)
#---------------------------------------------------------------------
{
  debug_out("Get JWC shutter PIDs");
  my @pid_list = s_get_jwc_executable_pid_list(SHUTTER_INITIAL,
					       SHUTTER_CLASS);
  debug_out("Done getting JWC shutter PIDs");
  return @pid_list;
}

sub read_pid_from_jwc_pid_file
#---------------------------------------------------------------------
# Function: Read the JWC PID from the JWC PID file
#
# Args    : Pid file
#
# Returns : JWC Container PID (if PID file is found)
#           NULL_PID (if PID file is not found)
#---------------------------------------------------------------------
{
  my ($pid_file) = @_;
  my $jwc_pid = NULL_PID;

  if (-e $pid_file && -r $pid_file)
  {
    open(PF, $pid_file);
    while(<PF>)
    {
      chomp;
      $jwc_pid = $_;
    }
    close(PF);
  }
  return $jwc_pid;
}

sub write_pid_to_jwc_pidfile_and_check
#---------------------------------------------------------------------
# Function: Write the JWC PID to the JWC PID file
#           and check that it was created successfully.
#
# Args    : PID file
#           JWC PID to write
#
# Returns : SUCC_CODE (operation succeeded)
#           FAIL_CODE (operation failed)
#---------------------------------------------------------------------
{
  my ($pid_file, $jwc_pid) = @_;

  open(PF, ">$pid_file");
  print PF "$jwc_pid\n";
  close(PF);

  my $read_pid = read_pid_from_jwc_pid_file($pid_file);
  if ($read_pid == NULL_PID)
  {
    print STDERR " CRS_ERROR:FATAL Error: Could not create $pid_file\n";
    return FAIL_CODE;
  }
  elsif ($read_pid != $jwc_pid)
  {
    print STDERR
      " CRS_ERROR:FATAL Error: PID file mismatch (PID $jwc_pid running," .
      " PID $read_pid read from PID file)\n";
    unlink($pid_file);
    return FAIL_CODE;
  }
  return SUCC_CODE;
}

sub send_signal
#---------------------------------------------------------------------
# Function: Send check, quit or kill signal
#
# Args    : Type of signal to send ("check", "quit" or "kill")
#           Process number
#
# Returns : SUCC_CODE (for "quit" and "kill", operation initiated;
#                      for "check", process still under control)
#           FAIL_CODE (operation not implemented)
#           ERROR_PID (process no longer under control)
#           NULL_PID  (process no longer exists)
#---------------------------------------------------------------------
{
  return s_send_signal(@_);
}

sub perform_jstack
#---------------------------------------------------------------------
# Function: Perform the jstack command 
#
# Args    : Process number
#
# Returns : jstack command return code
#---------------------------------------------------------------------
{
  my ($jwc_pid) = @_;
  my $catalina_base = $globhash{$catalina_base_key};
  my @jstack_command = (s_get_jstack(), "-l", $jwc_pid,
                        ">>",
                        catfile($catalina_base, "logs",
                                "jwc_jstack_stdout.log"));
  return system("@jstack_command");
}

sub get_jwc_pid
#---------------------------------------------------------------------
# Function: Get the JWC container PID(s).
#           If more than one, write an error message.
#
# Args    : None
#
# Returns : JWC Container PID (one JWC container found)
#           NULL_PID (no JWC container found)
#           ERROR_PID (multiple JWC containers found)
#---------------------------------------------------------------------
{
  my $jwc_pid;

  my @jwc_pid_list = get_jwc_pid_list();
  if (@jwc_pid_list > 1)
  {
    debug_out("... Multiple JWC Containers are running ...");
    print STDERR
      " CRS_ERROR:FATAL Error: Multiple JWC Containers running ".
      "with PIDs";
    foreach (@jwc_pid_list)
    {
      print STDERR " ", $_;
    }
    print STDERR "\n";
    $jwc_pid = ERROR_PID;
  }
  elsif (@jwc_pid_list == 1)
  {
    $jwc_pid = $jwc_pid_list[0];
    debug_out("... JWC Container (pid=$jwc_pid) ...");
  }
  else
  {
    $jwc_pid = NULL_PID;
    debug_out("... JWC containers not found ...");
  }
  return $jwc_pid;
}

sub jwc_start_post_processing
#---------------------------------------------------------------------
# Function: Verify that a single JWC container is running.
#           If so, write the PID file and check it.
#
# Args    : Pid file
#
# Returns : JWC Container PID (post-processing successful)
#           NULL_PID (no JWC container found)
#           ERROR_PID (error condition)
#---------------------------------------------------------------------
{
  my ($pid_file) = @_;
  my $jwc_pid = get_jwc_pid();

  if ($jwc_pid == NULL_PID)
  {
    debug_out("... JWC Container is not running ...");
  }
  elsif ($jwc_pid != ERROR_PID)
  {
    debug_out("... JWC Container running (pid=$jwc_pid) ...");
    my $rc = write_pid_to_jwc_pidfile_and_check($pid_file, $jwc_pid);
    if ($rc != SUCC_CODE)
    {
      $jwc_pid = ERROR_PID;
    }
  }
  return $jwc_pid;
}

sub jwc_check_servlet
#---------------------------------------------------------------------
# Function: Run JWC check servlet
#           Check if the resource running in the container is up/down.
#
# Args    : CHECK_ONCE (check only once) or
#           CHECK_UNTIL_ACTION_TIMEOUT (continue to check until
#           container comes up, has a fatal error or the action
#           script times out and is cancelled by the script agent)
#
# Returns : SUCC_CODE (Resource is o.k.)
#           FAIL_CODE (Resource has a fatal error)
#---------------------------------------------------------------------
{
  # HTTP State Code
  my ($checker_arg) = @_;
  my $http_port = $ENV{"HTTP_PORT"};

  # Prepare to run the Java checker
  my $check_rc = SUCC_CODE;
  my $jlib = $globhash{$jlib_key};
  my $jre = $globhash{$jre_key};
  my $catalina_base = $globhash{$catalina_base_key};
  my $path_separator = $Config{path_sep};

  # Change directories to $tomcat_log_dir before launching
  my $tomcat_log_dir = catfile($catalina_base, "logs");
  chdir($tomcat_log_dir) || 
    print STDERR "Could not change directory to $tomcat_log_dir\n";
  
  # Tracing options
  my $tomcat_stdout_err = catfile($tomcat_log_dir,
                                  $globhash{$checker_log_name_key});

  my @dbwlm_tracing_properties;
  if (defined($envhash{$dbwlm_logging_level_key}))
  {
    @dbwlm_tracing_properties = ("-Doracle.wlm.dbwlmlogger.logging.level=".
                                 $envhash{$dbwlm_logging_level_key});
  }
                              
  @dbwlm_tracing_properties
    = (@dbwlm_tracing_properties,
       "-Doracle.jwc.client.logger.file.name=".
       $tomcat_stdout_err,
       "-Doracle.jwc.client.logger.file.number=".
       $globhash{$checker_log_num_key},
       "-Doracle.jwc.client.logger.file.size=".
       $globhash{$checker_log_size_key});
       
  # Execute the checker
  my @command_parms = (split(/\s+/, $ENV{"JRE_OPTIONS"}),
                       "-Djava.net.preferIPv6Addresses=true",
                       "-Dcatalina.base=$catalina_base",
                       @dbwlm_tracing_properties,
                       "-classpath",
                       catfile($jlib, JWC_LOGGING_JAR) . $path_separator .
                       catfile($jlib, JWC_CLIENT_JAR),
                       CHECKER_CLASS,
                       LOCALHOST,
                       $http_port,
                       $checker_arg);

  my $initial = CHECKER_INITIAL;
  my $sync_rc = sync_start($initial, NULLFILE, NULLFILE, NULLFILE,
                           $jre, @command_parms);

  if ($sync_rc == -1)
  {
    print STDERR "sync_start failed to execute: $sync_rc\n";
    $check_rc = FAIL_CODE;
  }
  else
  {
    my $sync_exit_code = $sync_rc >> 8;
    if ($sync_exit_code > 0)
    {
      print STDERR "sync_start failed with exit code $sync_exit_code.\n";
      $check_rc = FAIL_CODE;
    }
    else
    {
      my $sync_signal = $sync_rc & 127;
      if ($sync_signal > 0)
      {
        my $sync_signal_coredump = ($sync_rc & 128) ? 'with' : 'without';
        print STDERR "sync_start child died with signal $sync_signal ".
                     "$sync_signal_coredump coredump.\n";
        $check_rc = FAIL_CODE;
      }
    }
  }
  return $check_rc;
}


sub jwc_start
#---------------------------------------------------------------------
# Function: Perform JWC start action
#
# Args    : None
#
# Returns : ENTRY_SUCCESS  (action succeeded)
#           ENTRY_FAIL     (action failed)
#---------------------------------------------------------------------
{
  print STDOUT "Start JWC\n";
  my $start_rc = ENTRY_FAIL;
  my $iter_cnt = 0;

  my $min_heap_size = $envhash{$min_heap_size_key};
  my $max_heap_size = $envhash{$max_heap_size_key};
  my $server_options = $envhash{$server_options_key};

  my $jre = $globhash{$jre_key};
  my $catalina_home = $globhash{$catalina_home_key};
  my $catalina_base = $globhash{$catalina_base_key};
  my $pid_file = $globhash{$pid_file_key};
  my $path_separator = $Config{path_sep};
  my $jlib = $globhash{$jlib_key};

  # Remove startup event log file before to start TOMCAT
  my %props = get_jwc_properties();
  my $startEvent = exists $props{'oracle.jwc.lifecycle.start.log.fileName'} 
    ? $props{'oracle.jwc.lifecycle.start.log.fileName'}
    : '';
  if ($startEvent eq '') {
    print STDERR "CRS_ERROR:Start event log file name is missing \n";
    return FAIL_CODE;
  }
  my $startEventFile = catfile($catalina_base, "logs", $startEvent);
  if (-f $startEventFile) {
    unlink($startEventFile);
  }
  
  # Collect Java properties

  my @jre_heap_options = ("-Xms$min_heap_size", "-Xmx$max_heap_size");
  if (defined($ENV{"JRE_HEAP_OPTIONS"}))
  {
    @jre_heap_options = (split(/\s+/, $ENV{"JRE_HEAP_OPTIONS"}),
                         @jre_heap_options);
  }
  if (defined($envhash{$max_metaspace_size_key}))
  {
    @jre_heap_options = (@jre_heap_options,
                         "-XX:MaxMetaspaceSize=" . 
                         $envhash{$max_metaspace_size_key});
  }

  my @java_properties = ("-Djava.awt.headless=true",
                         "-Ddisable.checkForUpdate=true",
                         s_java_path_defs());

  # Collect Tomcat properties

  my $logging_config_file;
  if (defined($envhash{$logging_config_key}))
  {
    $logging_config_file = $envhash{$logging_config_key};
  }
  else
  {
    $logging_config_file = catfile($catalina_base, "conf",
                                  "logging.properties");
  }
  my $logging_manager;
  if (defined($envhash{$logging_manager_key}))
  {
    $logging_manager = $envhash{$logging_manager_key};
  }
  else
  {
    $logging_manager = LOGGING_MANAGER_CLASS;
  }

  my @catalina_logging_properties = ("-Djava.util.logging.config.file=" .
                                     $logging_config_file,
                                     "-Djava.util.logging.manager=" .
                                     $logging_manager);

  my @srvm_tracing_properties;
  if (defined($envhash{$srvm_trace_level_key}))
  {
    @srvm_tracing_properties = ("-DTRACING.ENABLED=true",
                                "-DTRACING.LEVEL=" .
                                $envhash{$srvm_trace_level_key});
  }
  else
  {
    @srvm_tracing_properties = ("-DTRACING.ENABLED=false");
  }
  my @dbwlm_tracing_property;
  if (defined($envhash{$dbwlm_logging_level_key}))
  {
    @dbwlm_tracing_property = ("-Doracle.wlm.dbwlmlogger.logging.level=".
			       $envhash{$dbwlm_logging_level_key});
  }
  my @mpa_tracing_property;
  if (defined($envhash{$mpa_logging_level_key}))
  {
    @mpa_tracing_property = ("-Doracle.wlm.mpalogger.logging.level=".
			     $envhash{$mpa_logging_level_key});
  }
  my @cache_memory_pct_property;
  if (defined($envhash{$freeable_cache_memory_pct_key}))
  {
     @cache_memory_pct_property =
         ("-Doracle.wlm.mpa.freeable_cache_memory_pct=".
              $envhash{$freeable_cache_memory_pct_key});
  }
  my @dbwlm_test_properties;
  if (defined($envhash{$use_scan_ip_key}))
  {
    @dbwlm_test_properties = ("-Duse_scan_IP=" . $envhash{$use_scan_ip_key});


  }
  
  my @port_properties = ("-Doracle.http.port=" . $ENV{"HTTP_PORT"},
                         "-Doracle.jmx.port=" . $ENV{"RMI_PORT"});
                         
  # TLS protocol enablement
  my $tls_enabled = getTLSEnabledProperty();
  if ($tls_enabled eq "true") {
    genNodeKeyData();
  }

  my @tls_properties = ("-Doracle.tls.enabled=" . $tls_enabled);
                         
  my @catalina_properties = (@catalina_logging_properties,
                             @srvm_tracing_properties,
                             @dbwlm_tracing_property,
                             @mpa_tracing_property,
                             @cache_memory_pct_property,
                             @dbwlm_test_properties,
                             @port_properties,
                             @tls_properties,
                             "-Dcatalina.home=" . $catalina_home,
                             "-Dcatalina.base=" . $catalina_base,
                             "-Djava.io.tmpdir=" . catfile($catalina_base,
                                                           "temp"),
                             "-Doracle.home=" . $ENV{"ORACLE_HOME"});

  my $jwc_stdout_err = catfile($catalina_base, "logs", "catalina.out");
  my $crs_start_timeout = $ENV{"_CRS_START_TIMEOUT"};
  debug_out("... JWC start timeout is $crs_start_timeout seconds");
  my $sleep_interval = $envhash{$isleep_interval_key};

  # Change directories to $tomcat_log_dir before launching
  my $tomcat_log_dir = catfile($catalina_base, "logs");
  chdir($tomcat_log_dir) || 
    print STDERR "Could not change directory to $tomcat_log_dir\n";
  
  # First, check if a previous JWC process is running
  my $jwc_pid = get_jwc_pid();
  if ($jwc_pid != NULL_PID)
  {
    if ($jwc_pid != ERROR_PID)
    {
      print STDERR " CRS_ERROR:FATAL Error: JWC already " .
                   "running with PID ";
      print STDERR "$jwc_pid\n";
    }
    jwc_sleep_to_timeout($crs_start_timeout);
  }

  # Next, remove stale PID file (if any)
  unlink($pid_file);

  # Start the container
  my @command_parms = (split(/\s+/, $ENV{"JRE_OPTIONS"}),
                       split(/\s+/, $ENV{"JRE_OPTIONS_SERVER"}),
                       split(/\s+/, $server_options),
                       @jre_heap_options, @java_properties,
                       @catalina_properties,
                       "-classpath",
                       catfile($catalina_home, "lib", TOMCAT_LOGGING_JAR) .
                       $path_separator .
                       catfile($catalina_home, "lib", TOMCAT_BOOTSTRAP_JAR).
                       $path_separator .
                       catfile($jlib, JWC_LOGGING_JAR),
                       TOMCAT_BOOTSTRAP_CLASS, "start");

  my $initial = TOMCAT_INITIAL;
  async_start($initial, NULLFILE,
	      $jwc_stdout_err, $jwc_stdout_err,
	      $jre, @command_parms);

  # Begin JWC start post processing
  $jwc_pid = jwc_start_post_processing($pid_file);
  if ($jwc_pid == ERROR_PID)
  {
    jwc_sleep_to_timeout($crs_start_timeout);
  }

  while ($jwc_pid == NULL_PID)
  {
    sleep($sleep_interval);

    # Continue JWC start post processing
    $jwc_pid = jwc_start_post_processing($pid_file);
    if ($jwc_pid == ERROR_PID)
    {
        jwc_sleep_to_timeout($crs_start_timeout);
    }
  }
  
  # Run JWC check servlet using HTTP port from environment
  my $rc = jwc_check_servlet(CHECK_UNTIL_ACTION_TIMEOUT);
  if ($rc == FAIL_CODE)
  {
    debug_out("... JWC Container is not ready ...");
    jwc_sleep_to_timeout($crs_start_timeout);
  }
  $start_rc = ENTRY_SUCCESS;
  debug_out("... JWC Container is ready ...");

  return $start_rc;
}

sub get_jwc_shutter_pid
#---------------------------------------------------------------------
# Function: Get the JWC shutter command PID(s).
#           If more than one, write an error message.
#
# Args    : None
#
# Returns : JWC shutter command PID (one JWC shutter command found)
#           NULL_PID (no JWC shutter command found)
#           ERROR_PID (multiple JWC shutter commands found)
#---------------------------------------------------------------------
{
  my $jwc_shutter_pid;

  my @jwc_shutter_pid_list = get_jwc_shutter_pid_list();
  if (@jwc_shutter_pid_list > 1)
  {
    debug_out("... Multiple JWC Shutters are running ...");
    print STDERR
      " CRS_ERROR:FATAL Error: Multiple JWC Shutters " .
      "running with PIDs";
    foreach (@jwc_shutter_pid_list)
    {
      print STDERR " ", $_;
    }
    print STDERR "\n";
    $jwc_shutter_pid = ERROR_PID;
  }
  elsif (@jwc_shutter_pid_list == 1)
  {
    $jwc_shutter_pid = $jwc_shutter_pid_list[0];
    debug_out("... JWC shutter command (pid=$jwc_shutter_pid) ...");
  }
  else
  {
    $jwc_shutter_pid = NULL_PID;
    debug_out("... JWC shutter command not found ...");
  }
  return $jwc_shutter_pid;
}

sub jwc_stop
#---------------------------------------------------------------------
# Function: Perform JWC stop action
#
# Args    : None
#
# Returns : ENTRY_SUCCESS  (action succeeded)
#           ENTRY_FAIL     (action failed)
#---------------------------------------------------------------------
{
  print STDOUT "Stop JWC\n";
  my $stop_rc = ENTRY_FAIL;
  my $iter_cnt = 0;
  my $rc = SUCC_CODE;

  my $jlib = $globhash{$jlib_key};
  my $jre = $globhash{$jre_key};
  my $catalina_home = $globhash{$catalina_home_key};
  my $catalina_base = $globhash{$catalina_base_key};
  my $pid_file = $globhash{$pid_file_key};
  my $path_separator = $Config{path_sep};

  # Collect all properties

  my $crs_stop_timeout = $ENV{"_CRS_STOP_TIMEOUT"};
  debug_out("... JWC stop timeout is $crs_stop_timeout seconds");
  my $sleep_interval = $envhash{$isleep_interval_key};

  # Change directories to $tomcat_log_dir before launching
  my $tomcat_log_dir = catfile($catalina_base, "logs");
  chdir($tomcat_log_dir) || 
    print STDERR "Could not change directory to $tomcat_log_dir\n";
    
  # Tracing options
  my $tomcat_stdout_err = catfile($tomcat_log_dir,
                                  $globhash{$shutter_log_name_key});
  my @dbwlm_tracing_properties;
  if (defined($envhash{$dbwlm_logging_level_key}))
  {
    @dbwlm_tracing_properties = ("-Doracle.wlm.dbwlmlogger.logging.level=".
                                 $envhash{$dbwlm_logging_level_key});
  }
                              
  @dbwlm_tracing_properties
    = (@dbwlm_tracing_properties,
       "-Doracle.jwc.client.logger.file.name=".
       $tomcat_stdout_err,
       "-Doracle.jwc.client.logger.file.number=".
       $globhash{$shutter_log_num_key},
       "-Doracle.jwc.client.logger.file.size=".
       $globhash{$shutter_log_size_key});
       
  # Shut down the container
  my @command_parms = (split(/\s+/, $ENV{"JRE_OPTIONS"}),
                       "-Dcatalina.base=" . $catalina_base,
                       "-Doracle.tls.enabled=" . getTLSEnabledProperty(),
                       @dbwlm_tracing_properties,
                       "-classpath",
                       catfile($jlib, JWC_LOGGING_JAR) . $path_separator .
                       catfile($jlib, JWC_CLIENT_JAR) . $path_separator .
                       catfile($jlib, JWC_SECURITY_JAR) . $path_separator .
                       catfile($jlib, "srvm.jar") . $path_separator .
                       catfile($jlib, "srvmhas.jar") . $path_separator .
                       catfile($jlib, JWCCRED_JAR) . $path_separator .
                       catfile($catalina_home, "lib", TOMCAT_LOGGING_JAR) ,
                       SHUTTER_CLASS,
                       LOCALHOST,
                       $ENV{"RMI_PORT"});

  my $initial = SHUTTER_INITIAL;
  async_start($initial, NULLFILE, NULLFILE, NULLFILE,
	      $jre, @command_parms);

  my $jwc_shutter_pid = get_jwc_shutter_pid();
  if ($jwc_shutter_pid == ERROR_PID)
  {
    return $stop_rc;
  }
  elsif ($jwc_shutter_pid != NULL_PID)
  {
    debug_out("... Initial Check - JWC Shutter JVM waiting " .
              "(pid=$jwc_shutter_pid) ...");
  }
  else
  {
    # No JWC Shutter found - This is not expected, but it
    # could be caused by a race condition.  If there is
    # still no JWC Shutter after wait, assume success.
    debug_out("... Initial Check found no JWC Shutter ...");
    $jwc_shutter_pid = ERROR_PID;
  }

  my $jwc_pid;
  while (1)
  {
    debug_out("... Sleep for $sleep_interval seconds ...");
    sleep($sleep_interval);

    if ($jwc_shutter_pid != NULL_PID)
    {
      debug_out("... Iteration $iter_cnt Check for JWC Shutter " .
                "...");
      $jwc_shutter_pid = get_jwc_shutter_pid();
      if ($jwc_shutter_pid == ERROR_PID)
      {
        last;
      }
      elsif ($jwc_shutter_pid != NULL_PID)
      {
        debug_out("... Iteration $iter_cnt JWC Shutter ".
                  " waiting " .
                  "(pid=$jwc_shutter_pid) ...");
      }
    }

    # If there is no more JWC Shutter JVM, check for JWC Container
    if ($jwc_shutter_pid == NULL_PID)
    {
      # First, check if JWC process is running
      debug_out("... Iteration $iter_cnt Check for JWC Container ...");
      $jwc_pid = get_jwc_pid();
      if ($jwc_pid == ERROR_PID)
      {
        last;
      }
      elsif ($jwc_pid == NULL_PID)
      {
        # No JWC
        # Remove pidfile and say that stop succeeded
        debug_out("... JWC Container is stopped ...");
        unlink($pid_file);
        last;
      }
      # Normal case - one JWC
      # If no PIDFILE, JWC must already be on the way down
      my $read_pid = read_pid_from_jwc_pid_file($pid_file);
      if ($read_pid == NULL_PID)
      {
        # No JWC
        # Remove pidfile and say that stop succeeded
        debug_out("... JWC Container is stopped ...");
        unlink($pid_file);
        $jwc_pid = NULL_PID;
        last;
      }
      if ($read_pid != $jwc_pid)
      {
        print STDERR
          " CRS_ERROR:FATAL Error: PID file mismatch (PID " .
          "$jwc_pid running, PID $read_pid read from PID file)\n";
        $jwc_pid = ERROR_PID;
        last;
      }
      debug_out("... JWC Container shutdown pending " .
                "($iter_cnt) ...");
    }
    $iter_cnt++;
  }
  if (($jwc_shutter_pid == NULL_PID) &&
      ($jwc_pid == NULL_PID))
  {
    $stop_rc = ENTRY_SUCCESS;
  }
  elsif (($jwc_shutter_pid != NULL_PID) &&
         ($jwc_shutter_pid != ERROR_PID))
  {
    debug_out("... JWC Shutter (pid=$jwc_shutter_pid) did not complete ...");
  }
  elsif (($jwc_pid != NULL_PID) &&
         ($jwc_pid != ERROR_PID))
  {
    debug_out("... JWC Container (pid=$jwc_pid) was not stopped ...");
  }
  if ($stop_rc == ENTRY_FAIL)
  {
    print STDOUT "Stop JWC Failed";
    jwc_sleep_to_timeout($crs_stop_timeout);
  }
  return $stop_rc;
}

sub jwc_check
#---------------------------------------------------------------------
# Function: Perform JWC check action
#
# Args    : None
#
# Returns : STATE_ONLINE  (JWC is online)
#           STATE_OFFLINE (JWC is offline)
#           STATE_FAILED  (JWC check failed)
#---------------------------------------------------------------------
{
  print STDOUT "Check JWC\n";
  my $check_rc = STATE_OFFLINE;
  my $pid_file = $globhash{$pid_file_key};

  debug_out("... issuing CHECK action ...");

  # First, check if JWC process is running
  my $jwc_pid = NULL_PID;
  my $read_pid = read_pid_from_jwc_pid_file($pid_file);

  # If PIDFILE contains a non-null PID and it's under our control,
  # accept this as our container
  # If not, investigate matters further
  if (($read_pid != NULL_PID) &&
      (send_signal("check", $read_pid) == SUCC_CODE))
  {
    $jwc_pid = $read_pid;
  }
  else
  {
    my @jwc_pid_list = get_jwc_pid_list();
    if (@jwc_pid_list == 0)
    {
      # No JWC
      # Remove pidfile and say that JWC is OFFLINE
      unlink($pid_file);
      return $check_rc;
    }

    # Next, check for multiple JWCs
    if (@jwc_pid_list > 1)
    {
      # Multiple JWCs
      debug_out("... Multiple JWC Containers are running ...");
      print STDERR
        " CRS_ERROR:FATAL Error: Multiple JWC Containers running ".
        "with PIDs";
      foreach (@jwc_pid_list)
      {
        print STDERR " ", $_;
      }
      print STDERR "\n";

      # Remove pidfile and say that JWC is FAILED
      unlink($pid_file);
      $check_rc = STATE_FAILED;
      return $check_rc;
    }

    # Normal case - one JWC
    $jwc_pid = $jwc_pid_list[0];
  }
  if ($read_pid == NULL_PID)
  {
    my $rc = write_pid_to_jwc_pidfile_and_check($pid_file, $jwc_pid);
    if ($rc != SUCC_CODE)
    {
      # Remove pidfile and say that JWC is FAILED
      unlink($pid_file);
      $check_rc = STATE_FAILED;
      return $check_rc;
    }
    $read_pid = $jwc_pid;
  }
  elsif ($read_pid != $jwc_pid)
  {
    print STDERR
      " CRS_ERROR:FATAL Error: PID file mismatch (PID $jwc_pid " .
      "running, PID $read_pid read from PID file)\n";
    
    # Remove pidfile and say that JWC is FAILED
    unlink($pid_file);
    $check_rc = STATE_FAILED;
    return $check_rc;
  }
  debug_out("... Phase 1: JWC pid check succeeded (pid=$jwc_pid) ...");

  # Now issue Phase 2 of the check

  my $rc = jwc_check_servlet(CHECK_ONCE);
  if ($rc == SUCC_CODE)
  {
    $check_rc = STATE_ONLINE;
    debug_out("... Phase 2: JWC http check succeeded ...");
  }
  else
  {
    $check_rc = STATE_FAILED;
    debug_out("... Phase 2: JWC http check failed ...");
  }

  return $check_rc;
}

sub jwc_clean
#---------------------------------------------------------------------
# Function: Perform JWC clean action
#
# Args    : Number of seconds to wait after issuing SIGQUIT
#
# Returns : ENTRY_SUCCESS  (action succeeded)
#---------------------------------------------------------------------
{
  print STDOUT "Clean JWC\n";
  my $clean_rc = ENTRY_SUCCESS;
  my ($sigquit_sleep_interval) = @_;
  my $iter_cnt = 0;
  
  my $catalina_home = $globhash{$catalina_home_key};
  my $catalina_base = $globhash{$catalina_base_key};
  my $pid_file = $globhash{$pid_file_key};

  my $crs_clean_timeout = $ENV{"_CRS_STOP_TIMEOUT"};
  debug_out("... JWC clean timeout is $crs_clean_timeout seconds");
  my $sleep_interval = $envhash{$isleep_interval_key};

  # Change directories to $tomcat_log_dir before launching
  my $tomcat_log_dir = catfile($catalina_base, "logs");
  chdir($tomcat_log_dir) || 
    print STDERR "Could not change directory to $tomcat_log_dir\n";
  
  my %jwc_checker_pid_hash;
  my %jwc_shutter_pid_hash;
  my %jwc_pid_hash;
  my $more_work = 1;

  while (1)
  {
    if (($iter_cnt > 0) && ($sleep_interval > 0))
    {
      sleep($sleep_interval);
    }
    $more_work = 0;

    $iter_cnt++;

    foreach my $jwc_checker_pid (get_jwc_checker_pid_list())
    {
      $more_work = 1;
      if (!defined($jwc_checker_pid_hash{$jwc_checker_pid}))
      {
        # Kill the JWC Checker
        send_signal("kill", $jwc_checker_pid);
        debug_out("... Cleaned JWC Checker JVM shutdown " .
                  "process (pid=$jwc_checker_pid) ...");
        $jwc_checker_pid_hash{$jwc_checker_pid} = 1;
      }
    }

    foreach my $jwc_shutter_pid (get_jwc_shutter_pid_list())
    {
      $more_work = 1;
      if (!defined($jwc_shutter_pid_hash{$jwc_shutter_pid}))
      {
        # Kill the JWC Shutter
        send_signal("kill", $jwc_shutter_pid);
        debug_out("... Cleaned JWC Shutter JVM shutdown " .
                  "process (pid=$jwc_shutter_pid) ...");
        $jwc_shutter_pid_hash{$jwc_shutter_pid} = 1;
      }
    }

    foreach my $jwc_pid (get_jwc_pid_list())
    {
      $more_work = 1;
      if (!defined($jwc_pid_hash{$jwc_pid}))
      {
        # Send SIGQUIT to JWC (to get a dump)
        if(send_signal("quit", $jwc_pid) == FAIL_CODE)
        {
          # If no SIGQUIT support, perform jstack command instead
          perform_jstack($jwc_pid);
        }

        # Sleep on request after SIGQUIT or jstack command
        if ($sigquit_sleep_interval >= 0)
        {
          sleep($sigquit_sleep_interval);
        }

        # Send SIGKILL to JWC
        send_signal("kill", $jwc_pid);
        debug_out("... Cleaned JWC Container (pid=$jwc_pid) ...");
        $jwc_pid_hash{$jwc_pid} = 1;
      }
    }
    if (!$more_work)
    {
        last;
    }
  }
  unlink($pid_file);

  return $clean_rc;
}

sub jwc_sleep_to_timeout
#---------------------------------------------------------------------
# Function: Sleep to timeout
#
# Args    : Number of seconds to sleep
#
# Returns : Does not return
#---------------------------------------------------------------------
{
  my ($sleep_interval) = @_;
  while (1)
  {
    debug_out("... Sleep to timeout ...");
    sleep($sleep_interval);
  }
}

sub getTLSEnabledProperty
#---------------------------------------------------------------------
# Function: Returns TLS_ENABLED property value.
#
# Args    : None
#
# Returns : "true" if TLS_ENABLED environment variable is defined and
#           set to "true"; "false" otherwise.
#---------------------------------------------------------------------
{ 
  my $tls_property = "false";
  
  if ($ENV{"TLS_ENABLED"} && $ENV{"TLS_ENABLED"} eq "true") {
    $tls_property = "true";
  }
  
  return $tls_property;
}

sub genNodeKeyData 
#---------------------------------------------------------------------
# Function: Calls crskeytoolctl utility to regenerate the node key
#           data
#
# Args    : None
#
# Returns : None
#---------------------------------------------------------------------
{
  # crskeytoolctl rekey command
  my @crskeytoolctl = (get_crskeytoolctl(), "-rekey");

  # get security lock file path
  my $catalina_base = $globhash{$catalina_base_key};
  my $securitydir = catdir(File::Spec->splitdir($catalina_base), updir(),
                           "security");
  my $seclockfile = catfile($securitydir, "security.lck");
  
  #FIXME In Windows, s_jwcctl_lib.pm OSD file does not implement any of
  #      lock functions due to lack of support for Fcntl library. All should
  #      be implemented together

  # get file handle of the security lock file
  my $fh = create_lock_file($seclockfile);

  # acquire the lock on security lock file
  file_lock($fh);

  # call 'crskeytoolctl -rekey' to generate node key data
  system(@crskeytoolctl);

  # release the lock on security lock file
  file_unlock($fh);
}

1;