#!/bin/sh # # VIP implementation on Linux using ifconfig # It only supports IPv4 # # # Syntax: # # racgvip start vip_name # racgvip stop vip_name # racgvip check vip_name # racgvip create vip_name IP=vip_address NODE=preferred_node # [MASK=netmask IF="interfaces"] # racgvip delete vip_name # # vip_name must be unique across the cluster. In RAC HA, # the preferred node name is used for vip_name. # # # Note: It assumes interface name does not contain space character. # VIP must be in the form of n.n.n.n # # # Date: 2007 March 16 # # Changes: # - Skip RX packet checking when FAIL_WHEN_DEFAULTGW_NOT_FOUND is 0 for # VM environment (Bug 9401335) # - Reduce the wait timeout for Ping from 3 to 1 second(Bug 6007567) # - Remove Updation of Kernel Route table with default # gateway(Bug 5699792) # - Add more logging for ping functionality # - Check action does not set VIP and just returns error when # VIP is not configured. It fixes the race condition when check and stop # actions are running at the same time. # - Add ping_vip() # - Print message when ping to default gateway failed. # - If IP is up on other node in start action, wait for IP to be down for # timeout/3 seconds. # - Replace 'ifconfig del' with 'ifconfig down' for better compatibility # in various Linux platforms. # - Support interface name with more than 6 characters # - Add locking logic to protect the network interface # - Call arping in background to shorten the script execution time # - Handle when :0 logical interface is set # - Set LANG and LC_ALL to C # - Replace environment variable name FAIL_WHEN_DEFAULTGW_NO_FOUND with # FAIL_WHEN_DEFAULTGW_NOT_FOUND # - Fix pattern matching bug in getifbyip # - Set default value of FAIL_WHEN_ALL_LINK_DOWN as 1 # - Fix broadcast address # - In checkIf(), use ifconfig to check if the interface is UP. # - if mii-tool reports the link is down, use ifconfig to verify the link # status. # - some changes to optimize the execution time. # - temporary files are no longer used to store data. # - use mii-tool to check if the interface is up. If mii-tool does not exist # or fails, "ping -r -I " will be used instead. # - Fixing the known issue in the previous racgvip dated 2004 July 6. # The issue was after pulling cable from an interface, VIP will failover # to another interface but the external network still has trouble to # access VIP. It is fixed by updating the route table and using arping # to flush ARP cache in the machines in the same subnet. # When the script found the interface is down, it disables and enables the # interface to clear the entries of that interface in the route table. # When the script sets the VIP to an interface, it adds a route to default # gateway for that interface. It makes sure the node will use the interface # which VIP is set for going network traffic. # - Add -q to ARPING to suppress output. # - Add code to remove down logical interface which configured with VIP. # - Check RX packets number if checkIf() if mii-tool and ping failed. # - Variable FAIL_WHEN_ALL_LINK_DOWN to configure if the script returns failure # or not when all links are down - usually caused by network cable is pulled. # Note: it is different from marking the interfaces to down using ifconfig. # When all interfaces are marked down by ifconfig, the script will return # failure. # - Variable FAIL_WHEN_DEFAULTGW_NO_FOUND to configure if checkIf() returns # failure when default gateway is not found. If mii-tool works, # default gateway is not needed in checkIf(). # # Testing: # This script can be tested with the following steps # # 1. becomes root user # 2. set environment variables # - _USR_ORA_VIP for VIP address # - _USR_ORA_NETMASK for netmask address # - _USR_ORA_IF for interface names, they are separated by '|' character # - _CAA_NAME for the VIP resource name, ora..vip # 3. Test list command # # sh racgvip list # 4. Test start command # # sh racgvip start # # echo $? # # ifconfig (to check if the VIP is set) # 5. Test check command # # sh racgvip check # # echo $? # 6. Test stop command # # sh racgvip stop # # echo $? # # ifconfig (to check if the VIP is unset) # 7. If there is more than one interfaces, remove the cable on the interface # which VIP is set and run check action, the VIP should be set to another # interface. # Note: if cables are pulled from all interfaces or there is only one # interface, VIP will stay on the original interface and # the script returns success. This behavior is to keep VIP resource # from failover if there is a network brown out. # # # sh racgvip check # # echo $? # # ifconfig (to check if the VIP is set to another interface) IFCONFIG=/sbin/ifconfig GREP=/bin/grep SED=/bin/sed RM=/bin/rm MV=/bin/mv UNIQ=/usr/bin/uniq PING=/bin/ping WC=/usr/bin/wc NETSTAT=/bin/netstat AWK=/bin/awk WHOAMI=/usr/bin/whoami CAT=/bin/cat UNAME=/bin/uname SLEEP=/bin/sleep SORT=/bin/sort EXPR=/usr/bin/expr DATE=/bin/date RENICE=/usr/bin/renice ETHTOOL=/sbin/ethtool MIITOOL=/sbin/mii-tool ARPING=/sbin/arping IPCMD="/sbin/ip -f inet" # Set LANG and LC_ALL to C LANG=C LC_ALL=C export LANG LC_ALL # set it to 1 for VIP failover when all links are down, otherewise set it to 0 FAIL_WHEN_ALL_LINK_DOWN=1 # set it to 0 for checkIf() to return success if default gateway is not found, # otherwise set it to 1 FAIL_WHEN_DEFAULTGW_NOT_FOUND=1 # hard code default gateway here if needed DEFAULTGW= # renice the process $RENICE -20 -p $$ 2>/dev/null 1>&2 HOSTNAME=`/bin/hostname` # timeout of ping in number of loops # use count = 1 as ping will return right the way when the target IP is up PING_TIMEOUT="-w 1 -c 1" # time to do ping in ping_vip() PING_COUNT=10 LOCKED=0 CRS_STAT=$ORA_CRS_HOME/bin/crs_stat # number of time to check the interface to determine if the interface is down CHECK_TIMES=2 SUCCESS=0 ERROR=1 DEFAULT_TIMEOUT=60 IP=$_USR_ORA_VIP MASK=$_USR_ORA_NETMASK IF=$_USR_ORA_IF OP=$1 USER=`$WHOAMI` # Uncomment out the following line to enable debug tracing # _USR_ORA_DEBUG=1 if [[ $(uname) != Linux ]]; then echo "This is Linux code" exit $ERROR fi logx() { if [ -n "${_USR_ORA_DEBUG}" ] then if [ ${_USR_ORA_DEBUG} -ge 1 ]; then message=$* echo "`$DATE` [ $$ ] $message" 1>&2 fi fi } ping_vip() { logx "ping_vip $1 started" if [ -n "$1" ] then _count=1 while [ $_count -le $PING_COUNT ] do logx "About to execute $PING $1 $PING_TIMEOUT" $PING $1 $PING_TIMEOUT 2> /dev/null 1>&2 if [ $? -ne 0 ] then logx "ping_vip: $1 is not pingable, _count = $_count" return 1 else _count=$(($_count + 1)) fi done logx "ping_vip $1 exit" return 0 fi echo "ping_vip: IP address is not specified" return 1 } # # Lock to prevent other instances from running # get_lock() { TOUCH=/bin/touch LS=/bin/ls KILL=/bin/kill LOCK="/var/tmp/vip_$1_$HOSTNAME.lock" $TOUCH $LOCK.$$ if [ $? -ne 0 ] then echo "get_lock: touch $LOCK.$$ failed" return 1 fi if [ `$LS $LOCK.[1-9]* | $WC -l` -eq 1 ] then logx "get_lock: lock file $LOCK.$$ is created" LOCKED=1 return 0 else # clean up lock files for _f in $LOCK.[1-9]* do PID=`echo $_f | $SED -e 's/^.*\.//'` if [ -n "$PID" -a "$PID" != "$$" ] then $KILL -0 $PID > /dev/null 2>&1 if [ $? -ne 0 ] then logx "get_lock: owner (pid=$PID) of lock file $_f is not running, remove lock file" $RM -f $_f > /dev/null 2>&1 fi fi done fi if [ `$LS $LOCK.[1-9]* | $WC -l` -eq 1 ] then logx "get_lock: lock file $LOCK is created" LOCKED=1 return 0 else echo "get_lock: Failed to get lock for $1 (host=$HOSTNAME)" $RM -f $LOCK.$$ > /dev/null 2>&1 return 1 fi } release_lock() { if [ "$LOCKED" = 1 ] ; then $RM -f $LOCK.$$ > /dev/null 2>&1 logx "release_lock: remove lock file $LOCK.$$" LOCKED=0 else echo "release_lock: not locked" fi } # # global variable to store result of listif() # listif_result= listif() { logx "listif: starting" if [ -z "$listif_result" ] then listif_result=`$IPCMD -o addr | $AWK '{ print $NF }' | $GREP -vw lo` fi logx "listif: completed with $listif_result" echo "$listif_result" } # # prints the default gateway # defaultgw() { logx "defaultgw: started" _defaultgw=`$NETSTAT -rn --inet | $AWK '{ if (/^0.0.0.0/) { print $2; exit }}'` echo "$_defaultgw" logx "defaultgw: completed with $_defaultgw" } # # get the interface of the given IP # getifbyip() { __LOCAL_IP=$1 gf_retif="" logx "getifbyip: started for $1" # # Use ip to get the interface name. ifconfig only shows the first # 8 characters of the interface name # gf_retif=`$IPCMD -o addr | $GREP "inet $__LOCAL_IP/" | $AWK '{ print $NF }'` logx "getifbyip: returning IP $gf_retif" # # if $2 is not specified, only UP interfaces are returned # it is really for compatibility of the original implementation of # getifbyip which uses ifconfig. This code is used to remove the # logical interface which is down but VIP is configured. The remove code # is in the main routine. # if [ -z "$2" ] then for _i in $gf_retif do $IFCONFIG $_i | $GREP -q -w UP if [ $? -eq 0 ] then echo $_i fi done elif [ -n "$gf_retif" ] then echo "$gf_retif" fi } # # get the free next logical interface number of an given interface # limit the logical interface number to 255 # please note that a lock of the return logical interface name will be held # in this routine. That lock has to be release explicitly by release_lock getnextli() { _LOCAL_IF=$1 nextli="" # # logical interface numbers which are currently using # _LIN="" logx "getnextli: started for if=$1" _LIN=`listif | $GREP "$_LOCAL_IF:" | $SED -e's/^.*://' | $SORT -n` i=1 while [ $i -le 256 ] do # check if the logical interface name is taken _found=0 for j in ${_LIN} do if [ $j -eq 0 ] then # skip :0 logical interface continue fi if [ $i -eq $j ] then _found=1 break fi done if [ $_found -eq 0 ] then # test if we can get a lock for that logical interface name get_lock ${_LOCAL_IF}_${i} if [ $? -eq 0 ] then # check again while holding the lock # clear listif_result to get the latest data listif_result= listif | $GREP -w "$_LOCAL_IF:$i" > /dev/null 2>&1 if [ $? -ne 0 ] then break else # the logical interface is used, release the lock and try again release_lock fi fi fi i=$(($i+1)) done if [ $i -eq 256 ] then echo "getnextli: failed to get the next logical interface name" exit $ERROR fi nextli=${_LOCAL_IF}:$i logx "getnextli: completed with nextli=$nextli" # # logical interface number is returned as return code. # it is because if calling `getnextli`, the locking functions will # not work as getnextli will be running in the different shell # return $i } checkIf() { _IF=$1 _RET=0 _LINK_STAT= logx "checkIf: start for if=$_IF" if [ -z "$_IF" ] then echo "checkIf: interface name is NULL" return 1 fi # check if ther interface is up $IFCONFIG $_IF | $GREP -q -w UP if [ $? -ne 0 ] then echo "checkIf: interface $_IF is down" return 1 fi # do ethtool check first then fall back to mii-tool if [ -x $ETHTOOL ] then _LINK_STAT=`$ETHTOOL $_IF 2> /dev/null` if [ $? -eq 0 ] then echo "$_LINK_STAT" | $GREP -q "Link detected: yes" if [ $? -eq 0 ] then logx "checkIf: ethtool checked if=$_IF ok" _RET=0 else logx "checkIf: ethtool checked if=$_IF failed" # clear _LINK_STAT so that mii-tool check will be used _LINK_STAT= fi else logx "$ETHTOOL $_IF error" _LINK_STAT= fi fi if [ -z "$_LINK_STAT" -a -x $MIITOOL ] then _LINK_STAT=`$MIITOOL $_IF 2> /dev/null` if [ $? -eq 0 ] then echo "$_LINK_STAT" | $GREP -q "link ok" if [ $? -eq 0 ] then logx "checkIf: mii-tool checked if=$_IF ok" _RET=0 else logx "checkIf: mii-tool checked if=$_IF failed" # clear _LINK_STAT so that ifconfig check will be used _LINK_STAT= fi else logx "$MIITOOL $_IF error" _LINK_STAT= fi fi if [ -z "$_LINK_STAT" ] then if [ -z "$DEFAULTGW" ] then DEFAULTGW=`defaultgw` fi if [ -n "$DEFAULTGW" -a $FAIL_WHEN_DEFAULTGW_NOT_FOUND -eq 1 ] then _RET=1 # get RX packets numbers _O1=`$IFCONFIG $_IF | $AWK '{ if (/RX packets:/) { sub("packets:", "", $2); print $2}}'` x=$CHECK_TIMES while [ $x -gt 0 ] do logx "About to execute $PING -r -I $_IF $DEFAULTGW $PING_TIMEOUT" $PING -r -I $_IF $DEFAULTGW $PING_TIMEOUT > /dev/null 2>&1 rc=$? if [ $rc -eq 0 ] then _RET=0 break else echo "ping to $DEFAULTGW via $_IF failed, rc = $rc (host=$HOSTNAME)" fi x=$(($x-1)) done if [ $_RET -ne 0 ] then # # last chance, check if RX packets numbers changed # _O2=`$IFCONFIG $_IF | $AWK '{ if (/RX packets:/) { sub("packets:", "", $2); print $2}}'` if [ "$_O1" = "$_O2" ] then logx "checkIf: ping and RX packets checked if=$_IF failed" else _RET=0 logx "checkIf: RX packets checked if=$_IF OK" fi else logx "checkIf: ping checked if=$_IF ok" fi else [[ -z "$DEFAULTGW" ]] && echo "checkIf: Default gateway is not defined (host=$HOSTNAME)" if [ $FAIL_WHEN_DEFAULTGW_NOT_FOUND -eq 1 ] then _RET=1 fi fi fi if [ $_RET -eq 1 ] then echo "Interface $_IF checked failed (host=$HOSTNAME)" fi logx "checkIf: end for if=$_IF" return $_RET } if [ "$USER" != "root" -a "$OP" != "list" ] then # # # it must be run as root echo "It must be run as root user" exit $ERROR fi if [ -n "$IP" -a -n "$MASK" ] then # Get broadcast address from IP and MASK BROADCAST=$( IFS=. set $IP $MASK echo "$(($1 | (~$5 + 256))).$(($2 | (~$6 + 256))).$(($3 | (~$7 + 256))).$(($4 | (~$8 + 256)))") logx "Broadcast = $BROADCAST" fi # Main code if [ "$OP" = "list" ] then # # return all interface except loopback and logical interface # for li_I in `listif` do _li=`echo ${li_I} | $GREP -q ':'` if [ $? -eq 1 ]; then echo ${li_I} fi done if [ -n ${li_I} ] then exit $SUCCESS else echo "No interface found" exit $ERROR fi else # # if vip name is not found, extract it from CAA resource name # VIP_NAME=`echo $_CAA_NAME | $SED -e's/^ora\.//;s/\.vip$//'` NAME=${2:-$VIP_NAME} if [ -z "$NAME" ] then echo "There is no VIP name" exit $ERROR fi fi # Get the interface which the VIP is configured IF_USING= if [ -n "$IP" ] then logx Checking interface existance # # check if it is configured # as logical interface in the local node # logx "Calling getifbyip" LI=`getifbyip $IP` logx Completed getifbyip $LI logx "Calling getifbyip -a" LI_A=`getifbyip $IP -a` logx Completed getifbyip $LI_A # remove logical interface with VIP and down if [ "$LI" != "$LI_A" ] then for I in $LI_A do found=0 for J in $LI do if [ "$I" = "$J" ] then found=1 break fi done if [ $found = 0 ] then echo $I | $GREP -q ':' if [ $? -eq 0 ] then $IFCONFIG $I down logx "Removed down interface $I" fi fi done fi echo "$LI" | $GREP -q ':' if [ $? -ne 0 ] then # # IP is not configured as logical interface in the local node # if [ "$OP" = "stop" ] then logx "About to execute $PING $IP $PING_TIMEOUT" $PING $IP $PING_TIMEOUT 2> /dev/null 1>&2 if [ $? -eq 0 -a -z "$LI" ] then # IP is not configured on this node but ping still works # it may be IP is set but ifconfig output does not show it, # bounce all UP interfaces to clean up this state for I in `echo $IF | $SED -e's/|/ /g'` do $IFCONFIG $I | $GREP -q -w UP if [ $? -eq 0 ] then echo "bounce $I (host=$HOSTNAME)" # Get the free logical interface getnextli $I I="$I:$?" $IFCONFIG $I $IP netmask $MASK broadcast $BROADCAST up $IFCONFIG $I down release_lock fi done fi # exit success when it is a stop action exit $SUCCESS else # start or check ping_vip $IP if [ $? -eq 0 ] then echo "IP:$IP is already up in the network (host=$HOSTNAME)" if [ "$OP" = "start" ] then # # loop timeout/3 times before give up # timeout=${_CAA_START_TIMEOUT:-0} if [ $timeout -eq 0 ] then timeout=${_CAA_SCRIPT_TIMEOUT:-0} fi if [ $timeout -eq 0 ] then # default timeout timeout=$DEFAULT_TIMEOUT fi logx "timeout = $timeout" timeout=$(($timeout/3)) while [ $timeout -gt 0 ] do $SLEEP 1 ping_vip $IP if [ $? -ne 0 ] then break fi echo "IP:$IP is already up in the network (host=$HOSTNAME)" timeout=$(($timeout - 1)) done if [ $timeout -eq 0 ] then exit $ERROR fi else # OP = check exit $ERROR fi fi fi else IF_USING=`echo "$LI" | $SED -e's/:.*$//'` if [ -z "$IF_USING" ] then echo "Unable to get interface name which VIP is running on (host=$HOSTNAME)" if [ "$OP" = "stop" ] then # exit success when it is a stop action exit $SUCCESS else exit $ERROR fi else N_IF_USING=`echo "$IF_USING" | $WC -l` if [ $N_IF_USING -gt 1 ] then echo "Multiple logical interfaces configured for address $IP (host=$HOSTNAME)" if [ "$OP" != "stop" ] then exit $ERROR fi fi fi fi fi logx 'Completed with initial interface test' case $OP in 'create') # # nothing is required in create # exit $SUCCESS ;; 'delete') exit $SUCCESS ;; 'start'|'check') if [ "$OP" = "check" ] then if [ $FAIL_WHEN_ALL_LINK_DOWN -eq 0 -a "$IF_USING" = "$IF" ] then # # If it is the only interface, it means there is no other interface # to perform interfaces failover. # # Comment this if block if VIP failover to another node is required in # this case - interface is down and it is the only interface # exit $SUCCESS fi # # Check if the interface is OK # if [ -n "$IF_USING" ] then checkIf "$IF_USING" if [ $? -eq 0 ] then exit $SUCCESS fi fi if [ -z "$LI_A" ] then # # If VIP is not set in any interface, returns error. # It fixes the race condition when check and stop actions # are running at the same time. The problem is the VIP # address may remain configured even the corresponding # VIP resource is OFFLINE. # # Other platform may use IF_USING. LI_A is used on Linux # as some network product will bring the interface when # it detects network failure. It will leave VIP configured # but on a down interface. We still want to set the VIP # to the available inteface. # # Comment the exit statement if VIP could be configured # by the VIP resource check action when the VIP address # is deconfigured and does not worry about the race condition # described above. # exit $ERROR fi fi if [ "$OP" = "check" ] then # If the target of the VIP resource is OFFLINE, # don't execute the start part logx 'Performing CRS_STAT testing' if [ -x $CRS_STAT -a -n "$_CAA_NAME" ] then $CRS_STAT -u $_CAA_NAME | $GREP -q "^TARGET=OFFLINE" if [ $? -eq 0 ] then exit $SUCCESS fi else echo "cannot execute $CRS_STAT or cannot find _CAA_NAME (host=$HOSTNAME)" fi logx 'Completed CRS_STAT testing' fi # Here is the start code # # Should test if the IP address is already defined # in the cluster # if [ -n "$IP" -a -n "$MASK" -a -n "$IF" ] then if [ -n "$IF_USING" ] then # # check the interface if it is a start action # if it is a check action, the interface should be already checked # if [ "$OP" = "start" ] then checkIf "$IF_USING" if [ $? -eq 0 ] then exit $SUCCESS fi fi # # the interface is checked failed, remove the logical interface # and set it again. # # Save default gateway first. # if [ -z "$DEFAULTGW" ] then DEFAULTGW=`defaultgw` fi $IFCONFIG $LI down logx 'Completed second gateway test' fi logx 'Interface tests' # # interface names are separated by | character # IF=`echo $IF | $SED -e's/|/ /g'` for I in $IF do if [ "$I" = "$IF_USING" ] then # bypass the failed interface continue fi # # Check if the interface is OK # checkIf "$I" if [ $? -eq 0 ] then # Get the free logical interface getnextli $I LI="$I:$?" $IFCONFIG $LI $IP netmask $MASK broadcast $BROADCAST up if [ $? -ne 0 ] then echo "$IFCONFIG $LI $IP netmask $MASK broadcast $BROADCAST up failed (host=$HOSTNAME)" # # fail, remove the logical interface # $IFCONFIG $LI down else # # success # logx 'Success exit 1' if [ -n "$IF_USING" ] then # clear route table by temporary disable and enable # the physical interface $IFCONFIG $IF_USING down $IFCONFIG $IF_USING up fi # update ARP cache $ARPING -q -U -c 3 -I $I $IP 2> /dev/null 1>&2 & release_lock exit $SUCCESS fi release_lock fi done if [ $FAIL_WHEN_ALL_LINK_DOWN -eq 0 -a -n "$IF_USING" ] then # # all interfaces are down, put back the IP address to the original # interface # getnextli $IF_USING LI="$IF_USING:$?" $IFCONFIG $LI $IP netmask $MASK broadcast $BROADCAST up if [ $? -ne 0 ] then echo "$IFCONFIG $LI $IP netmask $MASK broadcast $BROADCAST up failed (host=$HOSTNAME)" else # update ARP cache $ARPING -q -U -c 3 -I $IF_USING $IP 2> /dev/null 1>&2 & release_lock exit $SUCCESS fi release_lock fi fi echo "Invalid parameters, or failed to bring up VIP (host=$HOSTNAME)" exit $ERROR ;; 'stop') # # remove all logical interfaces configured for IP # for I in $LI do echo $I | $GREP -q ':' if [ $? -eq 0 ] then # lock the interface and check its IP address before removing it LOCKNAME=`echo $I | $SED 's/:/_/'` get_lock $LOCKNAME if [ $? -eq 0 ] then $IFCONFIG $I | $GREP -q -w $IP if [ $? -eq 0 ] then $IFCONFIG $I down fi release_lock fi fi done # # Always return success in stop action # exit $SUCCESS ;; *) # # invalid option # echo "Invalid option: $OP (host=$HOSTNAME)" exit $ERROR ;; esac