KrazyWorks » Bounce Veritas HA Cluster resource

Bounce Veritas HA Cluster resource

Submitted by admin on December 7, 2005 – 11:01 amNo Comment

The following is a Korn shell script that can be used from command line or cron to bounce a Veritas HA Cluster resource. This can be useful to resolve certain problems with Oracle databases running under VHA Cluster.

The script has to be executed with root credentials. Sudo can be configured to allow authenticated user to run the db_bounce.ksh without the password.

The script includes configurable parameters for notification:

1.RECEPIENTS – a comma-separated list of e-mail addresses to be used for notification in case of a failure

2.SUBJECT – a subject line to use for e-mail notification

3.ENABLED – set to 0 to disable notification; set to 1 to enable notification

A notification is sent out only if the script fails to bring a resource online, or if the script fails to bounce a resource. A notification e-mail will include relevant error messages, the name of the server, and the name of the resource.

All information and error messages are logged to /var/log/db_bounce.log

To run the script from command line use the following syntax (database dba1t and server delphi1a are used as examples):

1.to view the list of available resources:

sudo /var/adm/bin/db_bounce.ksh -l

2.to view the properties of a particular resource:

sudo /var/adm/bin/db_bounce.ksh -v dba1t

3.to take a resource down:

sudo /var/adm/bin/db_bounce.ksh -d dba1t

4.to bring a resource up:

sudo /var/adm/bin/db_bounce.ksh -u dba1t delphi1a

5.to bounce a resource:

sudo /var/adm/bin/db_bounce.ksh -b dba1t

6.to take a resource down and bring it up on a specific server:

sudo /var/adm/bin/db_bounce.ksh -b dba1t delphi1a

Here’s an example of how to run this script from oracle cron at 3:30am Monday through Friday to bounce the dba1t database:

30 3 * * 1-5 sudo /var/adm/bin/db_bounce.ksh -b dba1t &gt; /dev/null 2&gt;&1

Script:

#!/bin/ksh
 
# igor@krazyworks.com
# December 6. 2005
 
# This script allows to restart database resources for
# Veritas Cluster Server service groups. Install this script in
# /var/adm/bin on the Veritas HA cluster nodes and run as root.
 
# SYNTAX:
# /var/adm/bin/db_bounce.ksh [-udbvl] [resource] [server]
 
# SUPPORTED SYSTEMS:
#---------------------------------------------------------------
# OS Name	OS Version	Veritas CS Version	Tested
#---------------------------------------------------------------
# SunOS		5.8		3.5			Y
#---------------------------------------------------------------
 
PARM=$(echo "$#")
ACTION=$(echo "$1")
RESOURCE=$(echo "$2")
SERVER=$(echo "$3")
 
#---------------------------------------------------------------
# FUNCTIONS
#---------------------------------------------------------------
 
help() {
	clear
	cat << EOF
 
	db_bounce.ksh
 
	This script allows to restart database resources for
	Veritas Cluster Server service groups. Install this script in
	/var/adm/bin on the Veritas HA cluster nodes and run as root.
 
	The script requires 1 minute to bounce a VHA cluster resource.
	---------------------------------------------------------------
	USAGE:
		db_bounce.ksh [-dubvl] [resource] [server]
	---------------------------------------------------------------
	OPTIONS:
		-d	Take down a VHA resource
		-u	Bring up a VHA resource
		-b	Bounce a VHA resource
		-v	View status of a VHA resource
		-l	List of VHA resources
	---------------------------------------------------------------
	EXAMPLES:
		1.to view the list of available resources:
			sudo /var/adm/bin/db_bounce.ksh -l
 
		2.to view the properties of a particular resource:
			sudo /var/adm/bin/db_bounce.ksh -v dba1t
 
		3.to take a resource down:
			sudo /var/adm/bin/db_bounce.ksh -d dba1t
 
		4.to bring a resource up:
			sudo /var/adm/bin/db_bounce.ksh -u dba1t delphi1a
 
		5.to bounce a resource:
			sudo /var/adm/bin/db_bounce.ksh -b dba1t
 
		6.to take a resource down and bring it up on a specific server:
			sudo /var/adm/bin/db_bounce.ksh -b dba1t delphi1a
	---------------------------------------------------------------
 
EOF
	exit 1
}
 
#---------------------------------------------------------------
 
log() {
	LOG="/var/log/db_bounce.log"
	TMP="/tmp/db_bounce.tmp"
	LINELIM=1000 ; LINEBUF=50 ; (( LINEMAX = LINELIM + LINEBUF ))
 
	if [ ! -r "$LOG" ]
	then
		touch "$LOG"
	fi
 
	if [ -f "$LOG" ] && [ `wc -l "$LOG" | awk '{print $1}'` -gt $LINEMAX ]
	then
		cat "$LOG" | tail -$LINELIM &gt; "$TMP"
		mv "$TMP" "$LOG"
	fi
}
 
#---------------------------------------------------------------
 
message() {
	if [ ! $TAIL ]
	then
		TAIL=1
	else
		(( TAIL = TAIL + 1 ))
	fi
	echo "`hostname`	`date +'%Y-%m-%d %T'`	$MSG" &gt;&gt; "$LOG"
	tail -1 "$LOG"
}
 
#---------------------------------------------------------------
 
warning() {
	cat << EOF
 
	This operation will require 40-60 seconds. Please wait...
 
EOF
}
 
#---------------------------------------------------------------
 
notify() {
		RECEPIENTS="admin@server.com"
		SUBJECT="Database bounce error on `hostname`"
		ENABLED=1	# Set to 0 to disable notification
 
		if [ $ENABLED -eq 1 ]
		then
			cat "$LOG" | tail -$TAIL | mailx -s "$SUBJECT" "$RECEPIENTS"
		fi
}
 
#---------------------------------------------------------------
 
configure() {
	HARES="/opt/VRTS/bin/hares"
	if [ ! -x "$HARES" ]
	then
		HARES=$(which hares)
		if [ ! -x "$HARES" ]
		then
			MSG=$(echo "Executable $HARES not found.") ; message
		fi
	fi
 
	HACONF="/opt/VRTSvcs/bin/haconf"
	if [ ! -x "$HACONF" ]
	then
		HACONF=$(which haconf)
		if [ ! -x "$HACONF" ]
		then
			MSG=$(echo "Executable $HACONF not found.") ; message
		fi
	fi
 
	HALOG="/opt/VRTS/bin/halog"
	if [ ! -x "$HALOG" ]
	then
		HALOG=$(which halog)
		if [ ! -x "$HALOG" ]
		then
			MSG=$(echo "Executable $HALOG not found.") ; message
		fi
	fi
}
 
#---------------------------------------------------------------
 
tockens() {
	if [ "$ACTION" ] || [ "$RESOURCE" ]
	then
		case "$ACTION" in
			-d|-u|-b|-v|-l)
				;;
			*)
				help
				;;
		esac
	else
		help
	fi
 
	if [ $PARM ]
	then
		if [ $PARM -gt 3 ]
		then
			help
		fi
	fi
}
 
#---------------------------------------------------------------
 
input_check() {
	if [ "$ACTION" != "-l" ]
	then
		if [ `$HARES -display "$RESOURCE" &gt; /dev/null 2&gt;&1 ; echo $?` -eq 0 ]
		then
			if [ "$SERVER" ]
			then
				if [ `$HARES -display "$RESOURCE" -attribute State | fgrep -v "#" | awk '{print $3}' | fgrep -cx "$SERVER"` -ne 1 ]
				then
					MSG=$(echo "Server $SERVER is not configured to run $RESOURCE.") ; message
					exit 1
				fi
			fi
		else
			MSG=$(echo "Resource $RESOURCE does not exist.") ; message
			exit 1
		fi
	fi
}
 
#---------------------------------------------------------------
 
view_db_list() {
	clear
	$HARES -list | more
}
 
#---------------------------------------------------------------
 
view_db_conf() {
	clear
	$HARES -display "$RESOURCE" | more
}
 
db_down() {
	if [ `$HARES -display "$RESOURCE" -attribute State | grep -v "#" | grep -c ONLINE` -eq 1 ]
	then
		warning
		if [ ! "$SERVER" ]
		then
			SERVER=$($HARES -display "$RESOURCE" -attribute State | grep -v "#" | grep ONLINE | tail -1 | awk '{print $3}')
		fi
 
		$HACONF -makerw
		$HARES -offline "$RESOURCE" -sys "$SERVER"
		sleep 20
		$HARES -modify "$RESOURCE" Enabled 0
 
		if [ $? -eq 0 ]
		then
			MSG=$(echo "Changed status of resource $RESOURCE to OFFLINE") ; message
			$HACONF -dump -makero
			exit 0
		else
			MSG=$(echo "Failed to change status of resource $RESOURCE to OFFLINE") ; message
			$HACONF -dump -makero
			exit 1
		fi
	else
		MSG=$(echo "Resource $RESOURCE is already offline.") ; message
		exit 1
	fi
}
 
#---------------------------------------------------------------
 
db_up() {
	if [ `$HARES -display "$RESOURCE" -attribute State | grep -v "#" | grep -c ONLINE` -eq 0 ]
	then
		warning
		if [ ! "$SERVER" ]
		then
			MSG=$(echo "No server specified to bring up resource $RESOURCE") ; message
			exit 1
		fi
 
		$HACONF -makerw
		$HARES -modify "$RESOURCE" Enabled 1
		sleep 20
		$HARES -online "$RESOURCE" -sys "$SERVER"
 
		if [ $? -eq 0 ]
		then
			MSG=$(echo "Changed status of resource $RESOURCE to ONLINE") ; message
			$HACONF -dump -makero
			exit 0
		else
			MSG=$(echo "Failed to change status of resource $RESOURCE to ONLINE") ; message
			$HACONF -dump -makero
			notify
			exit 1
		fi
	else
		MSG=$(echo "Resource $RESOURCE is already online.") ; message
		exit 1
	fi
}
 
#---------------------------------------------------------------
 
db_bounce() {
	if [ `$HARES -display "$RESOURCE" -attribute State | grep -v "#" | grep -c ONLINE` -eq 1 ]
	then
		warning
		if [ ! "$SERVER" ]
		then
			SERVER=$($HARES -display "$RESOURCE" -attribute State | grep -v "#" | grep ONLINE | tail -1 | awk '{print $3}')
		fi
 
		$HACONF -makerw
		$HARES -offline "$RESOURCE" -sys "$SERVER"
		sleep 20
		$HARES -modify "$RESOURCE" Enabled 0
 
		if [ $? -eq 0 ]
		then
			MSG=$(echo "Changed status of resource $RESOURCE to OFFLINE") ; message
			sleep 20
			$HARES -modify "$RESOURCE" Enabled 1
			sleep 20
			$HARES -online "$RESOURCE" -sys "$SERVER"
 
			if [ $? -eq 0 ]
			then
				MSG=$(echo "Changed status of resource $RESOURCE to ONLINE") ; message
				$HACONF -dump -makero
				exit 0
			else
				MSG=$(echo "Failed to change status of resource $RESOURCE to ONLINE") ; message
				$HACONF -dump -makero
				notify
				exit 1
			fi
		else
			MSG=$(echo "Failed to change status of resource $RESOURCE to OFFLINE") ; message
			$HACONF -dump -makero
			notify
			exit 1
		fi
	else
		MSG=$(echo "Resource $RESOURCE is already offline.") ; message
		exit 1
	fi
}
 
#---------------------------------------------------------------
# RUNTIME
#---------------------------------------------------------------
 
log
configure
tockens
input_check
 
case "$ACTION" in
	-d) db_down ;;
	-u) db_up ;;
	-b) db_bounce ;;
	-v) view_db_conf ;;
	-l) view_db_list ;;
	*) help	;;
esac