#!/bin/sh

# failover
#
# BIG BROTHER - FAILOVER SCRIPT
# Sean MacGuire - BB4 Technologies Inc.
# Version 1.5
# Aug 16th 2000
#
# This program is Copyright (c) 2000
# BB4 Technologies Inc
# All Rights Reserved

#
# failover WATCHES BBNET and BBPAGER 
#
# IF THE BBNET OR BBPAGER MACHINE BECOMES UNAVAILABLE, THEN 
# WE TAKE OVER UNTIL THEY RETURN
#
# Big Brother has 2 weak spots.  First, although multiple BBPAGERS can be
# declared, few people want to be notified by every BBPAGER every time 
# something goes wrong.  Next, multiple BBNET testing is awkward and can
# lead to inconsistent results (red/green/red/green) when one BBNET can
# connect and the other cannot.  
#
# This failover script will allow you to run multiple BBPAGERS but only
# use the alternate if the primary is down.  Same with BBNET.
#
# To use, just add failover to the BBEXT variable in runbb.sh.  
#
# BBNET failover is automatic when we can't reach the BBNET machine
# BBNET should only be defined ONCE in the bb-hosts file, and should
# not be defined on the failover machine (that's this one if you're
# reading this!).
#
# BBPAGER failover requires that this machine be defined as a BBPAGER
# in etc/bb-hosts.  Unless you want to be notified by the failover 
# machine for everything, set bbwarn: FAILOVER in etc/bbwarnsetup.cfg
#
# Note that bb-page.sh has a minor change to force BBPAGER on in failover
# mode.
#
BBPROG=failover; export BBPROG
#
TEST="fo"
COLOR="clear"				# ASSUME ALL IS WELL...

#
# BBHOME CAN BE SET MANUALLY WHEN TESTING.
# OTHERWISE IT SHOULD BE SET FROM THE BB ENVIRONMENT
#
# BBHOME=/home/sean/bb; export BBHOME	# FOR TESTING

if test "$BBHOME" = ""
then
	echo "BBHOME is not set... exiting"
	exit 1
fi

if test ! "$BBTMP"                      # GET DEFINITIONS IF NEEDED
then
	 # echo "*** LOADING BBDEF ***"
        . $BBHOME/etc/bbdef.sh          # INCLUDE STANDARD DEFINITIONS
fi

#
# BBPAGER FAILOVER 
#
# IF WE'RE A BBPAGER FAILOVER, THEN YOU'LL NEED 2
# BBPAGERS DEFINED IN etc/bb-hosts (PREFERABLY WITH 
# THE SAME RULES).  THIS MACHINE *MUST* BE DEFINED AS
# A BBPAGER FOR FAILOVER TO WORK.
#
# ALSO NEEDS THE SMALL CHANGE TO bb-page.sh TO OVERRIDE
# BBWARN WHILE WE'RE IN FAILOVER MODE.  (THE CHANGE 
# FORCES THE PAGER ONLINE IN FAILOVER MODE...
#
if test "$BBPAGERS"
then
	# echo "BBPAGERS: $BBPAGERS 	BBPAGE: $BBPAGE"
	# WE'LL SEND A PAGE TO ALL THE BBPAGERS
	# IF ANY FAIL WE'LL GO INTO FAILOVER MODE... THIS DOESN'T MATTER
	# BECAUSE IF THE FAILOVER FAILS IT MAKES NO DIFFERENCE!
	#
	$BB $BBPAGE "dummy ${MACHINE}.${TEST} failover test" > $BBTMP/FO.$$ 2>&1
	if test -s "$BBTMP/FO.$$"		# SOMETHING FAILED...
	then
		$BB $BBPAGE "dummy ${MACHINE}.${TEST} failover test" > $BBTMP/FO.$$ 2>&1
		if test -s $BBTMP/FO.$$		# AND FAILED AGAIN...
		then
			if test ! -f $BBTMP/FAILOVER-BBPAGER
			then
				date > $BBTMP/FAILOVER-BBPAGER
			fi
		fi
	fi
	
	if test ! -s "$BBTMP/FO.$$"			# PAGING IS ALIVE
	then
		if test -f "$BBTMP/FAILOVER-BBPAGER"	# FAILBACK
		then
			$RM -f $BBTMP/FAILOVER-BBPAGER
		fi
		BBPLINE="&clear BBPAGER on standby"
	else
		COLOR="yellow"
		BBPLINE="&yellow BBPAGER failover since `cat $BBTMP/FAILOVER-BBPAGER`"
	fi

	$RM -f $BBTMP/FO.$$
fi

#
# IS THE BBNET MACHINE ALIVE?
# SEE IF WE CAN PING IT... 
#
BBNET=`$GREP BBNET $BBHOSTS | $GREP "^[0-9]" | $GREP -v "^\#"`
if test "$BBNET"
then
	set $BBNET
	BBNET="$2"
	# echo "BBNET is $BBNET"
	# BBNET="bobo.bb4.com" 	# TO TEST BROKEN BBNETS
fi

LINE=`$BBHOME/bin/bb-ping.sh $BBNET FALSE`

if test "$?" != "0"
then
	COLOR="yellow"
	#
	# NOW WE HAVE TO FAILOVER... ONCE.
	# PHILOSOPHICAL QUESTION AS TO WHAT COLOR THE DOT SHOULD BE
	# WHEN WE'RE IN FAILOVER MODE...  MAYBE YELLOW (warning...)
	# OTHERWISE FAILOVER IS ON STANDBY (clear)
	#
	# BACKUP THE BBPID FILE
	if test ! -f $BBTMP/FAILOVER-BBNET
	then
		BBSLEEP=300; export BBSLEEP
		cp $BBTMP/BBPID $BBTMP/FAILOVER-BBNET
       		# echo "*** CALLING BB-NETWORK ***"
        	{ nohup $BBHOME/bin/bbrun $BBHOME/bin/bb-network.sh ;} >> $BBHOME/BBOUT 2>&1 &
	fi
	BBNETLINE="&yellow BBNET failover running"
else
	#
	# IF WE'RE BETTER, THEN WE CAN FAILBACK (IS THAT A WORD?)
	# 
	if test -f $BBTMP/FAILOVER-BBNET
	then
		# KILL THE LAST PID 
		kill -9 `tail -1 $BBTMP/BBPID`		# TOAST bb-network.sh
		cp $BBTMP/FAILOVER-BBNET $BBTMP/BBPID
		rm -f $BBTMP/FAILOVER-BBNET
	fi
	BBNETLINE="&clear BBNET on standby"
fi

if test "$COLOR" = "yellow"
then
	LINE="Failover is running"
else
	LINE="Failover on standby"
fi

LINE="$LINE
$BBPLINE
$BBNETLINE"

$BB $BBDISP "status ${MACHINE}.${TEST} $COLOR `date` $LINE"