#!/bin/ksh # # memory: Sun Enterprise Server check - BB external script test # # version 1.0 # version 1.1 - now includes some debug output for color info in comment area of html # version 1.2 - changed to ksh and modified sar -g section # version 1.3 - added inode cache check to $NF of sar -g check - non zero is bad # # BIG BROTHER / XXXXXXXXXXXXXXXX status # # Written by Paul A. Luzzi # on July 8, 2002 # ######################################## # NOTE # This has been tested with BB 1.2b,1.5a,h,g # # Tested on : # Sun Sparc5/10/20 # Sun Ultra1/2/5/10/30/60/80 # Sun Ultra 250/450/220R/420R/480R # Sun Enterprise 3000/3500/3501/4500/4501 # Sun Netra T1 ######################################## ######################################## # INSTALLATION # step 1 - update the EXT section of the runbb.sh script to include this # step 2 - update the $BBHOME/etc/bbsys.local to include the following lines : # SAR="/usr/sbin/sar" # SWAP="/usr/sbin/swap" # PRTMEM="/usr/bin/prtmem" # MEMPS="/usr/bin/memps" # export SAR SWAP PRTMEM MEMPS # step 3 - the "memtool" package must be loaded otherwise the prtmem and memps checks will fail # step 4 - the updated top tool (not the one shipped with the o/s) must be loaded for the top section # step 5 - requires restart of Big Brother client side process to initialize this agent ######################################## ################################## # CONFIGURE IT HERE ################################## ################################## # Start of script ################################## if test ! "$BBHOME" then echo "template: BBHOME is not set" exit 1 fi if test ! -d "$BBHOME" then echo "template: BBHOME is invalid" exit 1 fi if test ! "$BBTMP" # GET DEFINITIONS IF NEEDED then echo "template: The BB environment is not set" exit 1 fi ##### ##### Start of custom section added by Paul A. Luzzi ##### ##### ##### Get Status proc - used to get all responses ##### get_status() { ##### ##### Setup any and all variables before beginning. ##### ##### ##### Section copied from script originally written by Mike Arki ##### Modified extensively on 03-22-2000 by Paul A. Luzzi ##### for use with Big Brother ##### ##### Purpose is to report back to a central server, all "standard" ##### detailed information about network, disk, volume, cpu, memory ##### which is not reported thru regular "easy" checks. ##### ##### Any problem areas should be highlighted in : ##### bold, italics, and red font. ##### ##### ##### Setup some variables for use later ##### COLOR="green" ##### ##### Check the "prtmem" output - info only ##### if [ ! -z "$PRTMEM" -a -s "$PRTMEM" ] then echo "

" echo "============== Memory Usage Info ==============" echo "--- ( $PRTMEM ) ---" echo "
" echo "
" CATCH_COMMAND=` $PRTMEM | $GREP -v "^$"` ## if [ "$?" = "0" ] ## then echo "$CATCH_COMMAND" ## else ## COLOR="red" ## echo "Problem with $PRTMEM !!!" ## echo "$CATCH_COMMAND" ## fi echo "
" fi echo "" ## ##### ## ##### Check the "memps | head -5" output - info only ## ##### ## if [ ! -z "$MEMPS" ] ## then ## echo "

" ## echo "============== $MEMPS - top few hogs ==============" ## echo "--- ( $MEMPS | $HEAD -5 ) ---" ## echo "
" ## echo "
" ## CATCH_COMMAND=` $MEMPS | $HEAD -5 ` ## if [ "$?" = "0" ] ## then ## echo "$CATCH_COMMAND" ## else ## COLOR="red" ## echo "Problem with $MEMPS !!!" ## fi ## echo "
" ## fi ##### ##### check the "vmstat 2 3" for any non-zero swap, demand, page-ins ##### echo "

" echo "============== Virtual Memory Info ==============" echo "--- (vmstat 2 3) ---" echo "
" echo "
" PP_CATCH_COMMAND=` $VMSTAT 2 3 > /tmp/$THIS_HOST.mem_pp.log ` CATCH_COMMAND=` $TAIL +4 /tmp/$THIS_HOST.mem_pp.log > /tmp/$THIS_HOST.memory.log ` RESULT_FLAG="$?" COMMAND_1=` $CAT /tmp/$THIS_HOST.memory.log | $AWK '{print $1}' | $TAIL -9 ` COMMAND_2=` $CAT /tmp/$THIS_HOST.memory.log | $AWK '{print $2}' | $TAIL -9 ` COMMAND_3=` $CAT /tmp/$THIS_HOST.memory.log | $AWK '{print $3}' | $TAIL -9 ` ## COMMAND_8=` $CAT /tmp/$THIS_HOST.memory.log | $AWK '{print $8}' | $TAIL -9 ` COMMAND_9=` $CAT /tmp/$THIS_HOST.memory.log | $AWK '{print $9}' | $TAIL -9 ` COMMAND_12=` $CAT /tmp/$THIS_HOST.memory.log | $AWK '{print $12}' | $TAIL -9 ` if [ "$RESULT_FLAG" = "0" ] then ##### ##### Echo the results first ##### ## $CAT /tmp/$THIS_HOST.memory.log $CAT /tmp/$THIS_HOST.mem_pp.log echo "" ##### ##### Check for run queue length ##### for VALUE in $COMMAND_1 do if [ "$VALUE" -gt "5" ] then COMMAND_1_RESULTS="$VALUE" if [ "$COLOR" != "red" -a "$COLOR" != "purple" ] then COLOR="yellow" fi fi done ##### ##### Check for blocked processes ##### for VALUE in $COMMAND_2 do if [ "$VALUE" -gt "5" ] then COMMAND_2_RESULTS="$VALUE" if [ "$COLOR" != "red" -a "$COLOR" != "purple" ] then COLOR="yellow" fi fi done ##### ##### Check for runnable but swapped processes ##### for VALUE in $COMMAND_3 do if [ "$VALUE" -gt "5" ] then COMMAND_3_RESULTS="$VALUE" if [ "$COLOR" != "red" -a "$COLOR" != "purple" ] then COLOR="yellow" fi fi done ##### ##### Check for page in processes ##### ## for VALUE in $COMMAND_8 ## do ## if [ "$VALUE" -gt "10" ] ## then ## COMMAND_8_RESULTS="$VALUE" ## COLOR="yellow" ## fi ## done ##### ##### Check for page out processes ##### for VALUE in $COMMAND_9 do if [ "$VALUE" -gt "10" ] then COMMAND_9_RESULTS="$VALUE" if [ "$COLOR" != "red" -a "$COLOR" != "purple" ] then COLOR="yellow" fi fi done ##### ##### Check for high scan rate ##### for VALUE in $COMMAND_12 do if [ "$VALUE" -gt "10" ] then COMMAND_12_RESULTS="$VALUE" if [ "$COLOR" != "red" -a "$COLOR" != "purple" ] then COLOR="yellow" fi fi done ##### ##### If any are "warnable" then do so. ##### if [ "$COMMAND_1_RESULTS" -gt "0" ] then echo "There's $COMMAND_1_RESULTS processes in run queue." fi if [ "$COMMAND_2_RESULTS" -gt "0" ] then echo "There's $COMMAND_2_RESULTS blocked processes." fi if [ "$COMMAND_3_RESULTS" -gt "0" ] then echo "There's $COMMAND_3_RESULTS runnable, but swapped processes." fi ## if [ "$COMMAND_8_RESULTS" -gt "0" ] ## then ## echo "There's $COMMAND_8_RESULTS page-in processes going on." ## fi if [ "$COMMAND_9_RESULTS" -gt "0" ] then echo "There's $COMMAND_9_RESULTS page-out processes going on." fi if [ "$COMMAND_12_RESULTS" -gt "0" ] then echo "There's a scan rate of $COMMAND_12_RESULTS present." fi else COLOR="red" echo "Problem with virtual memory or command !!!" fi echo "
" echo "" ##### ##### Check the "top -o res -b 3" output - for info only ##### if [ ! -z "$TOP" ] then echo "

" echo "============== Top Resource Hogs ==============" echo "--- ( $TOP -o res -b 3) ---" echo "
" echo "
" CATCH_COMMAND=` $TOP -o res -b 3 | $AWK '/.PID./, /^$/ {print $0}' | $GREP -v "^$"` if [ "$?" = "0" ] then echo "$CATCH_COMMAND" else COLOR="red" echo "Problem with $TOP !!!" fi echo "
" fi echo "" ##### ##### Check the "top -o size -b 3 " output - for info only ##### if [ ! -z "$TOP" ] then echo "

" echo "============== Top Size Hogs ==============" echo "--- ( $TOP -o size -b 3) ---" echo "
" echo "
" CATCH_COMMAND=` $TOP -o size -b 3 | $AWK '/.PID./, /^$/ {print $0}' | $GREP -v "^$"` if [ "$?" = "0" ] then echo "$CATCH_COMMAND" else COLOR="red" echo "Problem with $TOP !!!" fi echo "
" fi echo "" ##### ##### Check the "sar -g 1 1" output (proc paging) for anything non-zero ##### if [ ! -z "$SAR" ] then echo "

" echo "============== Process Paging ==============" echo "--- ( $SAR -g 1 1) ---" echo "
" echo "
" CATCH_COMMAND=` $SAR -g 1 1 | $AWK '/.pgout./, /^$/ {print $0}' ` if [ "$?" = "0" ] then LAST_LINE=` $SAR -g 1 1 | $TAIL -1 | $AWK '{print $2,$3,$4,$5}' | $GREP -v "[ | ]0.00" ` if [ ! -z "$LAST_LINE" ] then if [ "$COLOR" != "red" -a "$COLOR" != "purple" ] then COLOR="yellow" fi echo "Some issues with paging - non zero values" fi echo "$CATCH_COMMAND" INODE_INFO=` $SAR -g 1 1 | $TAIL -1 | $AWK '{print $NF}' | $GREP -v "^0.00" ` if [ ! -z "$INODE_INFO" ] then if [ "$COLOR" != "red" -a "$COLOR" != "purple" ] then COLOR="yellow" fi echo "Looks like inode cache may need to be increased" fi else COLOR="red" echo "Problem with $SAR !!!" fi echo "
" fi echo "" ##### ##### Check the "sar -w 1 1" output (proc swapping) - for info only right now ##### if [ ! -z "$SAR" ] then echo "

" echo "============== Process Swapping ==============" echo "--- ( $SAR -w 1 1) ---" echo "
" echo "
" CATCH_COMMAND=` $SAR -w 1 1 | $AWK '/.swpin./, /^$/ {print $0}' ` if [ "$?" = "0" ] then echo "$CATCH_COMMAND" else COLOR="red" echo "Problem with $SAR !!!" fi echo "
" fi echo "" ##### ##### Check the "sar -k 1 1" output (mem alloc) for anything non-zero in fail columns ##### if [ ! -z "$SAR" ] then echo "

" echo "============== Memory Allocation Failures ==============" echo "--- ( $SAR -k 1 1) ---" echo "
" echo "
" CATCH_COMMAND=` $SAR -k 1 1 | $AWK '/.alloc./, /^$/ {print $0}' ` if [ "$?" = "0" ] then LAST_LINE=` $SAR -k 1 1 | $TAIL -1 | $AWK '{print $4,$7,$9}' | $GREP -v "[ | ]0" ` if [ ! -z "$LAST_LINE" ] then echo "Some alloc failures - non zero values" COLOR="red" fi echo "$CATCH_COMMAND" else COLOR="red" echo "Problem with $SAR !!!" fi echo "
" fi echo "" ##### ##### Check the "sar -r 1 1" output (swap info) for anything 1024 or less in freeswap ##### if [ ! -z "$SAR" ] then echo "

" echo "============== Free Memory/Swap ==============" echo "--- ( $SAR -r 1 1) ---" echo "
" echo "
" CATCH_COMMAND=` $SAR -r 1 1 | $AWK '/.freemem./, /^$/ {print $0}' ` if [ "$?" = "0" ] then FREESWAP=` $SAR -r 1 1 | $TAIL -1 | $AWK '{print $NF}' ` if [ "$FREESWAP" -lt "1024" ] then COLOR="red" export COLOR echo "Less than 1k free swap - $FREESWAP " fi echo "$CATCH_COMMAND" else COLOR="red" echo "Problem with $SAR !!!" fi echo "
" fi echo "" ##### ##### Check the "swap -l" output (swap list) for anything less than 1000 in free ##### if [ ! -z "$SWAP" ] then echo "

" echo "============== Swap List ==============" echo "--- ( $SWAP -l) ---" echo "
" echo "
" CATCH_COMMAND=` $SWAP -l ` if [ "$?" = "0" ] then FREE_LIST=` $SWAP -l | $GREP -v "swaplo" | $AWK '{print $1,$4,$5}' ` echo $FREE_LIST | while read SWAP_DEV_LINE do SWAP_FREE=` echo $SWAP_DEV_LINE | $AWK '{print $3} ' ` if [ "$SWAP_FREE" -lt "1024" ] then SWAP_DEV=` echo $SWAP_DEV_LINE | $AWK '{print $1} ' ` COLOR="red" export COLOR echo "Less than 1k free swap on $SWAP_DEV - $SWAP_FREE \n" fi done echo "$CATCH_COMMAND" else COLOR="red" echo "Problem with $SWAP !!!" fi echo "
" fi echo "" ##### ##### Check the "swap -s" output (swap size) for anything less than 1000 in available ##### if [ ! -z "$SWAP" ] then echo "

" echo "============== Swap Sizes ==============" echo "--- ( $SWAP -s ) ---" echo "
" echo "
" CATCH_COMMAND=` $SWAP -s ` if [ "$?" = "0" ] then FREESPACE=` $SWAP -s | $TAIL -1 | $AWK -F"," '{print $2}' | $AWK -F"k" '{print $1}' | $CUT -c2- ` if [ "$FREESPACE" -lt "1024" ] then COLOR="red" export COLOR echo "Less than 1k total swap free - $FREESPACE \n" fi echo "$CATCH_COMMAND" else COLOR="red" echo "Problem with $SWAP !!!" fi echo "
" fi echo "" ##### ##### Check the "top -b 0" output - for swap info only ##### if [ ! -z "$TOP" ] then echo "

" echo "============== Top's View of Memory/Swap ==============" echo "--- ( $TOP | $GREP Memory ) ---" echo "
" echo "
" CATCH_COMMAND=` $TOP -b 0 | $GREP "^Memory"` if [ "$?" = "0" ] then echo "$CATCH_COMMAND" else COLOR="red" echo "Problem with $TOP !!!" fi echo "
" fi echo "" ##### ##### Make sure to export COLOR so that it gets back to "central" ##### export COLOR ##### ##### End of get_status proc ##### } ##### ##### Main body ##### touch /tmp/$THIS_HOST.memory $GREP $THIS_HOST $BBHOSTS | $GREP "memory" | while read line do if [ ! -z "$line" ] then echo "
" > /tmp/$THIS_HOST.memory echo "Status of memory on $THIS_HOST " >> /tmp/$THIS_HOST.memory echo "Captured : `$DATE` " >> /tmp/$THIS_HOST.memory get_status >> /tmp/$THIS_HOST.memory $BB $BBDISP "status $THIS_HOST.memory $COLOR `$DATE` `$CAT /tmp/$THIS_HOST.memory` " else COLOR="clear" $BB $BBDISP "status $THIS_HOST.memory $COLOR `$DATE` `$CAT /tmp/$THIS_HOST.memory` " fi done ##### ##### End of custom section added by Paul A. Luzzi ##### ############################################## # end of script ##############################################