kudos/check_health
2
1
Fork 0
Code Issues 7 Pull Requests Projects Releases 1 Wiki Activity

Refactored reporting code into plugin report_std.sh

Browse Source
This commit is contained in:
patvdv
2017-12-17 14:53:22 +01:00
parent 44a87e3b9c
commit 9b25b358e2
4 changed files with 316 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sources/lib/core/display_csv.sh
+1 -1
View File
@@ -97,4 +97,4 @@ return 0
#******************************************************************************
# END of script
#**************************************************************************
#******************************************************************************
sources/lib/core/include_core.sh
+109 -34
View File
@@ -93,15 +93,17 @@ DO_DISPLAY_TERSE=0
DO_NOTIFY_EIF=0
DO_NOTIFY_MAIL=0
DO_NOTIFY_SMS=0
DO_REPORT_STD=0
HAS_DISPLAY_CSV=0
HAS_DISPLAY_INIT=0
HAS_DISPLAY_TERSE=0
HAS_NOTIFY_EIF=0
HAS_NOTIFY_MAIL=0
HAS_NOTIFY_SMS=0
HAS_REPORT_STD=0
# check which core display/notification plugins are installed
# do not use a while-do loop here because mksh/pdksh does not pass updated
# do not use a while-do loop here because mksh/pdksh does not pass updated
# variables back from the sub shell (only works for true ksh88/ksh93)
for FFILE in $(ls -1 ${FPATH_PARENT}/core/*.sh 2>/dev/null | grep -v "include_" 2>/dev/null)
do
@@ -130,6 +132,10 @@ do
HAS_NOTIFY_EIF=1
(( ARG_DEBUG != 0 )) && debug "notify_eif plugin is available"
;;
*report_std.sh)
HAS_REPORT_STD=1
(( ARG_DEBUG != 0 )) && debug "report_std plugin is available"
;;
esac
done
@@ -144,7 +150,7 @@ then
DO_DISPLAY_CSV=1
ARG_VERBOSE=0
else
warn "csv plugin for '--display' not present"
warn "csv plugin for '--display' not present"
fi
;;
init) # init/boot format
@@ -153,7 +159,7 @@ then
DO_DISPLAY_INIT=1
ARG_VERBOSE=0
else
warn "init plugin for '--display' not present"
warn "init plugin for '--display' not present"
fi
;;
terse) # terse format
@@ -162,7 +168,7 @@ then
DO_DISPLAY_TERSE=1
ARG_VERBOSE=0
else
warn "terse plugin for '--display' not present"
warn "terse plugin for '--display' not present"
fi
;;
*) # stdout default
@@ -172,7 +178,7 @@ fi
# --notify
if [[ -n "${ARG_NOTIFY}" ]]
then
# do not use a while-do loop here because mksh/pdksh does not pass updated
# do not use a while-do loop here because mksh/pdksh does not pass updated
# variables back from the sub shell (only works for true ksh88/ksh93)
for NOTIFY_OPTS in $(print "${ARG_NOTIFY}" | tr ',' ' ' 2>/dev/null)
do
@@ -192,6 +198,23 @@ then
esac
done
fi
# --report
if [[ -n "${ARG_REPORT}" ]]
then
# do not use a while-do loop here because mksh/pdksh does not pass updated
# variables back from the sub shell (only works for true ksh88/ksh93)
for REPORT_OPTS in $(print "${ARG_REPORT}" | tr ',' ' ' 2>/dev/null)
do
case "${REPORT_OPTS}" in
*std*) # STDOUT
DO_REPORT_STD=1
;;
*) # no valid option
die "you have specified an invalid option for '--report'"
;;
esac
done
fi
# --mail-to/--notify
if [[ -n "${ARG_MAIL_TO}" ]] && (( DO_NOTIFY_MAIL == 0 ))
then
@@ -218,6 +241,58 @@ if (( DO_NOTIFY_SMS != 0 )) && [[ -z "${ARG_SMS_PROVIDER}" ]]
then
die "you cannot specify '--notify=sms' without '--sms-provider'"
fi
# --report/--detail/--id/--reverse/--last/--today
if (( DO_REPORT_STD != 0 ))
then
if (( ARG_DETAIL != 0 )) && [[ -z "${ARG_FAIL_ID}" ]]
then
die "you must specify an unique value for '--id' when using '--detail'"
fi
if (( ARG_LAST != 0 )) && (( ARG_TODAY != 0 ))
then
die "you cannot specify '--last' with '--today'"
fi
if (( ARG_LAST != 0 )) && (( ARG_DETAIL != 0 ))
then
die "you cannot specify '--last' with '--detail'"
fi
if (( ARG_LAST != 0 )) && (( ARG_REVERSE != 0 ))
then
die "you cannot specify '--last' with '--detail'"
fi
if (( ARG_LAST != 0 )) && [[ -n "${ARG_FAIL_ID}" ]]
then
die "you cannot specify '--last' with '--id'"
fi
if (( ARG_TODAY != 0 )) && (( ARG_DETAIL != 0 ))
then
die "you cannot specify '--today' with '--detail'"
fi
if (( ARG_TODAY != 0 )) && (( ARG_REVERSE != 0 ))
then
die "you cannot specify '--today' with '--detail'"
fi
if (( ARG_TODAY != 0 )) && [[ -n "${ARG_FAIL_ID}" ]]
then
die "you cannot specify '--today' with '--id'"
fi
fi
if (( DO_REPORT_STD == 0 )) && (( ARG_LAST != 0 ))
then
die "you cannot specify '--last' without '--report'"
fi
if (( DO_REPORT_STD == 0 )) && (( ARG_REVERSE != 0 ))
then
die "you cannot specify '--reverse' without '--report'"
fi
if (( DO_REPORT_STD == 0 )) && (( ARG_DETAIL != 0 ))
then
die "you cannot specify '--detail' without '--report'"
fi
if (( DO_REPORT_STD == 0 )) && [[ -n "${ARG_FAIL_ID}" ]]
then
die "you cannot specify '--id' without '--report'"
fi
return 0
}
@@ -235,15 +310,15 @@ typeset EXISTS_HC="$1"
typeset FDIR=""
typeset EXISTS_RC=0
# do not use a while-do loop here because mksh/pdksh does not pass updated
# do not use a while-do loop here because mksh/pdksh does not pass updated
# variables back from the sub shell (only works for true ksh88/ksh93)
for FDIR in $(print "${FPATH}" | tr ':' ' ' 2>/dev/null)
do
$(data_contains_string "${FDIR}" "core")
if (( $? == 0 ))
then
ls "${FDIR}/${EXISTS_HC}" >/dev/null 2>&1 && EXISTS_RC=1
fi
$(data_contains_string "${FDIR}" "core")
if (( $? == 0 ))
then
ls "${FDIR}/${EXISTS_HC}" >/dev/null 2>&1 && EXISTS_RC=1
fi
done
return ${EXISTS_RC}
@@ -314,7 +389,7 @@ then
I=$(( I + 1 ))
done <${HC_MSG_FILE} 2>/dev/null
fi
# display routines
if (( ${#HC_MSG_STC[*]} > 0 ))
then
@@ -335,7 +410,7 @@ then
display_init "${HC_NAME}" "${HC_FAIL_ID}"
else
warn "display_init plugin is not avaible, cannot display_results!"
fi
fi
elif (( DO_DISPLAY_TERSE == 1 ))
then
if (( HAS_DISPLAY_TERSE == 1 ))
@@ -347,7 +422,7 @@ then
fi
else
# default STDOUT
if (( ARG_VERBOSE != 0 ))
if (( ARG_VERBOSE != 0 ))
then
I=0
MAX_I=${#HC_MSG_STC[*]}
@@ -372,25 +447,25 @@ then
# log routine (combined STC=0 or <>0)
I=0
MAX_I=${#HC_MSG_STC[*]}
while (( I < MAX_I ))
while (( I < MAX_I ))
do
printf "%s${SEP}%s${SEP}%s${SEP}%s${SEP}" \
"${HC_MSG_TIME[${I}]}" \
"${HC_NAME}" \
${HC_MSG_STC[${I}]} \
"${HC_MSG_TEXT[${I}]}" >>${HC_LOG}
if (( HC_MSG_STC[${I}] != 0 ))
if (( HC_MSG_STC[${I}] != 0 ))
then
printf "%s${SEP}\n" "${HC_FAIL_ID}" >>${HC_LOG}
printf "%s${SEP}\n" "${HC_FAIL_ID}" >>${HC_LOG}
else
printf "\n" >>${HC_LOG}
fi
HC_STC_COUNT=$(( HC_STC_COUNT + HC_MSG_STC[${I}] ))
I=$(( I + 1 ))
done
# notify routine (combined STC > 0)
if (( HC_STC_COUNT > 0 ))
if (( HC_STC_COUNT > 0 ))
then
# save stdout/stderr to HC events location
if [[ -s ${HC_STDOUT_LOG} ]] || [[ -s ${HC_STDERR_LOG} ]]
@@ -494,14 +569,14 @@ case "${REPORT_STYLE}" in
if (( HAS_DISPLAY_CSV == 1 ))
then
DO_DISPLAY_CSV=1
ARG_VERBOSE=0
ARG_VERBOSE=0
fi
;;
terse|TERSE) # terse format
if (( HAS_DISPLAY_TERSE == 1 ))
then
DO_DISPLAY_TERSE=1
ARG_VERBOSE=0
ARG_VERBOSE=0
fi
;;
*) # init/boot default, stdout fallback
@@ -511,7 +586,7 @@ case "${REPORT_STYLE}" in
ARG_VERBOSE=0
else
ARG_VERBOSE=1
warn "default boot/init display plugin not present"
warn "default boot/init display plugin not present"
fi
esac
@@ -519,10 +594,10 @@ esac
ARG_HC=""
grep -i '^hc:' ${HOST_CONFIG_FILE} 2>/dev/null |\
while IFS=':' read DUMMY HC_EXEC HC_CONFIG HC_DESC
do
do
ARG_HC="${ARG_HC},${HC_EXEC}"
done
return 0
}
@@ -551,7 +626,7 @@ case "${HC_VERSION}" in
;;
*)
die "version of the HC plugin $1 is not in YYYY-MM-DD format (${HC_VERSION})"
;;
;;
esac
return 0
@@ -579,13 +654,13 @@ case "${OS_NAME}" in
# check system crontabs
if (( CRON_COUNT == 0 ))
then
CRON_COUNT=$(cat ${CRON_SYS_LOCATIONS} 2>/dev/null | grep -c -E -e "^[^#].*${CRON_HC}" 2>/dev/null)
CRON_COUNT=$(cat ${CRON_SYS_LOCATIONS} 2>/dev/null | grep -c -E -e "^[^#].*${CRON_HC}" 2>/dev/null)
fi
# check anacron
if (( CRON_COUNT == 0 ))
then
CRON_COUNT=$(cat ${CRON_ANACRON_LOCATIONS} 2>/dev/null | grep -c -E -e "^[^#].*${CRON_HC}" 2>/dev/null)
fi
CRON_COUNT=$(cat ${CRON_ANACRON_LOCATIONS} 2>/dev/null | grep -c -E -e "^[^#].*${CRON_HC}" 2>/dev/null)
fi
;;
*)
# use default root crontab
@@ -640,7 +715,7 @@ do
fi
# check state (only for unlinked)
[[ -h ${FFILE%%.*} ]] || FSTATE="unlinked"
# show results
if [[ "${FACTION}" != "list" ]]
then
@@ -648,7 +723,7 @@ do
"${FNAME#function *}" \
"${FSTATE}" \
"${FVERSION#typeset _VERSION=*}" \
"${FCONFIG}"
"${FCONFIG}"
else
printf "%s\n" "${FNAME#function *}"
fi
@@ -711,7 +786,7 @@ then
else
FNEEDLE="${ARG_LIST}.sh"
fi
# print header
if [[ "${FACTION}" != "list" ]]
then
@@ -751,7 +826,7 @@ do
else
FSCHEDULED="Yes"
fi
# show results
if [[ "${FACTION}" != "list" ]]
then
@@ -760,7 +835,7 @@ do
"${FSTATE}" \
"${FVERSION#typeset _VERSION=*}" \
"${FCONFIG}" \
"${FSCHEDULED}"
"${FSCHEDULED}"
else
printf "%s\n" "${FNAME#function *}"
fi
@@ -830,7 +905,7 @@ return 0
# @(#) FUNCTION: log_hc()
# DOES: log a HC plugin result
# EXPECTS: 1=HC name [string], 2=HC status code [integer], 3=HC message [string],
# 4=HC found value [string] (optional),
# 4=HC found value [string] (optional),
# 5=HC expected value [string] (optional)
# RETURNS: 0
# REQUIRES: n/a
sources/lib/core/report_std.sh
+187
View File
@@ -0,0 +1,187 @@
#!/usr/bin/env ksh
#******************************************************************************
# @(#) report_std.sh
#******************************************************************************
# @(#) Copyright (C) 2017 by KUDOS BVBA (info@kudos.be). All rights reserved.
#
# This program is a free software; you can redistribute it and/or modify
# it under the same terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details
#******************************************************************************
#
# DOCUMENTATION (MAIN)
# -----------------------------------------------------------------------------
# @(#) MAIN: report_std
# DOES: report HC events on STDOUT
# EXPECTS: n/a
# RETURNS: 0
# REQUIRES: init_hc(), list_hc(), $EVENTS_DIR, $HC_LOG
#
# -----------------------------------------------------------------------------
# DO NOT CHANGE THIS FILE UNLESS YOU KNOW WHAT YOU ARE DOING!
#******************************************************************************
# -----------------------------------------------------------------------------
function report_std
{
# ------------------------- CONFIGURATION starts here -------------------------
typeset _VERSION="2017-12-15" # YYYY-MM-DD
typeset _SUPPORTED_PLATFORMS="AIX,HP-UX,Linux" # uname -s match
# ------------------------- CONFIGURATION ends here ---------------------------
# set defaults
(( ARG_DEBUG != 0 && ARG_DEBUG_LEVEL > 0 )) && set "${DEBUG_OPTS}"
init_hc "$0" "${_SUPPORTED_PLATFORMS}" "${_VERSION}"
typeset _DIR_PREFIX=""
typeset _EVENT_COUNT=0
typeset _FAIL_COUNT=0
typeset _FAIL_F1=""
typeset _FAIL_F2=""
typeset _FAIL_F3=""
typeset _FAIL_F4=""
typeset _HC_LAST=""
typeset _HC_LAST_TIME=""
typeset _HC_LAST_STC=0
typeset _HC_LAST_FAIL_ID="-"
typeset _HC_LAST_EVENT_FAIL_ID=0
typeset _HC_LAST_EVENT_STC=""
typeset _ID_NEEDLE=""
typeset _REPORT_LINE=""
typeset _SORT_CMD=""
# --last report
if (( ARG_LAST != 0 ))
then
printf "\n| %-30s | %-20s | %-14s | %-4s\n" "HC" "Timestamp" "FAIL ID" "STC (combined value)"
printf "%100s\n" | tr ' ' -
# loop over all HCs
list_hc "list" | while read -r _HC_LAST
do
_HC_LAST_TIME=""
_HC_LAST_STC=0
_HC_LAST_FAIL_ID="-"
# find last event or block of events (same timestamp)
# (but unfortunately this is only accurate to events within the SAME second!)
_HC_LAST_TIME="$(grep ${_HC_LAST} ${HC_LOG} 2>/dev/null | sort -n | cut -f1 -d${SEP} | uniq | tail -1)"
if [[ -z "${_HC_LAST_TIME}" ]]
then
_HC_LAST_TIME="-"
_HC_LAST_STC="-"
else
# find all STC codes for the last event and add them up
grep "${_HC_LAST_TIME}${SEP}${HC_LAST}" ${HC_LOG} 2>/dev/null |\
while read -r _REPORT_LINE
do
_HC_LAST_EVENT_STC=$(print "${_REPORT_LINE}" | cut -f3 -d"${SEP}")
_HC_LAST_EVENT_FAIL_ID=$(print "${_REPORT_LINE}" | cut -f5 -d"${SEP}")
_HC_LAST_STC=$(( _HC_LAST_STC + _HC_LAST_EVENT_STC ))
[[ -n "${_HC_LAST_EVENT_FAIL_ID}" ]] && _HC_LAST_FAIL_ID="${_HC_LAST_EVENT_FAIL_ID}"
done
fi
# report on findings
printf "| %-30s | %-20s | %-14s | %-4s\n" \
"${_HC_LAST}" "${_HC_LAST_TIME}" "${_HC_LAST_FAIL_ID}" "${_HC_LAST_STC}"
done
# disclaimer
print "Note: this report only shows the overall combined status of all events of each HC within exactly"
print " the *same* time stamp (seconds precise). It may therefore fail to report certain FAIL IDs."
print " Use $0 --report to get the exact list of failure events."
# other reports
else
_ID_NEEDLE="[0-9][0-9]*"
[[ -n "${ARG_FAIL_ID}" ]] && _ID_NEEDLE="${ARG_FAIL_ID}"
(( ARG_TODAY != 0 )) && _ID_NEEDLE="$(date '+%Y%m%d')" # refers to timestamp of HC FAIL_ID
# check fail count (look for unique IDs in the 5th field of the HC log)
_FAIL_COUNT=$(cut -f5 -d"${SEP}" ${HC_LOG} 2>/dev/null | grep -E -e "${_ID_NEEDLE}" | uniq | wc -l)
if (( _FAIL_COUNT != 0 ))
then
# check for detail or not?
if (( ARG_DETAIL != 0 )) && (( _FAIL_COUNT != 1 ))
then
ARG_LOG=1 die "you must specify a unique FAIL_ID value"
fi
# reverse?
if (( ARG_REVERSE == 0 ))
then
_SORT_CMD="sort -n"
else
_SORT_CMD="sort -rn"
fi
# global or detailed?
if (( ARG_DETAIL == 0 ))
then
printf "\n| %-20s | %-14s | %-30s | %-s\n" \
"Timestamp" "FAIL ID" "HC" "Message"
printf "%120s\n" | tr ' ' -
# print failed events
# no extended grep here and no end $SEP!
grep ".*${SEP}.*${SEP}.*${SEP}.*${SEP}${_ID_NEEDLE}" ${HC_LOG} 2>/dev/null |\
${_SORT_CMD} | while read -r _REPORT_LINE
do
_FAIL_F1=$(print "${_REPORT_LINE}" | cut -f1 -d"${SEP}")
_FAIL_F2=$(print "${_REPORT_LINE}" | cut -f2 -d"${SEP}")
_FAIL_F3=$(print "${_REPORT_LINE}" | cut -f4 -d"${SEP}")
_FAIL_F4=$(print "${_REPORT_LINE}" | cut -f5 -d"${SEP}")
printf "| %-20s | %-14s | %-30s | %-s\n" \
"${_FAIL_F1}" "${_FAIL_F4}" "${_FAIL_F2}" "${_FAIL_F3}"
done
printf "\n%-s\n" "SUMMARY: ${_FAIL_COUNT} failed HC event(s) found."
else
# print failed events (we may have multiple events for 1 FAIL ID)
_EVENT_COUNT=1
_DIR_PREFIX="$(expr substr ${ARG_FAIL_ID} 1 4)-$(expr substr ${ARG_FAIL_ID} 5 2)"
# no extended grep here!
grep ".*${SEP}.*${SEP}.*${SEP}.*${SEP}${_ID_NEEDLE}${SEP}" ${HC_LOG} 2>/dev/null |\
${_SORT_CMD} | while read -r _REPORT_LINE
do
_FAIL_F1=$(print "${_REPORT_LINE}" | cut -f1 -d"${SEP}")
_FAIL_F2=$(print "${_REPORT_LINE}" | cut -f2 -d"${SEP}")
_FAIL_F3=$(print "${_REPORT_LINE}" | cut -f4 -d"${SEP}")
printf "%36sMSG #%03d%36s" "" ${_EVENT_COUNT} "" | tr ' ' -
printf "\nTime : %-s\nHC : %-s\nDetail : %-s\n" \
"${_FAIL_F1}" "${_FAIL_F2}" "${_FAIL_F3}"
_EVENT_COUNT=$(( _EVENT_COUNT + 1 ))
done
printf "%37sSTDOUT%37s\n" | tr ' ' -;
# display non-empty STDOUT file(s)
if [[ -n "$(du -a ${EVENTS_DIR}/${_DIR_PREFIX}/${ARG_FAIL_ID}/*.stdout.log 2>/dev/null | awk '$1*512 > 0 {print $2}')" ]]
then
cat ${EVENTS_DIR}/${_DIR_PREFIX}/${ARG_FAIL_ID}/*.stdout.log
else
printf "%-s\n" "No STDOUT found"
fi
printf "%37sSTDERR%37s\n" | tr ' ' -;
# display non-empty STDERR file(s)
if [[ -n "$(du -a ${EVENTS_DIR}/${_DIR_PREFIX}/${ARG_FAIL_ID}/*.stderr.log 2>/dev/null | awk '$1*512 > 0 {print $2}')" ]]
then
cat ${EVENTS_DIR}/${_DIR_PREFIX}/${ARG_FAIL_ID}/*.stderr.log
else
printf "%-s\n" "No STDERR found"
fi
printf "%80s\n" | tr ' ' -
fi
else
printf "\n%-s\n" "SUMMARY: 0 failed HC events found."
fi
fi
return 0
}
#******************************************************************************
# END of script
#******************************************************************************