Refactored reporting code into plugin report_std.sh

sources/bin/check_health.sh

@@ -38,7 +38,7 @@
 
 # ------------------------- CONFIGURATION starts here -------------------------
 # define the version (YYYY-MM-DD)
-typeset -r SCRIPT_VERSION="2017-09-11"
+typeset -r SCRIPT_VERSION="2017-12-15"
 # location of parent directory containing KSH functions/HC plugins
 typeset -r FPATH_PARENT="/opt/hc/lib"
 # location of custom HC configuration files
@@ -96,6 +96,7 @@ typeset ARG_LOG=1               # logging is on by default
 typeset ARG_MONITOR=1           # killing long running HC processes is on by default
 typeset ARG_NOTIFY=""           # notification of problems is off by default
 typeset ARG_REVERSE=0           # show report in reverse date order is off by default
+typeset ARG_REPORT=""           # report of HC events is off by default
 typeset ARG_TERSE=0             # show terse help is off by default
 typeset ARG_TODAY=0             # report today's events
 typeset ARG_VERBOSE=1           # STDOUT is on by default
@@ -308,70 +309,6 @@ then
         exit 1
     fi
 fi
-# --report/--detail/--id/--reverse/--last/--today
-if (( ARG_ACTION == 8 ))
-then
-    if (( ARG_DETAIL != 0 )) && [[ -z "${ARG_FAIL_ID}" ]]
-    then
-        print -u2 "ERROR: you must specify an unique value for '--id' when using '--detail'"
-        exit 1
-    fi
-    if (( ARG_LAST != 0 )) && (( ARG_TODAY != 0 ))
-    then
-        print -u2 "ERROR: you cannot specify '--last' with '--today'"
-        exit 1
-    fi
-    if (( ARG_LAST != 0 )) && (( ARG_DETAIL != 0 ))
-    then
-        print -u2 "ERROR: you cannot specify '--last' with '--detail'"
-        exit 1
-    fi
-    if (( ARG_LAST != 0 )) && (( ARG_REVERSE != 0 ))
-    then
-        print -u2 "ERROR: you cannot specify '--last' with '--detail'"
-        exit 1
-    fi
-    if (( ARG_LAST != 0 )) && [[ -n "${ARG_FAIL_ID}" ]]
-    then
-        print -u2 "ERROR: you cannot specify '--last' with '--id'"
-        exit 1
-    fi
-    if (( ARG_TODAY != 0 )) && (( ARG_DETAIL != 0 ))
-    then
-        print -u2 "ERROR: you cannot specify '--today' with '--detail'"
-        exit 1
-    fi
-    if (( ARG_TODAY != 0 )) && (( ARG_REVERSE != 0 ))
-    then
-        print -u2 "ERROR: you cannot specify '--today' with '--detail'"
-        exit 1
-    fi
-    if (( ARG_TODAY != 0 )) && [[ -n "${ARG_FAIL_ID}" ]]
-    then
-        print -u2 "ERROR: you cannot specify '--today' with '--id'"
-        exit 1
-    fi
-fi
-if (( ARG_ACTION != 8 )) && (( ARG_LAST != 0 ))
-then
-    print -u2 "ERROR: you cannot specify '--last' without '--report'"
-    exit 1
-fi
-if (( ARG_ACTION != 8 )) && (( ARG_REVERSE != 0 ))
-then
-    print -u2 "ERROR: you cannot specify '--reverse' without '--report'"
-    exit 1
-fi
-if (( ARG_ACTION != 8 )) && (( ARG_DETAIL != 0 ))
-then
-    print -u2 "ERROR: you cannot specify '--detail' without '--report'"
-    exit 1
-fi
-if (( ARG_ACTION != 8 )) && [[ -n "${ARG_FAIL_ID}" ]]
-then
-    print -u2 "ERROR: you cannot specify '--id' without '--report'"
-    exit 1
-fi
 # --check-host,--check/--disable/--enable/--run/--show,--hc
 if [[ -n "${ARG_HC}" ]] && (( ARG_ACTION == 0 ))
 then
@@ -507,7 +444,7 @@ Syntax: ${SCRIPT_DIR}/${SCRIPT_NAME} [--help] | [--help-terse] | [--version] |
         (--check-host | ((--check | --enable | --disable | --run | --show) --hc=<list_of_checks> [--config-file=<configuration_file>] [hc-args="<arg1,arg2=val,arg3">]))
             [--display=<method>] ([--debug] [--debug-level=<level>]) [--no-monitor] [--no-log] [--log-dir=<log_directory>]
                 [--notify=<method_list>] [--mail-to=<address_list>] [--sms-to=<sms_rcpt> --sms-provider=<name>]
-                    --report ( ([--last] | [--today]) | ([--reverse] [--id=<fail_id> [--detail]]) )
+                    [--report=<method> ( ([--last] | [--today]) | ([--reverse] [--id=<fail_id> [--detail]]) ) ]
 
 EOT
 
@@ -545,7 +482,7 @@ Parameters:
 --no-log        : do not log any messages to the script log file or health check results.
 --no-monitor    : do not stop the execution of a HC after \$HC_TIME_OUT seconds
 --notify        : notify upon HC failure(s). Multiple options may be specified if comma-separated (see --list-core for availble formats)
---report        : report on failed HC events
+--report        : report on failed HC events (STDOUT is the default reporting method)
 --reverse       : show the report in reverse date order (newest events first)
 --run           : execute HC(s).
 --show          : show information/documentation on a HC
@@ -802,7 +739,19 @@ do
         -no-monitor|--no-monitor)
             ARG_MONITOR=0
             ;;
-        -report|--report)
+        -report|--report)   # compatability support <2017-12-15
+            # STDOUT as default
+            ARG_REPORT="std"
+            ARG_LOG=0; ARG_VERBOSE=0
+            ARG_ACTION=8
+            ;;
+        -report=*)
+            ARG_REPORT="${PARAMETER#-report=}"
+            ARG_LOG=0; ARG_VERBOSE=0
+            ARG_ACTION=8
+            ;;
+        --report=*)
+            ARG_REPORT="${PARAMETER#--report=}"
             ARG_LOG=0; ARG_VERBOSE=0
             ARG_ACTION=8
             ;;
@@ -1122,128 +1071,8 @@ case ${ARG_ACTION} in
             fi
         done
         ;;
-    8)  # report on last events or FAIL_IDs
+    8)  # report on HC events
-        if (( ARG_LAST != 0 ))
+        (( DO_REPORT_STD == 1 )) && report_std
-        then
-            printf "\n| %-30s | %-20s | %-14s | %-4s\n" "HC" "Timestamp" "FAIL ID" "STC (combined value)"
-            printf "%100s\n" | tr ' ' -
-            # loop over all HCs
-            list_hc "list" | while read -r HC_LAST
-            do
-                HC_LAST_TIME=""
-                HC_LAST_STC=0
-                HC_LAST_FAIL_ID="-"
-                # find last event or block of events (same timestamp)
-                # (but unfortunately this is only accurate to events within the SAME second!)
-                HC_LAST_TIME="$(grep ${HC_LAST} ${HC_LOG} 2>/dev/null | sort -n | cut -f1 -d${SEP} | uniq | tail -1)"
-                if [[ -z "${HC_LAST_TIME}" ]]
-                then
-                    HC_LAST_TIME="-"
-                    HC_LAST_STC="-"
-                else
-                    # find all STC codes for the last event and add them up
-                    grep "${HC_LAST_TIME}${SEP}${HC_LAST}" ${HC_LOG} 2>/dev/null |\
-                        while read -r REPORT_LINE
-                    do
-                        HC_LAST_EVENT_STC=$(print "${REPORT_LINE}" | cut -f3 -d"${SEP}")
-                        HC_LAST_EVENT_FAIL_ID=$(print "${REPORT_LINE}" | cut -f5 -d"${SEP}")
-                        HC_LAST_STC=$(( HC_LAST_STC + HC_LAST_EVENT_STC ))
-                        [[ -n "${HC_LAST_EVENT_FAIL_ID}" ]] && HC_LAST_FAIL_ID="${HC_LAST_EVENT_FAIL_ID}"
-                    done
-                fi
-                # report on findings
-                printf "| %-30s | %-20s | %-14s | %-4s\n" \
-                    "${HC_LAST}" "${HC_LAST_TIME}" "${HC_LAST_FAIL_ID}" "${HC_LAST_STC}"
-            done
-            # disclaimer
-            print "Note: this report only shows the overall combined status of all events of each HC within exactly"
-            print "      the *same* time stamp (seconds precise). It may therefore fail to report certain FAIL IDs."
-            print "      Use $0 --report to get the exact list of failure events."
-        else
-            ID_NEEDLE="[0-9][0-9]*"
-            [[ -n "${ARG_FAIL_ID}" ]] && ID_NEEDLE="${ARG_FAIL_ID}"
-            (( ARG_TODAY != 0 )) && ID_NEEDLE="$(date '+%Y%m%d')"    # refers to timestamp of HC FAIL_ID
-
-            # check fail count (look for unique IDs in the 5th field of the HC log)
-            FAIL_COUNT=$(cut -f5 -d"${SEP}" ${HC_LOG} 2>/dev/null | grep -E -e "${ID_NEEDLE}" | uniq | wc -l)
-            if (( FAIL_COUNT != 0 ))
-            then
-                # check for detail or not?
-                if (( ARG_DETAIL != 0 )) && (( FAIL_COUNT != 1 ))
-                then
-                    ARG_LOG=1 die "you must specify a unique FAIL_ID value"
-                fi
-                # reverse?
-                if (( ARG_REVERSE == 0 ))
-                then
-                    SORT_CMD="sort -n"
-                else
-                    SORT_CMD="sort -rn"
-                fi
-                # global or detailed?
-                if (( ARG_DETAIL == 0 ))
-                then
-                    printf "\n| %-20s | %-14s | %-30s | %-s\n" \
-                        "Timestamp" "FAIL ID" "HC" "Message"
-                    printf "%120s\n" | tr ' ' -
-
-                    # print failed events
-                    # no extended grep here and no end $SEP!
-                    grep ".*${SEP}.*${SEP}.*${SEP}.*${SEP}${ID_NEEDLE}" ${HC_LOG} 2>/dev/null |\
-                        ${SORT_CMD} | while read -r REPORT_LINE
-                    do
-                        FAIL_F1=$(print "${REPORT_LINE}" | cut -f1 -d"${SEP}")
-                        FAIL_F2=$(print "${REPORT_LINE}" | cut -f2 -d"${SEP}")
-                        FAIL_F3=$(print "${REPORT_LINE}" | cut -f4 -d"${SEP}")
-                        FAIL_F4=$(print "${REPORT_LINE}" | cut -f5 -d"${SEP}")
-
-                        printf "| %-20s | %-14s | %-30s | %-s\n" \
-                            "${FAIL_F1}" "${FAIL_F4}" "${FAIL_F2}" "${FAIL_F3}"
-                    done
-
-                    printf "\n%-s\n" "SUMMARY: ${FAIL_COUNT} failed HC event(s) found."
-                else
-                    # print failed events (we may have multiple events for 1 FAIL ID)
-                    EVENT_COUNT=1
-                    DIR_PREFIX="$(expr substr ${ARG_FAIL_ID} 1 4)-$(expr substr ${ARG_FAIL_ID} 5 2)"
-                    # no extended grep here!
-                    grep ".*${SEP}.*${SEP}.*${SEP}.*${SEP}${ID_NEEDLE}${SEP}" ${HC_LOG} 2>/dev/null |\
-                        ${SORT_CMD} | while read -r REPORT_LINE
-                    do
-                        FAIL_F1=$(print "${REPORT_LINE}" | cut -f1 -d"${SEP}")
-                        FAIL_F2=$(print "${REPORT_LINE}" | cut -f2 -d"${SEP}")
-                        FAIL_F3=$(print "${REPORT_LINE}" | cut -f4 -d"${SEP}")
-
-                        printf "%36sMSG #%03d%36s" "" ${EVENT_COUNT} "" | tr ' ' -
-                        printf "\nTime    : %-s\nHC      : %-s\nDetail  : %-s\n" \
-                            "${FAIL_F1}" "${FAIL_F2}" "${FAIL_F3}"
-                        EVENT_COUNT=$(( EVENT_COUNT + 1 ))
-                    done
-
-                    printf "%37sSTDOUT%37s\n" | tr ' ' -;
-                    # display non-empty STDOUT file(s)
-                    if [[ -n "$(du -a ${EVENTS_DIR}/${DIR_PREFIX}/${ARG_FAIL_ID}/*.stdout.log 2>/dev/null | awk '$1*512 > 0 {print $2}')"  ]]
-                    then
-                        cat ${EVENTS_DIR}/${DIR_PREFIX}/${ARG_FAIL_ID}/*.stdout.log
-                    else
-                        printf "%-s\n" "No STDOUT found"
-                    fi
-
-                    printf "%37sSTDERR%37s\n" | tr ' ' -;
-                    # display non-empty STDERR file(s)
-                    if [[ -n "$(du -a ${EVENTS_DIR}/${DIR_PREFIX}/${ARG_FAIL_ID}/*.stderr.log 2>/dev/null | awk '$1*512 > 0 {print $2}')" ]]
-                    then
-                        cat ${EVENTS_DIR}/${DIR_PREFIX}/${ARG_FAIL_ID}/*.stderr.log
-                    else
-                        printf "%-s\n" "No STDERR found"
-                    fi
-
-                    printf "%80s\n" | tr ' ' -
-                fi
-            else
-                printf "\n%-s\n" "SUMMARY: 0 failed HC events found."
-            fi
-        fi
         ;;
     9)  # list HC plugins
         list_hc "" "${ARG_LIST}"

sources/lib/core/display_csv.sh

@@ -97,4 +97,4 @@ return 0
 
 #******************************************************************************
 # END of script
-#**************************************************************************
+#******************************************************************************

sources/lib/core/include_core.sh

@@ -93,15 +93,17 @@ DO_DISPLAY_TERSE=0
 DO_NOTIFY_EIF=0
 DO_NOTIFY_MAIL=0
 DO_NOTIFY_SMS=0
+DO_REPORT_STD=0
 HAS_DISPLAY_CSV=0
 HAS_DISPLAY_INIT=0
 HAS_DISPLAY_TERSE=0
 HAS_NOTIFY_EIF=0
 HAS_NOTIFY_MAIL=0
 HAS_NOTIFY_SMS=0
+HAS_REPORT_STD=0
 
 # check which core display/notification plugins are installed
-# do not use a while-do loop here because mksh/pdksh does not pass updated 
+# do not use a while-do loop here because mksh/pdksh does not pass updated
 # variables back from the sub shell (only works for true ksh88/ksh93)
 for FFILE in $(ls -1 ${FPATH_PARENT}/core/*.sh 2>/dev/null | grep -v "include_" 2>/dev/null)
 do
@@ -130,6 +132,10 @@ do
             HAS_NOTIFY_EIF=1
             (( ARG_DEBUG != 0 )) && debug "notify_eif plugin is available"
             ;;
+        *report_std.sh)
+            HAS_REPORT_STD=1
+            (( ARG_DEBUG != 0 )) && debug "report_std plugin is available"
+            ;;
     esac
 done
 
@@ -144,7 +150,7 @@ then
                 DO_DISPLAY_CSV=1
                 ARG_VERBOSE=0
             else
-                warn "csv plugin for '--display' not present"             
+                warn "csv plugin for '--display' not present"
             fi
             ;;
         init) # init/boot format
@@ -153,7 +159,7 @@ then
                 DO_DISPLAY_INIT=1
                 ARG_VERBOSE=0
             else
-                warn "init plugin for '--display' not present"             
+                warn "init plugin for '--display' not present"
             fi
             ;;
         terse) # terse format
@@ -162,7 +168,7 @@ then
                 DO_DISPLAY_TERSE=1
                 ARG_VERBOSE=0
             else
-                warn "terse plugin for '--display' not present"             
+                warn "terse plugin for '--display' not present"
             fi
             ;;
         *) # stdout default
@@ -172,7 +178,7 @@ fi
 # --notify
 if [[ -n "${ARG_NOTIFY}" ]]
 then
-    # do not use a while-do loop here because mksh/pdksh does not pass updated 
+    # do not use a while-do loop here because mksh/pdksh does not pass updated
     # variables back from the sub shell (only works for true ksh88/ksh93)
     for NOTIFY_OPTS in $(print "${ARG_NOTIFY}" | tr ',' ' ' 2>/dev/null)
     do
@@ -192,6 +198,23 @@ then
         esac
     done
 fi
+# --report
+if [[ -n "${ARG_REPORT}" ]]
+then
+    # do not use a while-do loop here because mksh/pdksh does not pass updated
+    # variables back from the sub shell (only works for true ksh88/ksh93)
+    for REPORT_OPTS in $(print "${ARG_REPORT}" | tr ',' ' ' 2>/dev/null)
+    do
+        case "${REPORT_OPTS}" in
+            *std*) # STDOUT
+                DO_REPORT_STD=1
+                ;;
+            *) # no valid option
+                die "you have specified an invalid option for '--report'"
+                ;;
+        esac
+    done
+fi
 # --mail-to/--notify
 if [[ -n "${ARG_MAIL_TO}" ]] && (( DO_NOTIFY_MAIL == 0 ))
 then
@@ -218,6 +241,58 @@ if (( DO_NOTIFY_SMS != 0 )) && [[ -z "${ARG_SMS_PROVIDER}" ]]
 then
     die "you cannot specify '--notify=sms' without '--sms-provider'"
 fi
+# --report/--detail/--id/--reverse/--last/--today
+if (( DO_REPORT_STD != 0 ))
+then
+    if (( ARG_DETAIL != 0 )) && [[ -z "${ARG_FAIL_ID}" ]]
+    then
+        die "you must specify an unique value for '--id' when using '--detail'"
+    fi
+    if (( ARG_LAST != 0 )) && (( ARG_TODAY != 0 ))
+    then
+        die "you cannot specify '--last' with '--today'"
+    fi
+    if (( ARG_LAST != 0 )) && (( ARG_DETAIL != 0 ))
+    then
+        die "you cannot specify '--last' with '--detail'"
+    fi
+    if (( ARG_LAST != 0 )) && (( ARG_REVERSE != 0 ))
+    then
+        die "you cannot specify '--last' with '--detail'"
+    fi
+    if (( ARG_LAST != 0 )) && [[ -n "${ARG_FAIL_ID}" ]]
+    then
+        die "you cannot specify '--last' with '--id'"
+    fi
+    if (( ARG_TODAY != 0 )) && (( ARG_DETAIL != 0 ))
+    then
+        die "you cannot specify '--today' with '--detail'"
+    fi
+    if (( ARG_TODAY != 0 )) && (( ARG_REVERSE != 0 ))
+    then
+        die "you cannot specify '--today' with '--detail'"
+    fi
+    if (( ARG_TODAY != 0 )) && [[ -n "${ARG_FAIL_ID}" ]]
+    then
+        die "you cannot specify '--today' with '--id'"
+    fi
+fi
+if (( DO_REPORT_STD == 0 )) && (( ARG_LAST != 0 ))
+then
+    die "you cannot specify '--last' without '--report'"
+fi
+if (( DO_REPORT_STD == 0 )) && (( ARG_REVERSE != 0 ))
+then
+    die "you cannot specify '--reverse' without '--report'"
+fi
+if (( DO_REPORT_STD == 0 )) && (( ARG_DETAIL != 0 ))
+then
+    die "you cannot specify '--detail' without '--report'"
+fi
+if (( DO_REPORT_STD == 0 )) && [[ -n "${ARG_FAIL_ID}" ]]
+then
+    die "you cannot specify '--id' without '--report'"
+fi
 
 return 0
 }
@@ -235,15 +310,15 @@ typeset EXISTS_HC="$1"
 typeset FDIR=""
 typeset EXISTS_RC=0
 
-# do not use a while-do loop here because mksh/pdksh does not pass updated 
+# do not use a while-do loop here because mksh/pdksh does not pass updated
 # variables back from the sub shell (only works for true ksh88/ksh93)
 for FDIR in $(print "${FPATH}" | tr ':' ' ' 2>/dev/null)
 do
-	$(data_contains_string "${FDIR}" "core")
+    $(data_contains_string "${FDIR}" "core")
-	if (( $? == 0 ))
+    if (( $? == 0 ))
-	then
+    then
-		ls "${FDIR}/${EXISTS_HC}" >/dev/null 2>&1 && EXISTS_RC=1
+        ls "${FDIR}/${EXISTS_HC}" >/dev/null 2>&1 && EXISTS_RC=1
-	fi
+    fi
 done
 
 return ${EXISTS_RC}
@@ -314,7 +389,7 @@ then
         I=$(( I + 1 ))
     done <${HC_MSG_FILE} 2>/dev/null
 fi
-    
+
 # display routines
 if (( ${#HC_MSG_STC[*]} > 0 ))
 then
@@ -335,7 +410,7 @@ then
             display_init "${HC_NAME}" "${HC_FAIL_ID}"
         else
             warn "display_init plugin is not avaible, cannot display_results!"
-        fi      
+        fi
     elif (( DO_DISPLAY_TERSE == 1 ))
     then
         if (( HAS_DISPLAY_TERSE == 1 ))
@@ -347,7 +422,7 @@ then
         fi
     else
         # default STDOUT
-        if (( ARG_VERBOSE != 0 )) 
+        if (( ARG_VERBOSE != 0 ))
         then
             I=0
             MAX_I=${#HC_MSG_STC[*]}
@@ -372,25 +447,25 @@ then
     # log routine (combined STC=0 or <>0)
     I=0
     MAX_I=${#HC_MSG_STC[*]}
-    while (( I < MAX_I ))    
+    while (( I < MAX_I ))
     do
         printf "%s${SEP}%s${SEP}%s${SEP}%s${SEP}" \
                 "${HC_MSG_TIME[${I}]}" \
                 "${HC_NAME}" \
                 ${HC_MSG_STC[${I}]} \
                 "${HC_MSG_TEXT[${I}]}" >>${HC_LOG}
-        if (( HC_MSG_STC[${I}] != 0 )) 
+        if (( HC_MSG_STC[${I}] != 0 ))
         then
-            printf "%s${SEP}\n" "${HC_FAIL_ID}" >>${HC_LOG}     
+            printf "%s${SEP}\n" "${HC_FAIL_ID}" >>${HC_LOG}
         else
             printf "\n" >>${HC_LOG}
         fi
         HC_STC_COUNT=$(( HC_STC_COUNT + HC_MSG_STC[${I}] ))
         I=$(( I + 1 ))
     done
-    
+
     # notify routine (combined STC > 0)
-    if (( HC_STC_COUNT > 0 ))   
+    if (( HC_STC_COUNT > 0 ))
     then
         # save stdout/stderr to HC events location
         if [[ -s ${HC_STDOUT_LOG} ]] || [[ -s ${HC_STDERR_LOG} ]]
@@ -494,14 +569,14 @@ case "${REPORT_STYLE}" in
         if (( HAS_DISPLAY_CSV == 1 ))
         then
             DO_DISPLAY_CSV=1
-            ARG_VERBOSE=0          
+            ARG_VERBOSE=0
         fi
         ;;
     terse|TERSE) # terse format
         if (( HAS_DISPLAY_TERSE == 1 ))
         then
             DO_DISPLAY_TERSE=1
-            ARG_VERBOSE=0             
+            ARG_VERBOSE=0
         fi
         ;;
     *) # init/boot default, stdout fallback
@@ -511,7 +586,7 @@ case "${REPORT_STYLE}" in
             ARG_VERBOSE=0
         else
             ARG_VERBOSE=1
-            warn "default boot/init display plugin not present"  
+            warn "default boot/init display plugin not present"
         fi
 esac
 
@@ -519,10 +594,10 @@ esac
 ARG_HC=""
 grep -i '^hc:' ${HOST_CONFIG_FILE} 2>/dev/null |\
     while IFS=':' read DUMMY HC_EXEC HC_CONFIG HC_DESC
-do      
+do
     ARG_HC="${ARG_HC},${HC_EXEC}"
 done
-    
+
 return 0
 }
 
@@ -551,7 +626,7 @@ case "${HC_VERSION}" in
         ;;
     *)
         die "version of the HC plugin $1 is not in YYYY-MM-DD format (${HC_VERSION})"
-        ;;     
+        ;;
 esac
 
 return 0
@@ -579,13 +654,13 @@ case "${OS_NAME}" in
         # check system crontabs
         if (( CRON_COUNT == 0 ))
         then
-            CRON_COUNT=$(cat ${CRON_SYS_LOCATIONS} 2>/dev/null | grep -c -E -e "^[^#].*${CRON_HC}" 2>/dev/null) 
+            CRON_COUNT=$(cat ${CRON_SYS_LOCATIONS} 2>/dev/null | grep -c -E -e "^[^#].*${CRON_HC}" 2>/dev/null)
         fi
         # check anacron
         if (( CRON_COUNT == 0 ))
         then
-            CRON_COUNT=$(cat ${CRON_ANACRON_LOCATIONS} 2>/dev/null | grep -c -E -e "^[^#].*${CRON_HC}" 2>/dev/null) 
+            CRON_COUNT=$(cat ${CRON_ANACRON_LOCATIONS} 2>/dev/null | grep -c -E -e "^[^#].*${CRON_HC}" 2>/dev/null)
-        fi      
+        fi
         ;;
     *)
         # use default root crontab
@@ -640,7 +715,7 @@ do
         fi
         # check state (only for unlinked)
         [[ -h ${FFILE%%.*} ]] || FSTATE="unlinked"
-        
+
         # show results
         if [[ "${FACTION}" != "list" ]]
         then
@@ -648,7 +723,7 @@ do
                 "${FNAME#function *}" \
                 "${FSTATE}" \
                 "${FVERSION#typeset _VERSION=*}" \
-                "${FCONFIG}"        
+                "${FCONFIG}"
         else
             printf "%s\n" "${FNAME#function *}"
         fi
@@ -711,7 +786,7 @@ then
 else
     FNEEDLE="${ARG_LIST}.sh"
 fi
-    
+
 # print header
 if [[ "${FACTION}" != "list" ]]
 then
@@ -751,7 +826,7 @@ do
         else
             FSCHEDULED="Yes"
         fi
-        
+
         # show results
         if [[ "${FACTION}" != "list" ]]
         then
@@ -760,7 +835,7 @@ do
                 "${FSTATE}" \
                 "${FVERSION#typeset _VERSION=*}" \
                 "${FCONFIG}" \
-                "${FSCHEDULED}"         
+                "${FSCHEDULED}"
         else
             printf "%s\n" "${FNAME#function *}"
         fi
@@ -830,7 +905,7 @@ return 0
 # @(#) FUNCTION: log_hc()
 # DOES: log a HC plugin result
 # EXPECTS: 1=HC name [string], 2=HC status code [integer], 3=HC message [string],
-#          4=HC found value [string] (optional), 
+#          4=HC found value [string] (optional),
 #          5=HC expected value [string] (optional)
 # RETURNS: 0
 # REQUIRES: n/a

sources/lib/core/report_std.sh

@@ -0,0 +1,187 @@
+#!/usr/bin/env ksh
+#******************************************************************************
+# @(#) report_std.sh
+#******************************************************************************
+# @(#) Copyright (C) 2017 by KUDOS BVBA (info@kudos.be).  All rights reserved.
+#
+# This program is a free software; you can redistribute it and/or modify
+# it under the same terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details
+#******************************************************************************
+#
+# DOCUMENTATION (MAIN)
+# -----------------------------------------------------------------------------
+# @(#) MAIN: report_std
+# DOES: report HC events on STDOUT
+# EXPECTS: n/a
+# RETURNS: 0
+# REQUIRES: init_hc(), list_hc(), $EVENTS_DIR, $HC_LOG
+#
+# -----------------------------------------------------------------------------
+# DO NOT CHANGE THIS FILE UNLESS YOU KNOW WHAT YOU ARE DOING!
+#******************************************************************************
+
+# -----------------------------------------------------------------------------
+function report_std
+{
+# ------------------------- CONFIGURATION starts here -------------------------
+typeset _VERSION="2017-12-15"                               # YYYY-MM-DD
+typeset _SUPPORTED_PLATFORMS="AIX,HP-UX,Linux"              # uname -s match
+# ------------------------- CONFIGURATION ends here ---------------------------
+
+# set defaults
+(( ARG_DEBUG != 0 && ARG_DEBUG_LEVEL > 0 )) && set "${DEBUG_OPTS}"
+init_hc "$0" "${_SUPPORTED_PLATFORMS}" "${_VERSION}"
+
+typeset _DIR_PREFIX=""
+typeset _EVENT_COUNT=0
+typeset _FAIL_COUNT=0
+typeset _FAIL_F1=""
+typeset _FAIL_F2=""
+typeset _FAIL_F3=""
+typeset _FAIL_F4=""
+typeset _HC_LAST=""
+typeset _HC_LAST_TIME=""
+typeset _HC_LAST_STC=0
+typeset _HC_LAST_FAIL_ID="-"
+typeset _HC_LAST_EVENT_FAIL_ID=0
+typeset _HC_LAST_EVENT_STC=""
+typeset _ID_NEEDLE=""
+typeset _REPORT_LINE=""
+typeset _SORT_CMD=""
+
+# --last report
+if (( ARG_LAST != 0 ))
+then
+    printf "\n| %-30s | %-20s | %-14s | %-4s\n" "HC" "Timestamp" "FAIL ID" "STC (combined value)"
+    printf "%100s\n" | tr ' ' -
+    # loop over all HCs
+    list_hc "list" | while read -r _HC_LAST
+    do
+        _HC_LAST_TIME=""
+        _HC_LAST_STC=0
+        _HC_LAST_FAIL_ID="-"
+        # find last event or block of events (same timestamp)
+        # (but unfortunately this is only accurate to events within the SAME second!)
+        _HC_LAST_TIME="$(grep ${_HC_LAST} ${HC_LOG} 2>/dev/null | sort -n | cut -f1 -d${SEP} | uniq | tail -1)"
+        if [[ -z "${_HC_LAST_TIME}" ]]
+        then
+            _HC_LAST_TIME="-"
+            _HC_LAST_STC="-"
+        else
+            # find all STC codes for the last event and add them up
+            grep "${_HC_LAST_TIME}${SEP}${HC_LAST}" ${HC_LOG} 2>/dev/null |\
+            while read -r _REPORT_LINE
+            do
+                _HC_LAST_EVENT_STC=$(print "${_REPORT_LINE}" | cut -f3 -d"${SEP}")
+                _HC_LAST_EVENT_FAIL_ID=$(print "${_REPORT_LINE}" | cut -f5 -d"${SEP}")
+                _HC_LAST_STC=$(( _HC_LAST_STC + _HC_LAST_EVENT_STC ))
+                [[ -n "${_HC_LAST_EVENT_FAIL_ID}" ]] && _HC_LAST_FAIL_ID="${_HC_LAST_EVENT_FAIL_ID}"
+            done
+        fi
+        # report on findings
+        printf "| %-30s | %-20s | %-14s | %-4s\n" \
+            "${_HC_LAST}" "${_HC_LAST_TIME}" "${_HC_LAST_FAIL_ID}" "${_HC_LAST_STC}"
+    done
+    # disclaimer
+    print "Note: this report only shows the overall combined status of all events of each HC within exactly"
+    print "      the *same* time stamp (seconds precise). It may therefore fail to report certain FAIL IDs."
+    print "      Use $0 --report to get the exact list of failure events."
+# other reports
+else
+    _ID_NEEDLE="[0-9][0-9]*"
+    [[ -n "${ARG_FAIL_ID}" ]] && _ID_NEEDLE="${ARG_FAIL_ID}"
+    (( ARG_TODAY != 0 )) && _ID_NEEDLE="$(date '+%Y%m%d')"    # refers to timestamp of HC FAIL_ID
+
+    # check fail count (look for unique IDs in the 5th field of the HC log)
+    _FAIL_COUNT=$(cut -f5 -d"${SEP}" ${HC_LOG} 2>/dev/null | grep -E -e "${_ID_NEEDLE}" | uniq | wc -l)
+    if (( _FAIL_COUNT != 0 ))
+    then
+        # check for detail or not?
+        if (( ARG_DETAIL != 0 )) && (( _FAIL_COUNT != 1 ))
+        then
+            ARG_LOG=1 die "you must specify a unique FAIL_ID value"
+        fi
+        # reverse?
+        if (( ARG_REVERSE == 0 ))
+        then
+            _SORT_CMD="sort -n"
+        else
+            _SORT_CMD="sort -rn"
+        fi
+        # global or detailed?
+        if (( ARG_DETAIL == 0 ))
+        then
+            printf "\n| %-20s | %-14s | %-30s | %-s\n" \
+                "Timestamp" "FAIL ID" "HC" "Message"
+            printf "%120s\n" | tr ' ' -
+
+            # print failed events
+            # no extended grep here and no end $SEP!
+            grep ".*${SEP}.*${SEP}.*${SEP}.*${SEP}${_ID_NEEDLE}" ${HC_LOG} 2>/dev/null |\
+                ${_SORT_CMD} | while read -r _REPORT_LINE
+            do
+                _FAIL_F1=$(print "${_REPORT_LINE}" | cut -f1 -d"${SEP}")
+                _FAIL_F2=$(print "${_REPORT_LINE}" | cut -f2 -d"${SEP}")
+                _FAIL_F3=$(print "${_REPORT_LINE}" | cut -f4 -d"${SEP}")
+                _FAIL_F4=$(print "${_REPORT_LINE}" | cut -f5 -d"${SEP}")
+
+                printf "| %-20s | %-14s | %-30s | %-s\n" \
+                    "${_FAIL_F1}" "${_FAIL_F4}" "${_FAIL_F2}" "${_FAIL_F3}"
+            done
+
+            printf "\n%-s\n" "SUMMARY: ${_FAIL_COUNT} failed HC event(s) found."
+        else
+            # print failed events (we may have multiple events for 1 FAIL ID)
+            _EVENT_COUNT=1
+            _DIR_PREFIX="$(expr substr ${ARG_FAIL_ID} 1 4)-$(expr substr ${ARG_FAIL_ID} 5 2)"
+            # no extended grep here!
+            grep ".*${SEP}.*${SEP}.*${SEP}.*${SEP}${_ID_NEEDLE}${SEP}" ${HC_LOG} 2>/dev/null |\
+                ${_SORT_CMD} | while read -r _REPORT_LINE
+            do
+                _FAIL_F1=$(print "${_REPORT_LINE}" | cut -f1 -d"${SEP}")
+                _FAIL_F2=$(print "${_REPORT_LINE}" | cut -f2 -d"${SEP}")
+                _FAIL_F3=$(print "${_REPORT_LINE}" | cut -f4 -d"${SEP}")
+
+                printf "%36sMSG #%03d%36s" "" ${_EVENT_COUNT} "" | tr ' ' -
+                printf "\nTime    : %-s\nHC      : %-s\nDetail  : %-s\n" \
+                    "${_FAIL_F1}" "${_FAIL_F2}" "${_FAIL_F3}"
+                _EVENT_COUNT=$(( _EVENT_COUNT + 1 ))
+            done
+
+            printf "%37sSTDOUT%37s\n" | tr ' ' -;
+            # display non-empty STDOUT file(s)
+            if [[ -n "$(du -a ${EVENTS_DIR}/${_DIR_PREFIX}/${ARG_FAIL_ID}/*.stdout.log 2>/dev/null | awk '$1*512 > 0 {print $2}')"  ]]
+            then
+                cat ${EVENTS_DIR}/${_DIR_PREFIX}/${ARG_FAIL_ID}/*.stdout.log
+            else
+                printf "%-s\n" "No STDOUT found"
+            fi
+
+            printf "%37sSTDERR%37s\n" | tr ' ' -;
+            # display non-empty STDERR file(s)
+            if [[ -n "$(du -a ${EVENTS_DIR}/${_DIR_PREFIX}/${ARG_FAIL_ID}/*.stderr.log 2>/dev/null | awk '$1*512 > 0 {print $2}')" ]]
+            then
+                cat ${EVENTS_DIR}/${_DIR_PREFIX}/${ARG_FAIL_ID}/*.stderr.log
+            else
+                printf "%-s\n" "No STDERR found"
+            fi
+
+            printf "%80s\n" | tr ' ' -
+        fi
+    else
+        printf "\n%-s\n" "SUMMARY: 0 failed HC events found."
+    fi
+fi
+
+return 0
+}
+
+#******************************************************************************
+# END of script
+#******************************************************************************