#!/bin/dash
#
# baculabackupreport.sh
#
# ------------------------------------------------------------------------------
#
# waa - 20130428 - Initial release.
#                  Generate basic Bacula backup report.
#
# waa - 20170501 - Change Log moved to bottom of script.
#
# ------------------------------------------------------------------------------
#
# Copyright (c) 2013-2017, William A. Arlofski waa-at-revpol-dot-com
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1.  Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2.  Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# ------------------------------------------------------------------------------


# System variables
# ----------------
server="bacula.domain.tld"
admin="root@localhost"
bcbin="/usr/local/sbin/bconsole"
sendmail="/usr/bin/sendmail"
bcconfig="/etc/bacula/bconsole.conf"
baculafd="" # Wildcard client(s) to report on - default is any/all clients
hist="24"   # Default history in hours if not specified on command line

# Database variables
# ------------------
dbtype="postgresql"							# Supported options are sqlite, mysql, pgsql, mariadb
# dbfile="/catalog/bacula.db" # Uncomment and set to Bacula sqlite3 catalog database file
db="bacula"									# Not needed for sqlite
dbuser="bacula"							# Not needed for sqlite
dbbin="/usr/local/bin/psql"
# dbpass="-p"				# Uncomment and set db password if one is used. Also not needed for sqlite

# Formatting variables
# --------------------
# Toggles and other miscellaneous settings
# ----------------------------------------
html="yes"										# Generate HTML emails instead of plain text emails?
boldjobname="yes"							# Bold the Job Name in HTML emails?
centerjobname="yes"						# Center the Job Name in HTML emails?
boldstatus="yes"							# Set <b> tag on Status field (only if html="yes")
colorstatusbg="yes"						# Colorize the Status cell's background? (only if html="yes")
printsummary="yes"						# Print a short summary after the job list table? (Total Jobs, Files & Bytes)
emailsummaries="no"						# Email all job summaries. Be careful with this, it can generate very large emails
emailbadlogs="no"							# Email logs of bad jobs or jobs with JobErrors -ne 0. Be careful, this can generate very large emails.
addsubjecticon="yes"					# Prepend the email Subject with UTF-8 icons (a 'checkmark', 'circle with slash', or a bold 'x')
addsubjectrunningorcreated="yes"			# Append "(## jobs still runnning/queued)" to Subject if > running or queued job > 0
nojobsicon="=?utf-8?Q?=E2=8A=98?="		# utf-8 subject icon when no jobs have been run
# goodjobsicon="=?utf-8?Q?=E2=9C=94?="	# utf-8 subject icon when all jobs were "OK"
# goodjobsicon="=?UTF-8?Q?=E2=9C=94=EF=B8=8F?="	# Alternate (better) utf-8 subject icon when all jobs were "OK"
goodjobsicon="=?UTF-8?Q?=E2=9C=85?="	# Alternate (better) utf-8 subject icon when all jobs were "OK"
badjobsicon="=?utf-8?Q?=E2=9C=96?="		# utf-8 subject icon when there are jobs with errors etc
starbadjobids="yes"						# Prepend an asterisk "*" to jobids of "bad" jobs
sortfield="JobId"							# Which catalog db field to sort on? Multiple,fields,work,here
sortorder="DESC"							# Which direction to sort?
emailtitle="Jobs run on ${server} in the past ${1} hours"	# This is prepended at the top of the email, before the jobs table

# HTML colors
# -----------
jobtableheadercolor="#b0b0b0"	# Background color for the HTML table's header
jobtablejobcolor="#f4f4f4"		# Background color for the job rows in the HTML table
runningjobcolor="#4d79ff"			# Background color of the Status cell for "Running" jobs
createdjobcolor="#add8e6"			# Background color of the Status cell for "Created, but not yet running" jobs
goodjobcolor="#00f000"				# Background color of the Status cell for "OK" jobs
# warnjobcolor="#ffff00"				# (yellow) background color of the Status cell for "OK" jobs (with warnings - well, actually with 'joberrors')
warnjobcolor="#ffc800"				# Alternate (orange) background color of the Status cell for "OK" jobs (with warnings - well, actually with 'joberrors')
badjobcolor="#cc3300"					# Background color of the Status cell for "bad" jobs
errorjobcolor="#cc3300"				# Background color of the Status cell for jobs with errors
goodjobwitherrcolor="#cccc00"	# Background color of the Status cell for "OK" jobs (with errors) - Not implemented due to request

# HTML fonts
# ----------
fontfamily="Verdana, Arial, Helvetica, sans-serif"	# Set the font family to use for HTML emails
fontsize="16px"								# Set the font size to use for email title and print summaries
fontsizejobinfo="12px"				# Set the font size to use for job information inside of table
fontsizesumlog="10px"					# Set the font size of bad logs and job summaries


# --------------------------------------------------
# Nothing should need to be modified below this line
# --------------------------------------------------


# Check to see if there are command line variables,
# and if there are, attempt to assign  them to
# baculafd, admin (email), and hist  variables
# -------------------------------------------------
if [ $# -eq 3 ]; then
	admin=${3}
	baculafd=${2}
	hist=${1}
	elif [ $# -eq 2 ]; then
	baculafd=${2}
	hist=${1}
	elif [ $# -eq 1 ]; then
	hist=${1}
fi


# A simple test to make sure
# the hist variable is a number
# -----------------------------
# Short and sweet, but a non-POSIX bashism :(
# if [ -z ${hist} ] || ! [[ ${hist} =~ ^[0-9]+$ ]]; then
# so, instead, we do a POSIX-compaitble version:
# -------------------------------------------------------
histisnum=$(echo ${hist} | grep -E '^[0-9]+$')
if [ -z ${hist} ] || [ -z ${histisnum} ]; then
	echo
	echo "USE: $0 [history in hours] [Bacula Client] [Admin Email]"
	echo
	exit 1
fi


# Test the rest of the binaries, and configuration files
# ------------------------------------------------------
if [ ! -e ${bcconfig} ]; then
	echo
	echo "The bconsole configuration file does not seem to be '${bcconfig}'."
	echo "Please check the setting for the variable 'bcconfig'."
	echo
	exit 1
fi

if [ ! -x ${bcbin} ]; then
	echo
	echo "The bconsole binary does not seem to be '${bcbin}', or it is not executable."
	echo "Please check the setting for the variable 'bcbin'."
	echo
	exit 1
fi

if [ ! -x ${sendmail} ]; then
	echo
	echo "The sendmail binary does not seem to be '${sendmail}', or it is not executable."
	echo "Please check the setting for the variable 'sendmail'."
	echo
	exit 1
fi

if [ ! -x ${dbbin} ]; then
	echo
	echo "The database client binary does not seem to be '${dbbin}', or it is not executable."
	echo "Please check the setting for the variable 'dbbin'."
	echo
	exit 1
fi


# Build query based on dbtype. Good thing we have "standards" Sigh...
# -------------------------------------------------------------------
case ${dbtype} in

	sqlite )
		# waa - 20171215 - sqlite3 query is a work in progress!
		# -----------------------------------------------------
		sqlitetries=4
		sqlitetrydelay=10
		while test ${sqlitetries} -gt 0; do
			queryresult=$(echo "SELECT JobId, REPLACE(Name,' ','_'), JobStatus, \
			JobErrors, Type, Level, JobFiles, JobBytes, StartTime, EndTime, \
			time((strftime('%s',EndTime) - strftime('%s',StartTime)), 'unixepoch') as RunTime \
			FROM Job WHERE (RealEndTime >= datetime('now', 'localtime', '-${hist} hours') OR (JobStatus='R' OR JobStatus='C')) \
			ORDER BY ${sortfield} ${sortorder};" \
			| ${dbbin} -noheader -column ${dbfile})

#			if [ ! -z $(echo ${queryresult} | grep "database is locked") ]; then
				let sqlitetries-=1
#				echo "Pfffff!  Someone using sqlite...."
#				exit 1
#			fi
		done
		;;

	mysql )
		queryresult=$(echo "SELECT JobId, REPLACE(Name,' ','_'), JobStatus, \
		JobErrors, Type, Level, JobFiles, JobBytes, StartTime, EndTime, \
		TIMEDIFF (EndTime,StartTime) as RunTime \
		FROM Job \
		WHERE (RealEndTime >= DATE_ADD(NOW(), INTERVAL -${hist} HOUR) OR (JobStatus='R' OR JobStatus='C')) \
		ORDER BY ${sortfield} ${sortorder};" \
		| ${dbbin} -u ${dbuser} ${dbpass} ${db} \
		| sed '/^JobId/d' )
		;;

	pgsql )
		queryresult=$(echo "SELECT Job.JobId, REPLACE(Job.Name,' ','_'), Job.JobStatus, \
		Job.JobErrors, Job.Type, Job.Level, Job.JobFiles, Job.JobBytes, Job.StartTime, Job.EndTime, \
		AGE(Job.EndTime, Job.StartTime) as RunTime, Client.Name \
		FROM Job \
		INNER JOIN Client on Job.ClientID=Client.ClientID \
		WHERE (Job.RealEndTime >= CURRENT_TIMESTAMP(2) - cast('${hist} HOUR' as INTERVAL) OR (Job.JobStatus='R' OR Job.JobStatus='C')) \
		AND Client.Name LIKE '%${baculafd}%' \
		ORDER BY Job.${sortfield} ${sortorder};" \
		| ${dbbin} -U ${dbuser} ${dbpass} ${db} -0t \
		| sed -e 's/|//g' -e '/^$/d' )
		;;

	mariadb )
		queryresult=$(echo "SELECT JobId, REPLACE(Name,' ','_'), JobStatus, \
		JobErrors, Type, Level, JobFiles, JobBytes, StartTime, EndTime, \
		TIMEDIFF (EndTime,StartTime) as RunTime \
		FROM Job \
		WHERE (RealEndTime >= DATE_ADD(NOW(), INTERVAL -${hist} HOUR) OR (JobStatus='R' OR JobStatus='C')) \
		ORDER BY ${sortfield} ${sortorder};" \
		| ${dbbin} -u ${dbuser} -p${dbpass} ${db} -s -N )
		;;

	* )
		echo "dbtype of '${dbtype}' is invalid. Please set dbtype variable to 'sqlite', 'mysql', 'pgsql', or 'mariadb'"
		exit 1
		;;
esac


# Let's check for Running or "Created, but not yet running" jobs
# to append a nice comment to end of email Subject
# --------------------------------------------------------------
runningorcreated=$(echo "${queryresult}" | awk '{print $3}' | grep -c "R\|C")


# If we have no jobs to report on, then
# we need to skip the entire awk script
# and some bash stuff and jump all the
# way to about line 851
# -------------------------------------
if [ -z "${queryresult}" ]; then
	results="0"
	else
		results="1"
		totaljobs=$(echo "${queryresult}" | wc -l)


# Since exit codes can only be 0-255, we can not exit the awk script
# with the number of Jobs with errors as it will not be correct once
# the number of bad jobs is greater than 255. Instead we will write
# this number to a temp file, then read it from bash and delete it
# once the awk script exits.
# ------------------------------------------------------------------
awkerrfile=$(mktemp -t XXXXXX) 


# Now for some fun with awk
# -------------------------
msg=$(echo "${queryresult}" | \
LC_ALL=en_US.UTF-8 \
gawk \
-v html="${html}" \
-v awkerrfile="${awkerrfile}" \
-v totaljobs="${totaljobs}" \
-v boldjobname="${boldjobname}" \
-v centerjobname="${centerjobname}" \
-v boldstatus="${boldstatus}" \
-v colorstatusbg="${colorstatusbg}" \
-v jobtableheadercolor="${jobtableheadercolor}" \
-v jobtablejobcolor="${jobtablejobcolor}" \
-v runningjobcolor="${runningjobcolor}" \
-v createdjobcolor="${createdjobcolor}" \
-v goodjobcolor="${goodjobcolor}" \
-v errorjobcolor="${errorjobcolor}" \
-v goodjobwitherrcolor="${goodjobwitherrcolor}" \
-v warnjobcolor="${warnjobcolor}" \
-v badjobcolor="${badjobcolor}" \
-v printsummary="${printsummary}" \
-v starbadjobids="${starbadjobids}" \
'BEGIN {awkerr = 0}
{star = " "}

						# List of possible jobstatus codes
						# --------------------------------
						# Enter SQL query: SELECT * FROM status;
						# +-----------+---------------------------------+----------+
						# | jobstatus | jobstatuslong                   | severity |
						# +-----------+---------------------------------+----------+
						# | C         | Created, not yet running        |       15 |
						# | R         | Running                         |       15 |
						# | B         | Blocked                         |       15 |
						# | T         | Completed successfully          |       10 |
						# | E         | Terminated with errors          |       25 |
						# | e         | Non-fatal error                 |       20 |
						# | f         | Fatal error                     |      100 |
						# | D         | Verify found differences        |       15 |
						# | A         | Canceled by user                |       90 |
						# | F         | Waiting for Client              |       15 |
						# | S         | Waiting for Storage daemon      |       15 |
						# | m         | Waiting for new media           |          |
						# | M         | Waiting for media mount         |       15 |
						# | s         | Waiting for storage resource    |       15 |
						# | j         | Waiting for job resource        |       15 |
						# | c         | Waiting for client resource     |       15 |
						# | d         | Waiting on maximum jobs         |       15 |
						# | t         | Waiting on start time           |       15 |
						# | p         | Waiting on higher priority jobs |       15 |
						# | a         | SD despooling attributes        |       15 |
						# | i         | Doing batch insert file records |       15 |
						# | I         | Incomplete Job                  |       25 |
						# +-----------+---------------------------------+----------+


# If an ADMIN type job does not have a "Level" set anywhere in the Job, or
# overriden in a schedule, or specified on the command line it will have a NULL
# "Level". This shifts the columns we receive from the SQL query left
# by one, starting at the sixth column. Here we move everything right by one to
# mitigate this issue.
# This does not cover failed Control/Migration jobs with no start time due to
# the SQl query returning 0 jobs to copy or migrate. See below.
# -----------------------------------------------------------------------------
{ if ($3 != "R" && $5 != "c" && $5 != "m" && NF != 14)
	{
		$14 = $13
		$13 = $12
		$12 = $11
		$11 = $10
		$10 = $9
		$9  = $8
		$8  = $7
		$7  = $6
	}
}

# If the job is running, and it is an ADMIN type job, and the fifth
# field is not "Full", "Incr", or "Diff" then there was no level
# set for this ADMIN job, and the "Level" field will be NULL and we
# will have a fewer fields to shift right.
# -----------------------------------------------------------------
{ if ($3 == "R" && $5 == "D" && $6 !~ /[FID]/)
	{
		$14 = $13
		$13 = $12
		$12 = $11
		$11 = $10
		$10 = $9
		$9  = $8
		$8  = $7
	}
}

# If the job is still running, there is no EndTime, so we shift the
# RunTime two fields to the right and manually set the Endtime
# Additionally, we need to set shift the client name from $11 to $14
# and then we need to set $11 to the "Still Running" string
# ------------------------------------------------------------------
{ if ($3 == "R")
	{
		$14 = $11
		if (html == "yes")
			{
				$11 = "Still Running"
				$13 = "<hr width=\"50%\">"
			} else {
				$11 = "  =Still"
				$12 = "Running= "
				$13 = "--------"
			}
	}
}

# If the job is Created but not yet running, there is no EndTime,
# so we shift the RunTime two fields to the right and manually set
# the Endtime
# -----------------------------------------------------------------
{ if ($3 == "C")
	{
		if (html == "yes")
			{
				$9 = "Created, not yet running"
				$11 = "<hr width=\"55%\">"
				$13 = "<hr width=\"50%\">"
			} else {
				$9 = "=Created="
				$10 = ""
				$11 = "----------"
				$12 = "--------"
				$13 = "--------"
			}
	}
}

# If the job is a failed copy control job, and there is no StartTime
# and no RunTime, these last three fields are shifted two to the left
# -------------------------------------------------------------------
{ if ($3 == "f" && $5 ~ /[cm]/ && $9 == "")
	{
		if (html == "yes")
			{
				$9 = "Never Started"
				$10 = ""
				$11 = "<hr width=\"55%\">"
				$13 = "<hr width=\"50%\">"
				} else {
					$12 = $10
					$11 = $9
					$13 = "--------"
					$9 = "      Never"
					$10 = "Started "
			}
	}
}

# Assign words to job status code characters
# ------------------------------------------
# First, check to see if we need to generate an HTML email
{ if (html == "yes")
	{
			# Set default opening and closing tags for status cell
			# ----------------------------------------------------
			tdo = "<td align=\"center\">"
			tdc = "</td>"

			# Check to see if the job is "OK" then assign
			# the "goodjobcolor" to the cell background
			# -------------------------------------------
			if ($3 ~ /[T]/ && $4 == 0)
				{
					if (colorstatusbg == "yes")
						# Assign jobs that are OK or Running the goodjobcolor
						# ---------------------------------------------------
						{
							tdo = "<td align=\"center\" bgcolor=\"" goodjobcolor "\">"
						}

				# Should the status be bolded?
				# ----------------------------
				if (boldstatus == "yes")
					{
						tdo=tdo"<b>"
						tdc="</b>"tdc
					}
				status["T"]=tdo"-OK-"tdc

				# If it is a good job, but with errors or warnings
				# then we will assign the warnjobcolor
				# ------------------------------------------------
				} else if ($3 == "T" && $4 != 0)
					{
						if (colorstatusbg == "yes")
							# Assign OK jobs with errors the warnjobcolor
							# -------------------------------------------
							{
								tdo = "<td align=\"center\" bgcolor=\"" warnjobcolor "\">"
							}

						# Should the status be bolded?
						# ----------------------------
						if (boldstatus == "yes")
							{
								tdo=tdo"<b>"
								tdc="</b>"tdc
							}
						# Since the "W" jobstatus never appears in the DB, we manually
						# assign it here so it can be recognized later on in the script
						# -------------------------------------------------------------
						$3 = "W"
						status["W"]=tdo"OK/Warnings"tdc

				# If the job is still running we will
				# assign it the runningjobcolor
				# -----------------------------------
				} else if ($3 == "R")
					{
						if (colorstatusbg == "yes")
							# Assign running jobs the runningjobcolor
							# ---------------------------------------
							{
								tdo = "<td align=\"center\" bgcolor=\"" runningjobcolor "\">"
							}

						# Should the status be bolded?
						# ----------------------------
						if (boldstatus == "yes")
							{
								tdo=tdo"<b>"
								tdc="</b>"tdc
							}
						status["R"]=tdo"Running"tdc

				# If the job is created but not yet
				# running we will assign it the
				# createdjobcolor
				# ---------------------------------
				} else if ($3 == "C")
					{
						if (colorstatusbg == "yes")
							# Assign running jobs the createdjobcolor
							# ---------------------------------------
							{
								tdo = "<td align=\"center\" bgcolor=\"" createdjobcolor "\">"
							}

						# Should the status be bolded?
						# ----------------------------
						if (boldstatus == "yes")
							{
								tdo=tdo"<b>"
								tdc="</b>"tdc
							}
						status["C"]=tdo"Created"tdc

				# If it is a bad job, then
				# we assign the badjobcolor
				# -------------------------
				} else if ($3 ~ /[ABDef]/)
					{
						if (colorstatusbg == "yes")
							# Assign bad jobs the badjobcolor
							# -------------------------------
							{
								tdo = "<td align=\"center\" bgcolor=\"" badjobcolor "\">"
							}

						# Should the status be bolded?
						# ----------------------------
						if (boldstatus == "yes")
							{
								tdo=tdo"<b>"
								tdc="</b>"tdc
							}
						status["A"]=tdo"Aborted"tdc
						status["D"]=tdo"Verify Diffs"tdc
						status["f"]=tdo"Failed"tdc

				# If the job terminated "E", assign the job the errorjobcolor
				# -----------------------------------------------------------
				} else if ($3 == "E")
					{
						if (colorstatusbg == "yes")
							# Assign job the errorjobcolor
							# ----------------------------
							{
								tdo = "<td align=\"center\" bgcolor=\"" errorjobcolor "\">"
							}

						# Should the status be bolded?
						# ----------------------------
						if (boldstatus == "yes")
							{
								tdo=tdo"<b>"
								tdc="</b>"tdc
							}
						status["E"]=tdo"Errors"tdc
					

				# If the job terminated "I" (Incomplete), assign the job the warnjobcolor
				# -----------------------------------------------------------------------
				} else if ($3 == "I")
					{
						if (colorstatusbg == "yes")
							# Assign job the warnjobcolor
							# ---------------------------
							{
								tdo = "<td align=\"center\" bgcolor=\"" warnjobcolor "\">"
							}

						# Should the status be bolded?
						# ----------------------------
						if (boldstatus == "yes")
							{
								tdo=tdo"<b>"
								tdc="</b>"tdc
							}
						status["I"]=tdo"Incomplete"tdc
					}
		} else
		# $html is not "yes" so statuses will be normal text
		# --------------------------------------------------
			{
				status["A"]="     Aborted   "
				status["C"]="     Created   "
				status["D"]="  Verify Diffs "
				status["E"]="     Errors    "
				status["f"]="     Failed    "
				status["I"]="   Incomplete  "
				status["R"]="     Running   "
				status["T"]="      -OK-     "
				# Since the "W" jobstatus never appears in the DB, we manually
				# assign it here so it can be recognized later on in the script
				# -------------------------------------------------------------
				if ($3 == "T" && $4 != 0)
					{ $3 = "W"
						status["W"]="   OK/Warnings "
					}
			}
}


# These status characters seem to only
# be Director "in memory" statuses. They
# do not get entered into the DB ever so we
# cannot catch them with the db query we use
# I might have to query the DIR as well as
# the DB to be able to capture these
# ------------------------------------------
{
	status["B"]="     Blocked   "
	status["F"]="     Wait FD   "
	status["S"]="     Wait SD   "
	status["m"]=" Wait New Media"
	status["M"]="   Wait Mount  "
	status["s"]="   Wait Storage"
	status["j"]="    Wait Job   "
	status["c"]="   Wait Client "
	status["d"]="  Wait Max Jobs"
	status["t"]="Wait Start Time"
	status["p"]="  Wait Priority"
	status["a"]="  Despool Attrs"
	status["i"]="  Batch Insert "
	status["L"]="Spool Last Data"
}


# Assign words to job type code characters
# ----------------------------------------
{
	jobtype["D"]="Admin"
	jobtype["B"]="Backup"
	jobtype["C"]="Copy"
	jobtype["c"]="Control"
	jobtype["R"]="Restore"
	jobtype["V"]="Verify"
	jobtype["M"]="Migrated"
	jobtype["m"]="Migration"
}


# Assign words to job level code characters
# -----------------------------------------
{
	level["F"]="Full"
	level["I"]="Incr"
	level["D"]="Diff"
	level["f"]="VFul"
	level["-"]="Base"
	if (html == "yes")
		{
			level["-"]="<hr width=\"50%\">"
		} else {
			level["-"]="----"
		}
}


# Assign words to Verify job level code characters
# ------------------------------------------------
{
	level["A"]="VVol"
	level["C"]="VCat"
	level["V"]="Init"
	level["O"]="VV2C"
	level["d"]="VD2C"
}


# Check to see if the job did not "T"erminate OK then increment $awkerr,
# and prepend the JobId with an asterisk for quick visual identification
# of problem jobs.
# -------------------------------------------------------------------------
{ if ($3 ~ /[ABDEIef]/) { awkerr++; if (starbadjobids == "yes") { star  = "*" } } }


# If the job is an Admin, Control,
# or Migration job it will have
# no real "Level", so we set it to "----"
# Also need to set it for Client ($14) on
# Admin jobs
# -------------------------------------------
{ if ($5 ~ /[cDm]/)
	{ $6 = "-"
	 if (html == "yes")
		{
			$14 = "<hr width=\"50%\">"
		} else {
			$14 = "-"
		}
	}
}

# Print out each job, formatted with the following fields:
# JobId Name Status Errors Type Level Files Bytes StartTime EndTime RunTime
# -------------------------------------------------------------------------
{ if (html == "yes")
	{
		# Check to see if we center or bold the JobName
		# ---------------------------------------------
		{ if (boldjobname == "yes" && centerjobname == "yes")
			{ $2 = "<td align=\"center\"><b>"$2"</b></td>" }
			else if (boldjobname == "yes")
				{ $2 = "<td><b>"$2"</b></td>" }
				else if (centerjobname == "yes")
					{ $2 = "<td align=\"center\">"$2"</td>" }
						else { $2 = "<td>"$2"</td>" }
		}

		printf("<tr bgcolor=\"%s\"> \
		<td align=\"center\">%s%s%s</td> \
		%s \
		<td align=\"center\">%s</td> \
		%s \
		<td align=\"right\">%'"'"'d</td> \
		<td align=\"center\">%s</td> \
		<td align=\"center\">%s</td> \
		<td align=\"right\">%'"'"'d</td> \
		<td align=\"right\">%'"'"'9.2f GB</td> \
		<td align=\"center\">%s %s</td> \
		<td align=\"center\">%s %s</td> \
		<td align=\"center\">%s</td> \
		</tr>\n", \
		jobtablejobcolor, star, $1, star, $2, $14, status[$3], $4, jobtype[$5], level[$6], $7, $8/(1024*1024*1024), $9, $10, $11, $12, $13);
	} else
		{ printf("%s %-9s %-10s     %-14s %16s %'"'"'12d %8s %6s %'"'"'9d %'"'"'9.2f GB %11s %-9s %-10s %-9s %-9s\n", \
			star, $1, $2, $14, status[$3], $4, jobtype[$5], level[$6], $7, $8/(1024*1024*1024), $9, $10, $11, $12, $13);
		}
}


# Increment the job counter
# -------------------------
{ totaljobs++ }


# Count the files and bytes from all Backup jobs
# ----------------------------------------------
{ if ($5 == "B")
		{
			backupfiles += $7
			backupbytes += $8
		}
}


# Count the files and bytes from all
# Restore, Copy or Migration jobs. Not
# sure if I like this idea yet. Sticking
# to only counting Restore jobs for now.
# --------------------------------------
# { if ($5 == "R" || $5 == "C" || $5 == "M")
{ if ($5 == "R")
		{
			restorefiles += $7
			restorebytes += $8
		}
}


END {
# Finally, print out the summaries
# --------------------------------
if (printsummary == "yes")
	{ if (html == "yes")
		{
			printf("</table>")
			printf("<br>\
			<hr align=\"left\" width=\"25%\">\
			<table width=\"25%\">\
				<tr><td><b>Total Jobs</b></td><td align=\"center\"><b>:</b></td> <td align=\"right\"><b>%'"'"'15d</b></td></tr>\
				<tr><td><b>Bad Jobs</b></td><td align=\"center\"><b>:</b></td> <td align=\"right\"><b>%'"'"'15d</b></td></tr>\
				<tr><td><b>Total Backup Files</b></td><td align=\"center\"><b>:</b></td> <td align=\"right\"><b>%'"'"'15d</b></td></tr>\
				<tr><td><b>Total Backup Bytes</b></td><td align=\"center\"><b>:</b></td> <td align=\"right\"><b>%'"'"'15.2f GB</b></td></tr>\
				<tr><td><b>Total Restore Files</b></td><td align=\"center\"><b>:</b></td> <td align=\"right\"><b>%'"'"'15d</b></td></tr>\
				<tr><td><b>Total Restore Bytes</b></td><td align=\"center\"><b>:</b></td> <td align=\"right\"><b>%'"'"'15.2f GB</b></td></tr>\
			</table>\
			<hr align=\"left\" width=\"25%\">",\
			totaljobs, awkerr, backupfiles, backupbytes/(1024*1024*1024), restorefiles, restorebytes/(1024*1024*1024));
		} else
printf("\
  =================================\n\
  Total Jobs  : %'"'"'19d\n\
  Bad Jobs    : %'"'"'19d\n\
  Total Backup Files : %'"'"'12d\n\
  Total Backup Bytes : %'"'"'9.2f GB\n\
  Total Restore Files : %'"'"'11d\n\
  Total Restore Bytes : %'"'"'8.2f GB\n\
  =================================\n",\
totaljobs, awkerr, backupfiles, backupbytes/(1024*1024*1024), restorefiles, restorebytes/(1024*1024*1024));
print awkerr > awkerrfile;
} exit }
')


# Any failed jobs, or non "OK" jobs with errors?
# ----------------------------------------------
numbadjobs=$(cat $awkerrfile)
rm -f $awkerrfile


# Do we email the job summaries?
# ------------------------------
if [ ${emailsummaries} = "yes" ]; then
	# Get all of the jobids from the query results, but
	# skip any running or "created, but not yet running"
	# jobs because they will not have a summary in the
	# catalog DB until the job has terminated.
	# --------------------------------------------------
	alljobids=$(echo "${queryresult}" \
	| awk '{ if ($3 !~ /[RC]/) printf("%s ", $1) }')


	# If no jobids were returned, skip creating
	# the header and looping through zero records
	# -------------------------------------------
	if [ ! -z "${alljobids}" ]; then
		# Generate the header
		# -------------------
		msg="${msg}"$(
		if [ ${html} = "yes" ]; then
			echo "<pre>===================================="
				else
					echo
					echo
					echo
					echo "===================================="
		fi
			echo "Job Summaries of All Terminated Jobs"
			echo "===================================="
		)


		# Get the job logs from all jobs and just awk for the summary
		# -----------------------------------------------------------
		for jobid in ${alljobids}; do
			msg="${msg}"$(
				echo
				echo "------------"
				echo "JobId: ${jobid}"
				echo "------------"
				echo "llist joblog jobid=${jobid}" | ${bcbin} -c ${bcconfig} \
							| sed 's/^  //' \
							| awk "BEGIN {IGNORECASE=0} /^Build OS:.*/ || /^Build:.*/,/^Termination:/ {print}"
				echo "======================================================================"
			)
		done
		if [ ${html} = "yes" ]; then
			msg=${msg}$(echo "</pre>")
		fi
	fi
fi


# Do we email the bad job logs with the report?
# ---------------------------------------------
if [ ${emailbadlogs} = "yes" ]; then
	# Get the badjobs, or the good jobs with
	# JobErrors != 0 from the query results
	# --------------------------------------
	badjobids=$(echo "${queryresult}" \
	| awk '{ if ($3 ~ /[ABDEIef]/ || ($3 == "T" && $4 != 0)) printf("%s ", $1) }')


	# If no jobids were returned, skip creating
	# the header and looping through zero records
	# -------------------------------------------
	if [ ! -z "${badjobids}" ]; then
		# Generate the header
		# -------------------
		msg="${msg}"$(
		if [ ${html} = "yes" ]; then
			echo "<pre>===================================================="
				else
					echo
					echo
					echo
					echo "===================================================="
		fi
			echo "Job logs of failed jobs, or good jobs with JobErrors"
			echo "===================================================="
		)


		# Get the bad job logs from the Director via bconsole
		# ---------------------------------------------------
		for jobid in ${badjobids}; do
			msg="${msg}"$(
				echo
				echo "------------"
				echo "JobId: ${jobid}"
				echo "------------"
				echo "llist joblog jobid=${jobid}" | ${bcbin} -c ${bcconfig}
				echo "======================================================================"
			)
		done
		if [ ${html} = "yes" ]; then
			msg=${msg}$(echo "</pre>")
		fi
	fi
fi


# Prepend the header to the $msg output
# -------------------------------------
if [ ${html} = "yes" ]; then
	msg="<html>
	<head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">
		<style>
			body {font-family:$fontfamily; font-size:$fontsize;} td {font-size:$fontsizejobinfo;} pre {font-size:$fontsizesumlog;}
		</style>
	</head>
	<body>
		<p><u><b>${totaljobs} ${emailtitle}</b></u></p>
		<table width=\"98%\" align=\"center\" border=\"1\" cellpadding=\"2\" cellspacing=\"0\">
			<tr bgcolor=\"${jobtableheadercolor}\">
			<td align=\"center\"><b>Job ID</b></td>
			<td align=\"center\"><b>Job Name</b></td>
			<td align=\"center\"><b>Client Name</b></td>
			<td align=\"center\"><b>Status</b></td>
			<td align=\"center\"><b>Errors</b></td>
			<td align=\"center\"><b>Type</b></td>
			<td align=\"center\"><b>Level</b></td>
			<td align=\"center\"><b>Files</b></td>
			<td align=\"center\"><b>Bytes</b></td>
			<td align=\"center\"><b>Start Time</b></td>
			<td align=\"center\"><b>End Time</b></td>
			<td align=\"center\"><b>Run Time</b></td>
		</tr>
${msg}
</body></html>"
	else
		msg="
  ${totaljobs} ${emailtitle}
  -------------------------------------------------

  JobId     Job Name        Client Name         Status         Errors     Type    Level   Files       Bytes         Start Time            End Time        Run Time
  -----   -------------    --------------   ---------------  ----------  -------  -----  --------  -----------  -------------------  -------------------  --------
${msg}"
fi

fi


# If there were zero results returned from the
# SQL the query, we skip the entire awk script,
# and a lot of other bash stuff that generates
# the email body and we end up here.
# ---------------------------------------------
if [ ${results} -eq 0 ]; then
	status="No Jobs Have Been Run"
	subjecticon="${nojobsicon}"
	msg="These are not the droids you are looking for..."
	else
		# Totally unnecessary, but, well...  OCD... :)
		# --------------------------------------------
		if [ ${numbadjobs} -ne 0 ]; then
			if [ ${numbadjobs} -eq 1 ]; then
				job="Job"
				else
					job="Jobs"
			fi
			status="(${numbadjobs}) ${job} with Errors"
			subjecticon="${badjobsicon}"
			else
				if [ ${totaljobs} -eq 1 ]; then
					job="Job"
					else
						job="Jobs"
				fi
				status="All ${totaljobs} ${job} OK"
				subjecticon="${goodjobsicon}"
		fi
fi


# More silliness
# --------------
if [ ${hist} -eq 1 ]; then
	hour="Hour"
	else
		hour="Hours"
fi


# Touch up the Running or created message to append to Subject
# ------------------------------------------------------------
if [ ${addsubjectrunningorcreated} = "yes" ]; then
	if [ ${runningorcreated} -ne 0 ]; then
		if [ ${runningorcreated} -eq 1 ]; then
			job="job"
			else
				job="jobs"
		fi
		runningorcreatedsubject="(${runningorcreated} ${job} still queued/running)"
		else
			runningorcreatedsubject=""
	fi
fi


# Email the report
# ----------------
(
echo "To: ${admin}"
echo "From: ${server} <${admin}>"
if [ ${addsubjecticon} = "yes" ]; then
	echo "Subject: ${subjecticon} ${server} - ${status} in the Past ${hist} ${hour} ${runningorcreatedsubject}"
	else
		echo "Subject: ${server} - ${status} in the Past ${hist} ${hour} ${runningorcreatedsubject}"
fi
if [ ${html} = "yes" ]  &&  [ ${results} -ne 0 ]; then
 echo "Content-Type: text/html"
 echo "MIME-Version: 1.0"
fi
echo
echo "${msg}"
) | ${sendmail} -t


# -------------
# End of script
# -------------


# ----------
# Change Log
# ----------
# ----------------------------
# William A. Arlofski
# Reverse Polarity, LLC
# helpdesk@revpol.com
# http://www.revpol.com/bacula
# ----------------------------
#
#
# 20130428 - Initial release
#            Generate and email a basic Bacula backup report
#            1st command line parameter is expected to be a
#            number of hours. No real error checking is done
#
# 20131224 - Removed "AND JobStatus='T'" to get all backup jobs
#            whether running, or completed with errors etc.
#          - Added Several fields "StartTime", "EndTime",
#            "JobFiles"
#          - Removed "JobType" because we are only selecting
#            jobs of type "Backup" (AND Type='B')
#          - Modified header lines and printf lines for better
#            formatting
#
# 20140107 - Modified script to include more information and cleaned
#            up the output formatting
#
# 20150704 - Added ability to work with MySQL or Postgresql
#
# 20150723 - Modified query, removed "Type='B'" clause to catch all jobs,
#            including Copy jobs, Admin jobs etc. Modified header, and
#            output string to match new query and include job's "Type"
#            column.
#
# 20170225 - Rewrote awk script so that a status/summary could be set in
#            the email report's subject. eg:
#            Subject: "serverName - All Jobs OK in the past x hours"
#            Subject: "serverName - x Jobs FAILED in the past y hours"
#
# 20170303 - Fixed output in cases where there are jobs running and there
#            is no "Stop Time" for a job.
#
# 20170406 - Some major modifications:
#             - Added feature to spell out words instead of using the
#               single character codes for Job type, Job Status, and
#               Job Level - Including the different levels for Verify
#               jobs
#             - If a job terminates with an error or warning, then the
#               job's line in the output is prepended with an asterisk
#               "*" for quick visual identification
#             - Modified the outputs of the files and bytes fields to
#               include commas when the number is > 999
#             - Added totals to the end of the report for Jobs, Files,
#               and Bytes
#             - Added $sortfield and $sortorder variables to allow output
#               to be sorted as desired
#             - Set the level of a job to "----" when the level is not
#               applicable as in Restore jobs, Admin jobs etc.
#
# 20170408 - Some minor cleanup, and moving things around
#          - Added $emailsummaries variable to append the job summaries
#            to the end of the report.
#          - Added $emailbadlogs variable to append full joblogs of jobs
#            which have failed or jobs with errors to the end of the report
#            for quick access to investigate failed jobs.
#
# 20170417 - Added some tests for binaries and the bconsole config file
#
# 20170429 - Thanks to Chris Couture for contacting me and submitting a
#            working query for MariaDB. I have added 'mariadb' as a new
#            dbtype option.
#          - Thanks to Chris Couture for the ideas and some code examples
#            to create an HTML email.
#              - Added $html variable to enable HTML emails.
#              - Added $boldstatus variable to make the Status <b>bold</b>
#                in HTML emails.
#              - Added $colorstatusbg variable to color the background of
#                the Status cell in HTML emails.
#          - Thanks to Chris Couture for the idea of adding RunTime
#            to the email output.
#          - Thanks to Chris Couture for the idea of using some unicode
#            characters (a 'checkmark' or a bold 'x') in the Subject:
#            to quickly see if everything ran OK.
#              - Added $addsubjecticon variable to enable/disable the
#                prepending of this icon to the Subject.
#          - Added $printsumary variable to give the option to print the
#            total Jobs, Files, and Bytes after the job listing table.
#          - Added $starbadjobids variable to enable/disable prepending
#            the bad jobids with an asterisk "*".
#          - Modified the way the email is built at the end. Thanks to
#            Chris Courture again for this nice idea.
#          - Added $jobtableheadercolor, $jobtablejobcolor, $goodjobcolor,
#            $goodjobwitherrcolor, $runningjobcolor, $warnjobcolor, and
#            $badjobcolor variables to colorize HTML emails.
#          - Added $emailtitle variable for the title at the top.
#          - Added $fontfamily, $fontsize, $fontsizejobinfo, and $fontsizesumlog
#            variables to allow styling of the HTML output. (Thanks again Chris)
#          - Added $nojobsicon, $goodjobsicon, and $badjobsicon variables to
#            allow setting the prepended utf-8 subject icon character.
#          - Reformatted things so that if there are no jobs returned by the
#            SQL query, the email message sent is nice and short.
#          - Modified the license to allow for inclusion into Bacula Community,
#            and possibly the Enterprise Edition releases.
#
# 20170430 - Modified the order of the fields to make more sense.
#          - Re-aligned the text email so that when an asterisk is pre-pended it
#            does not shift the whole line.
#
# 20170508 - Re-worked some of the logic so that good jobs (JobStatus="T") which
#            have errors will have their status listed as "OK/Warnings", and it
#            will not trigger as a "bad job" on the JobErrors, so it will not
#            have an asterisk prepended to the JobId in the job listing. I think
#            this fix is more of a temporary change in the hopes that a "W"
#            status to represent "good jobs with warnings" is implemented in the
#            db in the future.
#          - Added an "Errors" column to the table to show "JobErrors" from the
#            db.
#          - Some minor variable name changes and other minor changes.
#
# 20170511 - Minor adjustments to the alignment formatting of the text email.
#          - Minor 'case' changes to a couple levels (Init & VCat).
#
# 20170517 - Rewrote the SQL queries so that the order of the fields returned
#            matched the order of the colums in the email output. This allowed
#            easier fixing of a strange issue where Bacula Enterprise does not
#            set a "Level" for ADMIN type jobs that are started by the scheduler.
#            This also forced modifications to all of the printf() output order,
#            and allowed for easier shifting of variables in situations where
#            some columns were null, causing the rest of the columns to shift
#            left.
#
# 20170922 - Added a check for "Created, not yet running" jobs. (JobStatus="C").
#            Added $createdjobcolor variable to display using their own color.
#            Set any blank fields to "----"
#
# 20170926 - Fixed several bugs including a missing "Incomplete" status, and
#            some incorrect placement of curly brackets in the awk script.
#          - Added missing JobTypes for Copy/Control, Migration/Migrated.
#          - Used "<hr>" in html output for blank fields.
#          - Fixed an issue where a copy or migration job fails but does not
#            have a start time. It has an end time, so this shifts everything
#            two fields to the left. This is caused when the copy or the
#            migration job is started, but the selection of jobs to copy/migrate
#            returns an errror. (e.g.: the SQL Selection query returns an error)
#
# 20171002 - Fixed alignment problems with the text version of the report. Made
#            modifications to some of the wording and filled blank fields with
#            "-" characters.
#          - Fixed the job summary output so that it properly prints all summary
#            lines from the "Build OS:" line until the "Termination:" line and
#            nothing else. For example, Verify job summaries are shorter than
#            backup job summaries, and Copy/Migration jobs summaries are longer.
#          - Added the number of "Bad Jobs", "Total Restore Files" and
#            "Total Restore Bytes" to the Job Summary section.
#
# 20171005 - Added small change that someone submitted to my website which
#            replaces any spaces in Job names with underscores. This keeps all of
#            the columns in order for awk to work with. Thank you anonymous. :)
#
# 20171007 - Minor fix for failed Copy control jobs output (check for null $9)
#
# 20171201 - Removed all 'bashisms' to make this script as POSIX compliant as
#            possible.
#          - So, no more '[[ some test ]]' and no more 'echo -e'
#          - Tested with 'dash' which is a strictly POSIX compliant shell,
#            so this should be very portable now.
#          - Minor reorganization of the variables at the top.
#
# 20171202 - Added $boldjobname and $centerjobname variables to allow the jobname
#            in HTML emails to be bolded and/or centered.
#
# 20171215 - Began to implement sqlite3 support
#
# 20200811 - Set 24 hours as a default number of hist hours. Now, no command line
#            parameters are required.
#          - Added 'hist', 'baculafd', and 'admin' variable command line overrides.
#          - Added the ability to report on one Client or multiple Clients based
#            on second command line parameter and a SQL "LIKE" query.
#          - Now, one (hist), two (hist Client), or three (hist client admin_email)
#            command line parameters may be supplied. If you want to specify admin
#            email on command line *and* you want to report on all clients, then
#            just set the second parameter to "".
#          - Keep in mind: Admin, Copy Control, and Migration jobs do not have a
#            a Client. When filtering by Client, these jobs cannot be included in
#            the report.
#
# 20200820 - Write the number of Jobs with errors (awkerr varable) to a temporary
#            file instead of exiting the main awk script with the awkerr variable.
#            This solves the issue of "wrap around" when this number exceeds 255
#            because POSIX exit result codes are only 0-255 (8-bit number)
#          - Added the total number of jobs to the "All Jobs OK..." email subject
#          - Added the total number of jobs to the top of the email body
#
# 20200925 - Swap the Job Name and Client columns so that the Job Name is first
# ---------------------------------------------------------------------------------


# I like small tabs. Use :set list in vim to see tabbing etc
# vim: tabstop=2 softtabstop=2 shiftwidth=2
