#!/bin/sh
# PURPOSE:   	call QC procedure for pipeline products
# AUTHOR:    	Reinhard Hanuschik / ESO
# DATE:      	September 2006
# VERSIONS:   	1.0 -- created from measureQuality -a (v1.1), aligned with processAB call 
#		1.0.1- implement workaround to stabilize MIDAS sessions (2006-12-05)
#		1.0.2- workaround changed (2008-03-14)
#		1.0.3- workaround switched off (2008-03-18)
#		1.1 -- measures execution time as TQCEXEC (configurable) (2009-08-04)
#		1.1.1- drop TQCEXEC config key (always YES) (2009-08-20)
#		1.1.2- fix bug with time_QC_$$ (2009-10-14)
#		1.1.3- calls funpack if tile-compressed files found (2009-12-21)
#		1.1.4- added line :46 as required by ScientificLinux 5.3 (2010-07-30)
#		1.2  - enabled for QC coversheet; scoreQC now called here (2011-05-31)
#		1.3 -- support for AB tab files (2013-01-09)
#		1.3.1- bug fix in line :285 (2013-01-11)
#		1.3.2- bug fix related to AMODE=TECHNICAL or TEST (2013-01-15)
#		1.3.3- calls the same FINAL_PLUGIN as processAB (2013-01-24)
#		1.3.4- bug with HC_MARK fixed (2013-04-08)
#		1.3.5- no incremental AB monitor (2013-04-26)
#		1.4 -- ACQUISITIONS included in CALIB mode (line :279) [BWo] (2013-07-31)
#		1.4.1- sets CALL_ENABLED=YES for condor environment (parallel calls of scoreQC) (2013-10-28) 
#		1.5 -- support for DFOS and PHOENIX (2014-08-20)
#		1.5.1- dependency on config.scoreQC removed, redundant update of score info in ATAB files removed (2015-04-13)
#		1.6 -- supporting RAWDISP plots [BWo] (2015-06-18)
#		1.6.1- new optional config key RAWDISP_SUPPORT [BWo] (2015-07-22)
#		1.6.2- FINAL_PLUGIN de-coupled from processAB, to be configured now (2015-08-31)
# 		1.6.3- Bug fix for tool call (2015-09-03)
#		1.6.4- PROD_ROOT_NAME exported (2018-06-01)
#		1.6.5- exits gracefully if no QC procedure configured (2019-01-21)
#		1.6.6- call qc_rawdisp.py if no QC procedure configured, only exit afterwards (2019-03-07) [BWo]
#
# CONFIG:	config.processQC
# TOOLS CALLED: funpack (tile uncompression tool); QC procedures as configured
#
# COMMENTS:	enabled for parallel processing
#		funpack (tile-compression): used for VIRCAM
#		supporting PHOENIX
# ======================================================================
TOOL_VERSION="1.6.6"

# =========================================================================
# 0. Initialize
# 0.1 read params
# =========================================================================

if [ $# = 0 ]
then
        cat $DFO_DOC_DIR/processQC.h | more
        exit 0
fi

while getopts a:u:hv OPTION
do
	case "$OPTION" in
         v ) echo $TOOL_VERSION
             exit 0 ;;
         h ) cat $DFO_DOC_DIR/processQC.h | more
             exit 0 ;;
	 a ) export AB=$OPTARG;;
	 u ) export USER="$OPTARG"
             export HOME="/home/$USER"
	     export CALL_ENABLED=YES	# for triggering trendPlotter updates in execHC1_CALIB_$DATE 
	     export PATH
	     source $HOME/.bashrc    	# CONDOR has no environment variables
	     source $HOME/.dfosrc
	     ;;
	 ? ) cat $DFO_DOC_DIR/processQC.h | more
	     exit 0 ;;
	esac
done

# =========================================================================
# 0.2 initial checks
# =========================================================================

CHECK=`printenv | grep DFO`
if [ "Q$CHECK" = "Q" ]
then
        echo "*** ERROR: DFO variables not defined. Check ~/.dfosrc and restart."
        exit 1
fi

if [ ! -s $DFO_CONFIG_DIR/config.processQC ]
then
        echo "*** ERROR: No configuration file $DFO_CONFIG_DIR/config.processQC found. Check and re-start."
        exit 2
fi

# =========================================================================
# 0.3 read configuration
# =========================================================================

# uncompress VCAM raw files: requires funpack
CHECK_FUNPACK=`which funpack 2>/dev/null | wc -w`

# optional pgi to create coversheet; if none configured, default procedure createCS is taken
PGI_COVER=`grep "^PGI_COVER"    $DFO_CONFIG_DIR/config.processQC | awk '{print $2}'`
if [ Q$PGI_COVER != Q ] && [ ! -s $DFO_BIN_DIR/$PGI_COVER ]
then
	echo "***ERROR: $PGI_COVER is configured as \$PGI_COVER but is not found in \$DFO_BIN_DIR. We switch to default."
	PGI_COVER=""
fi

FINAL_PLUGIN=`grep "^FINAL_PLUGIN" 	$DFO_CONFIG_DIR/config.processQC | sed "s/\&\&$//" | sed "s/^.*\&\&//"`
if [ "Q$FINAL_PLUGIN" = "Q" ]
then
        FINAL_PLUGIN=NONE
fi

# check for RAWDISP support
RAWDISP_SUPPORT=`grep ^RAWDISP_SUPPORT	$DFO_CONFIG_DIR/config.processQC | awk '{print $2}'`
if [ "Q$RAWDISP_SUPPORT" = "Q" ]
then
	RAWDISP_SUPPORT=YES
fi

# =========================================================================
# 0.4 createCS: procedure to create a coversheet, either as standard output 
#     or by a pgi script
#       $1: $JTMP_DIR
# =========================================================================

createCS(){
CS_NAME=`echo ${AB} | sed "s/.ab//" | sed "s/^.*/&_cs.html/"`
if [ Q$PGI_COVER != Q ]
then
        eval "$DFO_BIN_DIR/$PGI_COVER $1 $CS_NAME"
else
        cat > $1/$CS_NAME <<EOT
<html>
<head>
<title> QC report coversheet for $AB</title>
</head>

<body>
<h2>QC report coversheet for $AB</h2>
<font size=2>This coversheet collects all QC reports for $AB.</font> 

<table style="border-collapse: collapse;" border="1" bordercolor="#333333">
EOT

	for G in `cat $1/list_graphics`
        do
                cat >> $1/$CS_NAME <<EOT
<tr>
  <td>
    <a href="./$G"><img src="./$G" width=300 height=200 title="click to enlarge"></a>
  </td>
  <td valign=bottom><font size=2>$G</font></td>
</tr>
EOT
        done

        echo "</table>
</body>
</html>" >> $1/$CS_NAME
fi
}

# =========================================================================
# 1. Get required information from AB
# =========================================================================

if [ ! -s $DFO_AB_DIR/$AB ]
then
	echo "***ERROR: $AB not found in $DFO_AB_DIR. Exit."
	exit -1
fi

DATE=`grep 		"^DATE"		$DFO_AB_DIR/$AB | awk '{print $2}'`
RAW_TYPE=`grep 		"^RAW_TYPE" 	$DFO_AB_DIR/$AB | awk '{print $2}'`
PROD_PATH=`grep "^PROD_PATH"            $DFO_AB_DIR/$AB | awk '{print $2}'`
export PROD_PATH=`eval "echo $PROD_PATH"`
export PROD_ROOT_NAME=`grep "^PROD_ROOT_NAME"	$DFO_AB_DIR/$AB | awk '{print $2}'`

# find procedure call in config
CALL=`grep 	"^CALL_SYNTAX"	$DFO_CONFIG_DIR/config.processQC | grep "[[:space:]]${RAW_TYPE}[[:space:]]" | sed "s/&&$//" | sed "s/^.*&&//"`
CALL1=`eval "echo $CALL"`
# measure elapsed real time in seconds and write into $TMP_DIR/time_processQC_$$ for later reference (QC_TEXEC)
rm -f $TMP_DIR/time_processQC_$$
CALL=`echo /usr/bin/time -f %e -o $TMP_DIR/time_processQC_$$ $CALL`

# find PF=PRIM_FILE for procedure call
PRIM_FILE=`grep "^CALL_SYNTAX"	$DFO_CONFIG_DIR/config.processQC | grep "[[:space:]]${RAW_TYPE}[[:space:]]" | awk '{print $3}'`

if [ ! -s $PROD_PATH ]
then
	echo "***ERROR: $PROD_PATH not found. Exit."
	exit -1
fi

# =========================================================================
# 2. Call QC procedure for AB
#    workflow: uncompress fz files | create RAWDISPs | QC proc | coversheet | scoreQC
# =========================================================================

# =========================================================================
# 2.1 check for .fz files in AB, uncompress if found (VCAM files),
#     then call qc_rawdisp.py and QC procedure
# =========================================================================

# unpack
if [ $CHECK_FUNPACK != 0 ]
then
	cd $PROD_PATH
	for FILE in `grep "^SOF_CONTENT"   $DFO_AB_DIR/$AB | grep "[[:space:]]RAW$" | awk '{print $2}'`
	do
		FILE=`eval echo $FILE`
		FILEZ=${FILE}.fz
		if [ ! -f $FILE ] && [ -f $FILEZ ]
		then
			BASEZ=`basename $FILEZ`
			funpack $FILEZ
			rm $FILEZ
			echo "***INFO: $BASEZ unpacked."
		fi
	done
fi

# check for creating RAWDISP plots
TODAY=`date +%Y-%m-%d`
OUTDATED=`qcdate $TODAY -7`
MJD_OUTDATED=`qcdate $OUTDATED`
MJD_DATE=`qcdate $DATE`
DPR_CATG=`grep "^DPR_CATG" $DFO_AB_DIR/$AB | awk '{print $2}'`
if [ $RAWDISP_SUPPORT = NO ]
then
	echo
	echo "*** INFO: RAWDISP_SUPPORT=NO, not creating RAWDISP plots for this instrument."
elif [ $DPR_CATG = SCIENCE ]
then
	echo
	echo "*** INFO: DPR_CATG=SCIENCE, not creating RAWDISP plots."
elif [ $MJD_DATE -lt $MJD_OUTDATED ]
then
	echo
	echo "*** INFO: AB older than 7 days, not creating RAWDISP plots."
else

# call qc_rawdisp.py
	echo 
	echo "- call qc_rawdisp.py ..."

	if [ ! -x ${DFO_BIN_DIR}/qc_rawdisp.py ]
	then
		echo "*** WARNING: ${DFO_BIN_DIR}/qc_rawdisp.py not existing; cannot create RAWDISPs."
	else
		if [ ! -d ${DFS_PRODUCT}/RAWDISP ]
		then
			mkdir ${DFS_PRODUCT}/RAWDISP
		fi
		if [ ! -d ${DFS_PRODUCT}/RAWDISP/${DATE} ]
		then
			mkdir ${DFS_PRODUCT}/RAWDISP/${DATE}
		fi
		${DFO_BIN_DIR}/qc_rawdisp.py -n -a $AB
	fi
fi

# call QC procedure
if [ "Q$CALL1" = Q ]
then
	echo "No QC procedure defined for ${RAW_TYPE}. Exit."
	exit
fi

echo "
- call QC procedure ..."
echo "	[$CALL1]"

cd $PROD_PATH
PF=`ls | grep $PROD_ROOT_NAME | grep "fits$" | grep $PRIM_FILE`

if [ "Q$PF" = "Q" ]
then
	echo "***INFO: No product found for $AB. Exit."
	exit
fi

# =========================================================================
# 2.2 launcher_$$: enabled for parallel processing
# =========================================================================

eval "echo $CALL" > $TMP_DIR/launcher_$$
chmod u+x $TMP_DIR/launcher_$$
$TMP_DIR/launcher_$$
rm -f $TMP_DIR/launcher_$$

# =========================================================================
# 2.3 call createCS if more than one graphics file is found
# =========================================================================

PID=$$
export JTMP_DIR=${TMP_DIR}/processQC_${PID}
if [ ! -d $JTMP_DIR ]
then
        mkdir $JTMP_DIR
fi

export PROD_ROOT_NAME=`echo $PROD_ROOT_NAME | sed "s/_tpl_1001/_tpl/"`
rm -f $JTMP_DIR/list_graphics $JTMP_DIR/list_graphics1
ls | grep ${PROD_ROOT_NAME} | egrep "gif|png|jpg" > $JTMP_DIR/list_graphics

if [ -s $JTMP_DIR/list_graphics ]
then
	rm -f $TMP_DIR/list_graphics_$$
	cp $JTMP_DIR/list_graphics $TMP_DIR/list_graphics_$$
	NUM_G=`cat $JTMP_DIR/list_graphics | wc -l`
	if [ $NUM_G -gt 1 ]
	then
		createCS $JTMP_DIR
		if [ -s $JTMP_DIR/$CS_NAME ]
		then
			mv $JTMP_DIR/$CS_NAME ./
		fi
	fi
fi
rm -rf $JTMP_DIR

# =========================================================================
# 2.4 call scoreQC 
# =========================================================================

echo "- call scoreQC ..."
scoreQC -a $AB | sed "s/^.*/[scoreQC]	&/"

# =========================================================================
# 3. Finish
# 3.1 catch execution time (TQCEXEC) and write into AB; LINE_TEXEC always exists
# =========================================================================

TQCEXEC=`cat $TMP_DIR/time_processQC_$$ | awk '{printf"%i\n",$1}'`
LINE_TEXEC=`grep -n "^TEXEC" $DFO_AB_DIR/$AB | awk '{print $1}' | sed "s/:.*//"`

cat $DFO_AB_DIR/$AB | grep -v TQCEXEC | awk '{if (NR <= n ) {print $0}}' n=$LINE_TEXEC > $DFO_AB_DIR/TMP/AB_$$
echo "TQCEXEC		$TQCEXEC" >> $DFO_AB_DIR/TMP/AB_$$
cat $DFO_AB_DIR/$AB | grep -v TQCEXEC | awk '{if (NR > n ) {print $0}}' n=$LINE_TEXEC >> $DFO_AB_DIR/TMP/AB_$$
mv $DFO_AB_DIR/TMP/AB_$$ $DFO_AB_DIR/$AB

# BWo 2013-07-11 v1.4, for processing acquisitions
AMODE=`grep "^DPR_CATG" $DFO_AB_DIR/$AB | awk '{print $2}' | sed "s/TECHNICAL/CALIB/" | sed "s/TEST/CALIB/" | sed "s/ACQUISITION/CALIB/"`

ATAB=`echo $AB | sed "s/.ab/.tab/"`
RECIPE=`grep "^RECIPE"  $DFO_AB_DIR/$AB | awk '{print $2}'`
PROC_STATUS=`grep "^PROCESS_STATUS" $DFO_AB_DIR/$AB | awk '{print $2}'`

if [ Q$TQCEXEC = QNONE ] || [ Q$TQCEXEC = Q ]
then
	QC_TEXEC=0
else
	QC_TEXEC=`echo $TQCEXEC | awk '{printf"%3.1f\n",$1/60.}'`
fi

# QC_REPORT and COVER_SHEET
QC_STATUS="QCSTATUS_DONEQCSTATUS"
CS_EXISTS="COVER_NONE"

if [ -s $TMP_DIR/list_graphics_$$ ]
then
	QC_LINK=`egrep "gif|png|jpg" $TMP_DIR/list_graphics_$$ | head -1`
	QC_STATUS="${RAW_TYPE}/$DATE/$QC_LINK"

	COVERSHEET=`echo $AB | sed "s/.ab/_cs.html/"`

	if [ -s $DFS_PRODUCT/${RAW_TYPE}/$DATE/$COVERSHEET ]
        then
		QC_STATUS="QCSTATUS_${QC_STATUS}CSSTATUS"
        	CS_EXISTS="COVER_${RAW_TYPE}/$DATE/${COVERSHEET}_COVER"
        else
		QC_STATUS="QCSTATUS_${QC_STATUS}QCSTATUS"
        fi

# special case: no graphics and no recipe --> check raw files
elif [ $RECIPE = RECIPEnone ]
then
	QC_STATUS=QCSTATUS_CHECKRAW
fi

# revise
if [ $RECIPE = RECIPEnone ] && [ $QC_STATUS = QCSTATUS_CHECKRAW ] && [ $PROC_STATUS = SUCCESSFUL ]
then
	AB_COLOR=`grep "^#STATUS" $DFO_BIN_DIR/getStatusAB | grep " $AMODE " | grep " CREATED NO WAITING" | awk '{print $6}'`
else
	AB_COLOR=`grep "^#STATUS" $DFO_BIN_DIR/getStatusAB | grep " $AMODE " | grep " SUCCESSFUL YES DONE" | awk '{print $6}'`
fi
AB_COLOR=`grep "^#COLOUR" $DFO_BIN_DIR/getStatusAB | grep " $AB_COLOR " | awk '{print $3}'`

# update ATAB file
if [ -s $DFO_AB_DIR/$ATAB ]
then
	cat $DFO_AB_DIR/$ATAB |\
 awk '{print abcolor,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,"+"qctexec,qcstatus,csexists,$27,$28,$29,$30,$31,$32,$33}' \
 abcolor=$AB_COLOR qctexec=$QC_TEXEC qcstatus=$QC_STATUS csexists=$CS_EXISTS \
 > $TMP_DIR/ATAB_$$
	mv $TMP_DIR/ATAB_$$ $DFO_AB_DIR/$ATAB
fi 

rm -f $TMP_DIR/time_processQC_$$ $TMP_DIR/list_graphics_$$

# =========================================================================
# 3.2 Apply FINAL_PLUGIN
# =========================================================================

if [ "$FINAL_PLUGIN" != NONE ]
then
	eval "$DFO_BIN_DIR/$FINAL_PLUGIN $AB"
fi

# =========================================================================
# 3.3 Exit
# =========================================================================

echo " ... done."
exit 0
