#!/bin/sh
# PURPOSE:      verifyAB: call the dynamic calSelector for verification of SCIENCE ABs 
# AUTHOR:       Reinhard Hanuschik / ESO
# VERSIONS:     1.0 -- copied from harvestAB v1.3.3 and calselManager v2.4, and rewritten
#		1.0.1- qc@qcweb replaced by ${DFO_WEB_SERVER} (2015-03-23)
#		1.0.2- uses DFS calSelector version (2015-05-06)
#		1.0.3- bug fix in :1451 (2015-06-11)
#		1.0.4- improved flagging for calSelector v2. (2015-06-22)
#		1.0.5- bug fix in :473 (2015-06-23)
#		1.1 -- option R: forced Raw2Raw; several bug fixes: calSelector list displayed if no AB found; calselector.log suppressed; cleanup bug fixed (2015-06-29)
#		1.2 -- introducing SUPPRESS_EXT (2016-08-03)
#		1.2.1- updates for P100 (0060) (2017-03-21)
#		1.2.2- changes needed for calSelector 3 (2018-12-18, BWo)
#		1.2.3- support multiple SUPPRESS_EXT values (2019-04-09, BWo)
#		1.2.4- support qcFlow naming scheme (2023-09-25)
# PARAMETERS:	-d <date> [-P]
#		-f <file> 
#		-f CONFIG (interactive selection)
#		-f LIST -l <file_list> 
# OPTIONS:	-P (ABs are still in $DFO_AB_DIR)
#		-M If set then do not replace M. names with ORIGFILE
#
# TOOLS CALLED:	calSelector 
#		The tool exposes a list of dp_id to calselector, gets the association 
#		results and compares them to the created ABs
#
# CONFIG:	$DFO_CONFIG_DIR/CALSELECTOR/CSConfiguration.properties
# CONFIG:	$DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB
#
# INPUT:	SCIENCE ABs either in $DFO_AB_DIR or in $DFO_LOG_DIR/<date>
# OUTPUT:	xml, txt files in $DFO_LOG_DIR/CALSELECTOR/<date>
# NOTES:	You can call the tool for dp_ids from any instrument, the tool asks
#		for the proper DFO_INSTRUMENT then.
# =========================================================================
TOOL_VERSION=1.2.4
TOOL_NAME=verifyAB

XMLFORMATTER=XMLFormatter

CALSELECTOR_CONFIG=$DFO_CONFIG_DIR/CALSELECTOR/CSConfiguration.properties # database configuration

JQUERY_URL="http://www.eso.org/observing/dfo/quality/ALL/jscript"       # URL for jquery / tablesorter

SCI_COL="#CCCCFF" 	# aligned with AB monitor
RED_COL="#FF6666"	# aligned with AB monitor

# =========================================================================
# 0. preparation
# 0.1 check for config file
# =========================================================================

if [ $# = 0 ] || [ $1 = -h ] 
then
	cat $DFO_DOC_DIR/verifyAB.h 
        exit 0
fi

if [ ! -s $CALSELECTOR_CONFIG ]
then
	echo "***ERROR: $CALSELECTOR_CONFIG not found."
	exit -1
fi

# =========================================================================
# 0.2 parameter check
# =========================================================================

MESSAGES=NORMAL
VERIFY_SCIENCE=NO	# technical param to control wrapped called of verifyAB for -d 
IGNORE_CERTIF=NO
REPLACE_MNAMES=YES	# new with v1.2.4; if YES then replace M. names with ORIGFILE, only useful for master calibrations created and renamed with DFOS-v1 

ASSOC_MODE="Raw2Master"

# P, S: hidden option 
while getopts d:f:l:o:hivDHPRSVM OPTION
do
	case "$OPTION" in
	 v ) echo $TOOL_VERSION ; exit 0 ;;
	 V ) CalSelector --version; exit ;;
	 h ) cat $DFO_DOC_DIR/verifyAB.h | more
	     exit 0 ;;
	 H ) CalSelector --help | more; exit ;;

	 D ) MESSAGES=DEBUG; DEBUG=-D ;;
	 P ) SOURCE=PRELIM ;;
	 d ) DATE=$OPTARG ;;
	 R ) ASSOC_MODE="Raw2Raw" ;;
	 S ) VERIFY_SCIENCE=YES ;;

	 f ) FILE=$OPTARG ;; 
	 o ) OCA_RULES=$OPTARG ;;
	 l ) LIST=$OPTARG ;;
	 i ) IGNORE_CERTIF=YES ;;
	 M ) REPLACE_MNAMES=NO;;
	 * ) exit -1 ;;
	esac
done

if [ Q$DATE != Q ]
then
	DATE1=`qcdate ${DATE} +1`
fi

if [ Q$SOURCE = Q ]
then
	SOURCE=$DFO_LOG_DIR/$DATE
else
	SOURCE=$DFO_AB_DIR
fi

if [ $MESSAGES = DEBUG ]
then
        MESSAGES=--debug
else
        MESSAGES=""
fi

if [ Q$DATE = Q ] && [ Q$FILE = Q ]
then
	echo "***ERROR: you must specify either a date or a file id. Exit."
	exit -1
fi

if [ Q$OCA_RULES != Q ]
then
	if [ ! -s $OCA_RULES ]
	then
		echo "***ERROR: $OCA_RULES does not exist. Exit."
		exit -1
	fi

	echo "We use the local OCA file $OCA_RULES."
	OCA_RULES1="--oca-rules-file=$OCA_RULES"
	OCA_RULES2="-o $OCA_RULES"
fi

# CERTIF_MODE

if [ $IGNORE_CERTIF = NO ]
then
	CERTIF_MODE=""				# if tool goes R2R: deliver closest certified; if none found, deliver nothing
else
	CERTIF_MODE="--ignore-certification=true"	# if tool goes R2R: deliver closest, no matter if certif or not
fi

# for Science Verification runs
ENABLE_60=NO
if [ -s $DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB ]
then
	ENABLE_60=`grep "^ENABLE_60" $DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB | awk '{print $2}'`
	if [ Q$ENABLE_60 != QYES ]
	then
		ENABLE_60=NO
	else
		echo "***INFO: \$ENABLE_60 is set to YES, this must change eventually."
	fi
else
	ENABLE_60=NO
fi

SUPPRESS_EXT=""
if [ -s $DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB ]
then
	SUPPRESS_EXT=`grep "^SUPPRESS_EXT" $DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB | awk '{print $2}'`
fi

if [ Q$FILE != Q ]
then
	CHECK_FILE=`echo $FILE | egrep "ab|fits"`
	if [ Q$CHECK_FILE != Q ]
	then
		echo "***ERROR: you must submit a dp_id (no .ab, no .fits). Exit."
		exit
	fi
fi

# =========================================================================
# 0.3 procedures
# 0.3.1 remove_entry
# =========================================================================
# remove_entry: cleanup entry in JOBS_INGEST (obsolete?)
remove_entry(){
if [ -s $DFO_JOB_DIR/JOBS_INGEST ]
then
	sed -i -e "/verifyAB -d $DATE/d" $DFO_JOB_DIR/JOBS_INGEST
fi
}

# =========================================================================
# 0.3.2 replaceXML: procedure to replace M.names with ORIGFILE names in the 
#       XML files, to make them readable for QC; all
# =========================================================================
replaceXML(){
rm -f $TMP_DIR/cm_list_mdot
grep "name=\"M" *.xml | sed "s/^.*name=\"//" | sort -u | sed "s/\".*//" | sed "s/^.*/\"&\",/" | sed "$,$ s/,//" > $TMP_DIR/cm_list_mdot

cat > $TMP_DIR/cm_march_query <<EOT
SELECT
origfile,
dp_id
FROM
qc_metadata..qc_products
WHERE
dp_id in (
EOT
cat $TMP_DIR/cm_list_mdot >> $TMP_DIR/cm_march_query
cat >> $TMP_DIR/cm_march_query <<EOT
)
GO
EOT

isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $TMP_DIR/cm_march_query | sed "1,2 d" | sed "/^$/d" | grep -v affected > $TMP_DIR/cm_march_results
}

# =========================================================================
# 0.3.3 createTXT: procedure createTXT (create txt file from the xml output)
#     $1: $XML
# =========================================================================
createTXT(){
XML=$1
TXT=`echo $XML | sed "s/xml/txt/"`
DPID=`echo $1 | sed "s/xml/fits/"`

# ASSOC_MODE = Raw2Raw (upon request only)
if [ $ASSOC_MODE = Raw2Raw ]
then
	cat > $TXT <<EOT
# calSelector output ($ASSOC_MODE)
# ========================================
# This is the table of all raw files, and static calibration files, directly or indirectly linked to the SCIENCE raw file.
# We list:
# - CATG (CALIB or SCIENCE),
# - dp_id (where static calibrations start with an 'M'),
# - RAW_TYPE,
# - name of the data set,
# - completeness flag of that data set (true/false).

EOT
# normal case: ASSOC_MODE = Raw2Master
else
	cat > $TXT <<EOT
# Dynamic calSelector output ($ASSOC_MODE)
# ========================================
# This is the table of the raw science files, their associated raw files (RASSOC), and their associated calibration products (MCALIB).
# We list:
# - CATG (CALIB or SCIENCE),
# - (m)calib name/science arcfile name
# - DO_CLASS (pro.catg or data_type)
# - compl: completeness flag for the association (true/false)
# - atype: association type (main=SCIENCE/MCALIB or auxiliary=MASSOC)
# - certif: certified (meaningful for raw CALIB only)
# - match: calib_plan (if tryBetween satisfied), extended (if between satisfied), or N/A (no time match defined)

EOT
fi

rm -f $TMP_DIR/rm_txt $TMP_DIR/rm_txt1  $TMP_DIR/rm_txt2

# XMLFormatter has a headline which we suppress here but could be useful in case of doubt about fields
if [ -s $XML ]
then
# Raw2Raw
	if [ $ASSOC_MODE = Raw2Raw ]
	then
		$XMLFORMATTER $XML |\
  awk '{
 if ($1 == "SCIENCE" ) {print $1"       "$2"    "$3"    "first_raw"     "$4"    "$5"    "$6}
 if ($1 != "SCIENCE" ) {print $0}
        }' first_raw=${DP_ID} >> $TXT

# store info about certified for raw calib files
		rm -f $TMP_DIR/va_raw_calib
		grep "CALIB" $TXT | grep -v MISSING | grep $DFO_FILE_NAME | sort -u | awk '{print $2,$8}' > $TMP_DIR/va_raw_calib

		grep "Missing" $XML |\
 awk '{print "CALIB     MISSING ",$3,$5,$6,$7,$8,$9}' |\
 sed "s|\"/>||" \
>> $TXT

# Raw2Master
	elif [ $ASSOC_MODE = Raw2Master ]
        then
		echo "dataset: ${DP_ID}" >> $TXT
                echo "CATG	name						DO_CLASS	compl	atype	certif	match" >>$TXT
                echo "=================================================================================================" >>$TXT

                $XMLFORMATTER $XML |\
 sed "1,2 d" |\
 awk '{
 if ($1 == "SCIENCE" ) {print $1"	"$2"	"$3"	"$4"	"$6"	"$7"	"$8}
 if ($1 != "SCIENCE" ) {print $1"	"$2"	"$3"	"$5"	"$7"	"$8"	"$9}
       }' >> $TXT

# store info about certified for raw calib files
		rm -f $TMP_DIR/va_raw_calib
		grep "CALIB" $TXT | grep -v MISSING | grep $DFO_FILE_NAME | sort -u | awk '{print $2,$8}' > $TMP_DIR/va_raw_calib

		grep "Missing" $XML |\
 awk '{print "CALIB     MISSING ",$3,$5,$6,$7,$8,$9}' |\
 sed "s|\"/>||" \
>> $TXT
        fi
# else
else
        echo "***WARNING: no XML file created, something went wrong with calSelector. Exit." > $TMP_DIR/va_mail
	cat $TMP_DIR/va_mail 
	mail -s "[verifyAB] Error: no XML file created for $DPID " $OP_ADDRESS <$TMP_DIR/va_mail
        echo "SCIENCE $DPID calselector failed" >> $TXT
        exit
fi
}

# =========================================================================
# 0.4 Prepare
# =========================================================================

if [ ! -d $DFO_LOG_DIR/CALSELECTOR ]
then
	mkdir $DFO_LOG_DIR/CALSELECTOR
fi

if [ ! -s $DFO_LOG_DIR/CALSELECTOR/README ]
then
	cat > $DFO_LOG_DIR/CALSELECTOR/README <<EOT
This directory is used by verifyAB.
The ABs are for temporary use only (filtering for fake-SCIENCE ABs). 
EOT
fi

if [ ! -d $DFO_LOG_DIR/CALSELECTOR/WORK_DIR ]
then
	mkdir $DFO_LOG_DIR/CALSELECTOR/WORK_DIR
fi

if [ ! -d $DFO_LST_DIR/VERIFYAB ]
then
	mkdir $DFO_LST_DIR/VERIFYAB
fi

if [ $REPLACE_MNAMES = YES ]
then
	echo "*** INFO: verifyAB has been called WITHOUT option -M. "
	echo "    The M. names in the calSelector output will be replaced."
	echo "    This is appropriate for the old DFOS-v1 naming scheme for master calibrations."
	echo
else
	echo "*** INFO: verifyAB has been called WITH option -M. "
	echo "    The M. names in the calSelector output will NOT be replaced."
	echo "    This is appropriate for the new qcFlow naming scheme for master calibrations."
	echo
fi

# =========================================================================
# 1. Verify ABs from DFOS_OPS workflow or from HISTORY
# 1.1 filter
# 1.1.1 suppress all pseudo-science ABs
# =========================================================================

if [ Q$DATE != Q ]
then
	echo "- Verification of SCIENCE ABs for $DATE started ..."
	echo "  (AB source: $SOURCE)"
	if [ $ASSOC_MODE != "Raw2Master" ]
	then
		echo "  (calSelector mode: $ASSOC_MODE)"
	fi


	LOG_FILE=$DFO_LST_DIR/VERIFYAB/list_verify_SCIENCE_${DATE}.txt
	if [ -s $LOG_FILE ]
	then
		echo "" >> $LOG_FILE
	fi

	rm -f $DFO_LOG_DIR/CALSELECTOR/WORK_DIR/*
	rm -f $DFO_AB_DIR/CALSELECTOR/*
	rm -f $TMP_DIR/va_list_all_ab

	CHECK_SCI="0"
	if [ -d $SOURCE ]
	then
		cd $SOURCE
		CHECK_AB=`ls *.ab | head -1`
		if [ Q$CHECK_AB != Q ]
		then
			rm -f $TMP_DIR/va_abs
			CHECK_SCIENCE=`grep  "DPR_CATG[[:space:]]SCIENCE" ${DFO_FILE_NAME}.*ab | sed "s/:DPR.*//" | head -1`
			if [ "Q$CHECK_SCIENCE" != Q ]
			then
				#grep -l SCIENCE_$DATE `grep  "DPR_CATG[[:space:]]SCIENCE" ${DFO_FILE_NAME}.*ab | sed "s/:DPR.*//"` > $TMP_DIR/va_abs
				grep -l "DPR_CATG[[:space:]]SCIENCE" ${DFO_FILE_NAME}.*ab > $TMP_DIR/va_abs
				CHECK_SCI=`cat $TMP_DIR/va_abs | wc -l`
			else
				CHECK_SCI=0
			fi
		fi
	else
		echo "***INFO: no SCIENCE ABs found in $SOURCE, can't do. Exit."
		remove_entry
		exit 0
	fi

	if [ $CHECK_SCI = 0 ]
	then
		echo "***INFO: no SCIENCE ABs found in $SOURCE, can't do. Exit."
		remove_entry
		exit 0
	else
		case $ENABLE_60 in
		 "NO" )  echo "SCIENCE ABs found. Skipping fake SCIENCE ABs (daytime or 60.) ..." ;;
		 "YES" ) echo "SCIENCE ABs found. Skipping fake SCIENCE ABs (daytime) ..." ;;
		esac
	fi
	
	for AB in `cat $TMP_DIR/va_abs`
	do
		ALOG=`echo $AB | sed "s/.ab/.alog/"`
		cp $AB $DFO_LOG_DIR/CALSELECTOR/WORK_DIR
# ALOGs copied to $DFO_LOG_DIR some time in 2006
		if [ -s $ALOG ]
		then
			cp $ALOG $DFO_LOG_DIR/CALSELECTOR/WORK_DIR
		fi
	done

	cd $DFO_LOG_DIR/CALSELECTOR/WORK_DIR

	echo "=======================================================================" >> $LOG_FILE

# =========================================================================
# 1.1.2 filter for daytime science
#       all timestamps <T22 and >T11 are considered fake SCIENCE
# =========================================================================

	CHECK_OUTLIER=`ls ${DFO_FILE_NAME}.*ab | egrep "${DATE}T[0-1]|${DATE}T2[0-1]|${DATE1}T1[1-9]|${DATE1}T2" | head -1`
	if [ "Q$CHECK_OUTLIER" != Q ]
	then
	       	echo "AB(s) rejected because they are outside the time window T22...T10:" | tee -a $LOG_FILE
	       	ls ${DFO_FILE_NAME}.*ab | egrep "${DATE}T[0-1]|${DATE}T2[0-1]|${DATE1}T1[1-9]|${DATE1}T2" | tee -a $LOG_FILE
	       	rm -f ${DFO_FILE_NAME}.${DATE}T[0-1]*ab ${DFO_FILE_NAME}.${DATE}T2[0-1]*ab
	
# all timestamps >T11 are considered fake SCIENCE
	        rm -f ${DFO_FILE_NAME}.${DATE1}T1[1-9]*ab ${DFO_FILE_NAME}.${DATE1}T2*ab
		echo ""
	fi

# =========================================================================
# 1.1.3 all run_IDs with 60./060./0060. considered fake SCIENCE
#	(unless ENABLE_60 = YES)
# =========================================================================

	if [ $ENABLE_60 = NO ]
	then
		CHECK_60=`grep  -l "^OBS_PROG_ID[[:space:]]*60\."  ${DFO_FILE_NAME}.*ab | head -1`
		CHECK_060=`grep -l "^OBS_PROG_ID[[:space:]]*060\." ${DFO_FILE_NAME}.*ab | head -1`
		CHECK_0060=`grep -l "^OBS_PROG_ID[[:space:]]*0060\." ${DFO_FILE_NAME}.*ab | head -1`
	
		if [ "Q$CHECK_60" != Q ] || [ "Q$CHECK_060" != Q ] || [ "Q$CHECK_0060" != Q ]
		then
			echo "AB(s) rejected because of PROG_ID starting with 60./060./0060.:" | tee -a $LOG_FILE
	       		grep -l "^OBS_PROG_ID[[:space:]]*60\."  ${DFO_FILE_NAME}.*ab | tee -a $LOG_FILE
	       		grep -l "^OBS_PROG_ID[[:space:]]*060\." ${DFO_FILE_NAME}.*ab | tee -a $LOG_FILE
	       		grep -l "^OBS_PROG_ID[[:space:]]*0060\." ${DFO_FILE_NAME}.*ab | tee -a $LOG_FILE
	       		rm -f `grep -l "^OBS_PROG_ID[[:space:]]*60\."  ${DFO_FILE_NAME}.*ab`
	       	 	rm -f `grep -l "^OBS_PROG_ID[[:space:]]*060\." ${DFO_FILE_NAME}.*ab`
	       	 	rm -f `grep -l "^OBS_PROG_ID[[:space:]]*0060\." ${DFO_FILE_NAME}.*ab`
	       	 	echo "" | tee -a $LOG_FILE
		fi
	else
		echo "ENABLE_60 = YES; ABs are included in verifyAB."
	fi

# =========================================================================
# 1.2 call calSelector 
# =========================================================================

	rm -f $TMP_DIR/va_verifier $TMP_DIR/va_dpid 
	CHECK_REMAIN=`ls | grep .ab | head -1`

	if [ Q$CHECK_REMAIN = Q ]
	then
		echo "***INFO: no SCIENCE ABs remaining after cleaning."
		remove_entry
		exit 0
	else
		for AB in `ls | grep .ab`
		do
			grep "^RAWFILE" $AB | head -1 | awk '{print $2}' | sed "s|/| |g" | awk '{print $3}' | sed "s/.fits//" >> $TMP_DIR/va_dpid
		done
# consolidate
		cat $TMP_DIR/va_dpid | sort -u | sed "/^$/d" > $TMP_DIR/va_dpid1
		mv $TMP_DIR/va_dpid1 $TMP_DIR/va_dpid
	fi

# we call verifyAB in LIST mode
	rm -f $TMP_DIR/va_mail
	if [ $ASSOC_MODE = Raw2Raw ]
	then
		RAW2RAW=-R
	else
		RAW2RAW=""
	fi

	if [ $SOURCE = $DFO_AB_DIR ]
	then
		$DFO_BIN_DIR/$TOOL_NAME -f LIST -l $TMP_DIR/va_dpid -P -S $DEBUG $RAW2RAW
	else
		$DFO_BIN_DIR/$TOOL_NAME -f LIST -l $TMP_DIR/va_dpid -S $DEBUG $RAW2RAW
	fi

# =========================================================================
# 1.3 Update ATAB files
# =========================================================================

	for AB in `cat $TMP_DIR/va_abs`
	do
		AB1=`basename $AB`
		XML=`echo $AB1 | sed "s/_.*/.xml/" | sed "s/.ab/.xml/"`
		TXT=`echo $XML | sed "s/.xml/.txt/"`
		ATAB=`echo $AB1 | sed "s/\.ab/.tab/"`
		F=`echo $TXT | sed "s/.txt//"`

		if [ -s ${DFO_AB_DIR}/CALSELECTOR/$TXT ]
		then
# $4: complete, $6: certif (false=R2R, true=R2M)
			RESULT=`grep "^SCIENCE" ${DFO_AB_DIR}/CALSELECTOR/$TXT | grep $F | awk '{print $4$6}'`
			TXT_STRING="SCIENCETXT_EXIST${TXT}SCIENCETXT"
		else
			RESULT=""
			TXT_STRING="SCIENCETXT"
		fi
	
		if [ -s ${DFO_AB_DIR}/CALSELECTOR/$XML ]
		then
			XML_STRING="SCIENCEXML_EXIST${XML}${RESULT}"
	        else
			XML_STRING="SCIENCEXML_NOTEXIST"
		fi

# ahtml is only a dummy, this is easier than going through all tools and remove the html file (was used for calSelector 1.x)
		if [ -s $SOURCE/$ATAB ]
		then
			cat $SOURCE/$ATAB |\
	 awk '{print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,axml,ahtml,atxt,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29,$30,$31,$32,$33}' \
 axml=$XML_STRING atxt=$TXT_STRING ahtml="dummy" \
 > $SOURCE/atab
			mv $SOURCE/atab $SOURCE/$ATAB
		fi
	done

# create the HTML overview 
	if [ $SOURCE != $DFO_AB_DIR ]
	then
		mv $DFO_AB_DIR/CALSELECTOR/* $DFO_LOG_DIR/CALSELECTOR/WORK_DIR
		cd $DFO_LOG_DIR/CALSELECTOR/WORK_DIR
	else
		cp $DFO_AB_DIR/CALSELECTOR/* $DFO_LOG_DIR/CALSELECTOR/WORK_DIR
		cd $DFO_LOG_DIR/CALSELECTOR/WORK_DIR
	fi

	cat > calsel_overview.html <<EOT
<h2> calSelector results for $DATE:</h2>
<table>
 <tr bgcolor=#cccccc><th><font size=2>AB</font></th><th><font size=2>TXT</font></th><th><font size=2>complete?</font></th><th><font size=2>mode</font></th></tr>
EOT
	cat calsel_overview.txt |\
 awk '{print "<tr bgcolor="col"><td><font size=2><a href=./"$1".ab>"$1".ab</a></font></td><td><font size=2><a href=./"$1".txt>txt</a></font></td><td><font size=2>"$2"</font></td><td><font size=2>"$3"</font></td></tr>"}' col=$SCI_COL >> calsel_overview.html
     	echo "</table>" >> calsel_overview.html
	sed -i -e "s/<td><font size=2>false/<td bgcolor=$RED_COL><font size=2>false/" calsel_overview.html

# =========================================================================
# 1.4 Set DFO flag and exit (only if -P is set)
# =========================================================================

	if [ $SOURCE = $DFO_AB_DIR ]
	then
		remove_entry

		echo "All result files in $DFO_AB_DIR/CALSELECTOR/.
(Call 'verifyAB -f <file_id>' for a detailed overview.)"
	else
		echo "All result files in $DFO_LOG_DIR/CALSELECTOR/$DATE/.
(Call 'verifyAB -f <file_id>' for a detailed overview.)
Check out $DFO_LOG_DIR/CALSELECTOR/$DATE/calsel_overview.html for an overview."
		mv $DFO_LOG_DIR/CALSELECTOR/WORK_DIR/* $DFO_LOG_DIR/CALSELECTOR/$DATE/
	fi	

	exit 0
fi

# =========================================================================
# 2. Testing file associations, with local or db OCA rules 
#    Output goes to $DFO_AB_DIR/CALSELECTOR; it's not stored!
# 2.1 Select the file if called with '-f CONFIG'
# =========================================================================

if [ $VERIFY_SCIENCE = NO ] && [ Q$LIST = Q ] 
then
	if [ Q$DEBUG != Q-D ]
	then
		echo "We enter the rules test mode of verifyAB."
	fi
fi

if [ $FILE = CONFIG ]
then
	if [ ! -s $DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB ]
	then
		echo "***ERROR: There is no config file \$DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB. Create and re-try."
		exit 
	fi
	
	grep "^FILE_NAME" $DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB | sed "s/^FILE_NAME[[:space:]]*//" | sed "s/\&\&//g" | sed "s/SPACE/     /"
	echo "========================================================================="
	echo ""
	echo "Select a file from the configured list (\$DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB):"

	read FILE

	if [ Q$FILE = Q ] || [ Q$FILE = QCONFIG ]
	then
		echo "***ERROR: You must select a file. Try again."
		exit
	fi
	DP_ID=`echo $FILE | sed "s/.fits//"`
	COMMENT=`grep $DP_ID $DFO_CONFIG_DIR/CALSELECTOR/config.verifyAB | grep -v "^#" | sed "s/^FILE_NAME.*${DP_ID}//" | sed "s/\&\&//g"`
fi

if [ $FILE = LIST ] && [ Q$LIST = Q ]
then
	echo "***ERROR: you must specify a filelist if you select -f LIST: 
'$TOOL_NAME -f LIST -l <filelist>'    with <filelist> the pathname to the file list."
	exit
fi

if [ $FILE != LIST ]
then
	DP_ID=`echo $FILE | sed "s/.fits//"`
fi

# =========================================================================
# 2.2 Get calSelector associations
# 2.2.1 Prepare
# =========================================================================

XML="${DP_ID}.xml"
TXT="${DP_ID}.txt"
STAT_TXT="${DP_ID}_stat.txt"

OUT_DIR=$DFO_AB_DIR/CALSELECTOR
if [ Q$OUT_DIR = Q ]
then
	echo "\$OUT_DIR=$OUT_DIR, error! Exit."
	exit -1
fi

if [ ! -d $OUT_DIR ]
then
	mkdir $OUT_DIR
fi

if [ $VERIFY_SCIENCE = NO ]
then
	echo "1. Start calSelector associations ..."
fi
rm -f $TMP_DIR/rm_calsel_log

# =========================================================================
# 2.2.2 DEBUG mode: create calSelector output and exit
# =========================================================================

if [ Q$MESSAGES = Q--debug ] && [ Q$LIST = Q ]
then
	cd $OUT_DIR
	rm -f *

	echo "DEBUG mode: create calSelector output and exit ..."
	echo "CalSelector --conf-file=$CALSELECTOR_CONFIG --input-files=$DP_ID --mode=$ASSOC_MODE $OCA_RULES1 $CERTIF_MODE $MESSAGES" > $TMP_DIR/rm_calsel_log
	echo "" >> $TMP_DIR/rm_calsel_log
	cat $TMP_DIR/rm_calsel_log
	CalSelector --conf-file=$CALSELECTOR_CONFIG --input-files="$DP_ID" --mode=$ASSOC_MODE $OCA_RULES1 $CERTIF_MODE $MESSAGES >> $TMP_DIR/rm_calsel_log

	echo "DEBUG mode: find the tool log in $TMP_DIR/rm_calsel_log. The xml output is in $OUT_DIR. "
	exit

# =========================================================================
# 2.2.3 LIST mode: work on that list and exit
# =========================================================================

elif [ Q$LIST != Q ]
then
	if [ $VERIFY_SCIENCE = YES ]
	then
		FILE_LIST=$LIST
	else
		echo "  Check calSelector $ASSOC_MODE associations ..."
		FILE_LIST=$LIST
	fi

	if [ ! -s $FILE_LIST ]
	then
		echo "***ERROR: $FILE_LIST not found, can't do."
		exit
	fi

	rm -f $TMP_DIR/rm_stripped_off
	grep -v "^#" $FILE_LIST | sed "s/FILE_NAME//" | awk '{print $1}' | sort -u | sed "/^$/d" > $TMP_DIR/rm_stripped_off
	if [ $VERIFY_SCIENCE = YES ]
	then
		echo "  Creating associations for DATE mode (`grep -v "^#" $TMP_DIR/rm_stripped_off | wc -l | awk '{print $1}'` files) ..."
	else
		echo "  Creating associations for $FILE_LIST (`grep -v "^#" $TMP_DIR/rm_stripped_off | wc -l | awk '{print $1}'` files) ..."
	fi
	echo ""

	if [ $VERIFY_SCIENCE = YES ]
	then
# fast mode, for new science ABs
		cd $OUT_DIR
		rm -f *
		CalSelector --conf-file=$CALSELECTOR_CONFIG --input-files=FILE --input-file=$TMP_DIR/rm_stripped_off --mode=$ASSOC_MODE $OCA_RULES1 $CERTIF_MODE $MESSAGES > $TMP_DIR/rm_calsel_log
	else
# interactive mode, inspect SDIFF files
		for F in `cat $TMP_DIR/rm_stripped_off`
		do
			echo "$F ..."
			$TOOL_NAME -f $F $OCA_RULES2 $DEBUG | sed "s/^.*/	&/"
			$DFO_GUI_DIR/refresh_browser $OUT_DIR/SDIFF.html
			echo "Hit return to continue:"
			read input
		done	
		echo "... done."
		exit
	fi

	RLS_FILE_USED=`grep "Using OCA rules file" $TMP_DIR/rm_calsel_log | tail -1 | sed "s/^.*Using OCA rules file//"`
	CHECK_ERROR=`egrep "WARN|ERR|is incomplete" $TMP_DIR/rm_calsel_log | head -1`

	if [ "Q$CHECK_ERROR" != Q ]
	then
		rm -f $TMP_DIR/auto_mail
		egrep "WARN|ERROR|is incomplete" $TMP_DIR/rm_calsel_log >> $TMP_DIR/auto_mail
		echo "" >> $TMP_DIR/auto_mail
		for F in `cat $TMP_DIR/rm_stripped_off`
		do
			if [ ! -s $DFO_AB_DIR/CALSELECTOR/$F.xml ]
			then
				echo $F >> $TMP_DIR/auto_mail
			else
				CHECK_FALSE=`cat $DFO_AB_DIR/CALSELECTOR/$F.xml | grep "mode=" | grep false`
				if [ "Q$CHECK_FALSE" != Q ]
				then
					echo $F >> $TMP_DIR/auto_mail
				fi
			fi
		done
		cat $TMP_DIR/auto_mail
	else
		echo " ... done. No errors found."
	fi

# TXT files and overview file
	if [ $VERIFY_SCIENCE = NO ]
	then
		echo "  Creating txt files ..."
	else
		echo "  Creating txt and html files ..."
	fi
	
	rm -f calsel_overview.txt	
	if [ $REPLACE_MNAMES = YES ]
	then
		replaceXML 
	fi

	for F in `cat $TMP_DIR/rm_stripped_off`
	do
		if [ $REPLACE_MNAMES = YES ]
		then
			for M in `cat $TMP_DIR/cm_march_results | awk '{print $2}'`
			do
				CDB=`grep "$M" $TMP_DIR/cm_march_results | awk '{print $1}'`
				sed -i -e "s/$M/$CDB/" $F.xml
			done
		fi

		createTXT $F.xml
# $4 is completeness flag
		RESULT=`grep "^SCIENCE" $F.txt | grep $F | awk '{print $4}'`
		MODE=`cat $F.xml | grep mode= | sed "s/^.*mode=//" | sed "s/type=.*//" | sed "s/\"//g"`
		echo "	$F $RESULT $MODE"
		echo "$F $RESULT $MODE" >> calsel_overview.txt
	done

	if [ $VERIFY_SCIENCE = NO ]
	then
		echo "
  Find the tool log in $TMP_DIR/rm_calsel_log. The xml output is in $OUT_DIR."
	fi
	exit
fi

# =========================================================================
# 2.2.4 Do the association and continue for comparison
# =========================================================================

cd $OUT_DIR
rm -f *

if [ Q$MESSAGES = Q"--debug" ]
then
	echo "Tool call:" >> $TMP_DIR/rm_calsel_log
	echo "CalSelector --conf-file=$CALSELECTOR_CONFIG --input-files=$DP_ID --mode=$ASSOC_MODE $OCA_RULES1 $CERTIF_MODE $MESSAGES" >> $TMP_DIR/rm_calsel_log
	echo "" >> $TMP_DIR/rm_calsel_log
fi

CalSelector --conf-file=$CALSELECTOR_CONFIG --input-files="$DP_ID" --mode=$ASSOC_MODE $OCA_RULES1 $CERTIF_MODE $MESSAGES >> $TMP_DIR/rm_calsel_log

cat $TMP_DIR/rm_calsel_log
if [ Q$MESSAGES = Q"--debug" ]
then
	echo "Find tool log in $TMP_DIR/rm_calsel_log."
fi

# Check for lock error
CHECK_LOCK=`grep "If this is the case you can manually delete" $TMP_DIR/rm_calsel_log | head -1`
if [ "Q$CHECK_LOCK" != Q ]
then
	exit
fi

# check for dataset warning
CHECK_DATASET=`grep "The following dp_ids do not belong to any datasets" $TMP_DIR/rm_calsel_log | sed "s/^.*to any datasets://" | sort -u`
if [ "Q$CHECK_DATASET" != Q ]
then
	echo "
*** WARNING: The dp_id(s) have no dataset (virtual product) defined in the archive:"
	echo $CHECK_DATASET
	echo "Each of these files will be assumed to represent one dataset. If this is incorrect, try to create the proper dataset (virtual product)."
fi	

# Check for "No OCA rules found" error
CHECK_NOOCA=`grep "No OCA rules found" $TMP_DIR/rm_calsel_log`
if [ "Q$CHECK_NOOCA" != Q ]
then
	CHECK_NOOCA=YES
	echo "***ERROR: No OCA rules found. Check and fix this issue. We continue with displaying the AB content:"
fi

# construct a nicer log
echo "<html>" > $DFO_AB_DIR/CALSELECTOR/$DP_ID.log.html
echo "<table>" >> $DFO_AB_DIR/CALSELECTOR/$DP_ID.log.html
cat $TMP_DIR/rm_calsel_log |\
 sed "s/^.*/<tr bgcolor=BGCOLOR><td><font size=2>&<\/font><\/td><\/tr>/" |\
 sed "/INFO/s/BGCOLOR/#EEEEEE/" |\
 sed "/WARN/s/BGCOLOR/#FF9999/" |\
 sed "/ ERROR/s/BGCOLOR/#FF6666/" |\
 sed "s/BGCOLOR/#CCCCCC/" \
 >> $DFO_AB_DIR/CALSELECTOR/$DP_ID.log.html
echo "</table></html>" >> $DFO_AB_DIR/CALSELECTOR/$DP_ID.log.html

# check if ${DP_ID}.xml is still the proper name, or if XML file has different name
CHECK_XML=`ls *xml | wc -l`
if [ $CHECK_XML -gt 1 ]
then
	echo "   There is more than 1 xml file (presumably from the whole dataset):"
	ls *xml

	echo "   We take the one corresponding to dp_id=${DP_ID}."
	XML_CAL=`ls ${DP_ID}*.xml | head -1`
else
	XML_CAL=`ls ${DP_ID}*.xml`
fi

if [ $XML != $XML_CAL ]
then
	echo "*** INFO: calSelector call on $DP_ID has returned $XML_CAL."
	NEW=`echo $XML_CAL | sed "s/.xml//"`
	XML=$NEW.xml
	TXT=$NEW.txt
else
	NEW=$DP_ID
fi

# replace M.names with ORIGFILE names in the XML file, to make it readable for QC
if [ $REPLACE_MNAMES = YES ]
then
	if [ -s $XML ]
	then
		replaceXML 
		for M in `cat $TMP_DIR/cm_march_results | awk '{print $2}'`
		do
			CDB=`grep "$M" $TMP_DIR/cm_march_results | awk '{print $1}'`
			sed -i -e "s/$M/$CDB/" $XML
		done
	fi
fi

# construct the TXT file
createTXT $XML

# try to find the parent AB, for comparison
FILE_NAME=`echo $DP_ID | sed "s/\./ /" | awk '{print $1}'`
DFO_DATE=`echo $DP_ID | sed "s/\./ /" | awk '{print $2}' | cut -c 1-13`
HH=`echo $DP_ID | sed "s/\./ /" | awk '{print $2}' | cut -c 12-13`
DFO_DATE=`echo $DFO_DATE | cut -c 1-10`
if [ $HH -lt 21 ]
then
	DFO_DATE=`qcdate $DFO_DATE -1`
fi

if [ $FILE_NAME != $DFO_FILE_NAME ] && [ Q$INSTRUMENT = Q ]
then
	echo "" 
	echo "Enter DFO_INSTRUMENT for file $DP_ID:"
	read INSTRUMENT
else
	INSTRUMENT=$DFO_INSTRUMENT
fi

scp ${DFO_WEB_SERVER}:public_html/$INSTRUMENT/logs/${DFO_DATE}/${NEW}*.ab ./ 1>/dev/null 2>/dev/null
scp ${DFO_WEB_SERVER}:public_html/$INSTRUMENT/logs/${DFO_DATE}/${NEW}*.alog ./ 1>/dev/null 2>/dev/null

if [ Q$SUPPRESS_EXT != Q ]
then
	AB=`ls | grep \.ab | grep ${NEW} | egrep -v -e "$SUPPRESS_EXT"`
else
	AB=`ls | grep \.ab | grep ${NEW}`
fi

if [ Q$AB = Q ]
then
	echo "*** WARNING: No AB found on $DFO_WEB_SERVER in $INSTRUMENT/logs/${DFO_DATE} for $DP_ID, can't do the consistency check."
	AB=UNKNOWN
	ALOG=UNKNOWN
	AB_DATE=UNKNOWN
else
	ALOG=`echo $AB | sed "s/.ab/.alog/"`
	AB_DATE=`grep "^DATE" $AB | awk '{print $2}'`
fi

# =========================================================================
# 2.3 Compare results to AB
# 2.3.1 Prepare and start HTML output; calculate TDIFFs
# =========================================================================

if [ $AB = UNKNOWN ]
then
	echo "
2. Create overview file ..."
else
	echo "
2. Compare to stored AB ..."
fi

if [ ! -s $ALOG ]
then	
	echo "***INFO:  no SETUP info available."
	SETUP="unknown, no ALOG file found"
else	
	SETUP=`grep "^[[:space:]]*${AB}" $ALOG | awk '{print $3}'`
fi

# queries to retrieve MJD_OBS: cm_getMjd for raw files, cm_getMjd_mcal for mcalibs
rm -f $TMP_DIR/cm_arcfiles $TMP_DIR/cm_getMjd $TMP_DIR/cm_arcfiles_mcal
cat > $TMP_DIR/cm_getMjd <<EOT
select
dp_id, mjd_obs
from
obs_metadata..data_products
where
dp_id in (
EOT
cat $TMP_DIR/cm_getMjd | sed "s/obs_metadata..data_products/qc_metadata..qc_products/" | sed "s/dp_id/origfile/" > $TMP_DIR/cm_getMjd_mcal
echo "\"$DP_ID \" ," > $TMP_DIR/cm_arcfiles

cat $TXT | grep "^CALIB" | grep -v MISSING | awk '{print $2}' | sort -u | sed "s/^.*/\"&\" ,/" >> $TMP_DIR/cm_arcfiles_mcal
cat $TMP_DIR/cm_arcfiles >> $TMP_DIR/cm_getMjd
cat $TMP_DIR/cm_arcfiles_mcal >> $TMP_DIR/cm_getMjd

cat $TMP_DIR/cm_arcfiles >> $TMP_DIR/cm_getMjd_mcal
cat $TMP_DIR/cm_arcfiles_mcal >> $TMP_DIR/cm_getMjd_mcal

sed -i -e "$,$ s/,//" $TMP_DIR/cm_getMjd
sed -i -e "$,$ s/,//" $TMP_DIR/cm_getMjd_mcal

cat >> $TMP_DIR/cm_getMjd <<EOT
)
go
EOT
cat >> $TMP_DIR/cm_getMjd_mcal <<EOT
)
go
EOT

rm -f $TMP_DIR/cm_arc_mjd	
isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $TMP_DIR/cm_getMjd | sed "1,2 d" | sed "/affected/d" | awk '{print $1,$2}'  > $TMP_DIR/cm_arc_mjd
isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $TMP_DIR/cm_getMjd_mcal | sed "1,2 d" | sed "/affected/d" | awk '{print $1,$2}' >> $TMP_DIR/cm_arc_mjd

MJD_SCI=`grep $DP_ID $TMP_DIR/cm_arc_mjd | awk '{print $2}'`

rm -f $TMP_DIR/cs_timediff*
for F in `cat $TMP_DIR/cm_arc_mjd | awk '{print $1}' | grep -v $DP_ID`
do
	MJD_CAL=`grep $F $TMP_DIR/cm_arc_mjd | awk '{print $2}'`
	TDIFF=`echo $MJD_CAL $MJD_SCI | awk '{print $1-$2}'`
	echo $F $TDIFF >> $TMP_DIR/cs_timediff2
done

# find the calSelector mode used
if [ -s $XML ]
then
	CALSEL_MODE=`grep "mode=" $XML | sed "s/^.*mode=.//" | awk '{print $1}' | sed "s/\"//"`

	if [ Q$CALSEL_MODE = Q ]
	then
		CALSEL_MODE=failed
	fi
else
	CALSEL_MODE=failed
fi

# start creating HTML
cat > $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
<html>
<head>
<script type="text/javascript" src="${JQUERY_URL}/jquery-latest.js"></script>
<script type="text/javascript" src="${JQUERY_URL}/jquery.dataTables.js"></script>
<!-- dataTable features, like filtering; taken from calChecker and getStatusAB-->

<script type="text/javascript" charset="utf-8">
  \$(document).ready(function()
    {
    \$("#mainTable").dataTable({
    "aaSorting": [[0,'asc']],
    "aoColumnDefs":
        [
        ],
    "bPaginate": false,
    "sDom": '<ift>',
    "oLanguage": {
       "sSearch": "Search (browser <a href=\"javascript:location.reload()\">refresh</a>):",
       "sInfo": "Showing _START_ to _END_ of _TOTAL_ files",
       "sInfoEmpty": "Showing 0 to 0 of 0 files",
       "sInfoFiltered": "(filtered from _MAX_ total files)"
                 }
     });
    });
</script>
<!-- styles for datatable -->
<style type="text/css">
table.display {
        text-align:     left;
        font-family:    arial;
        font-size:      small;
        height:         10px;
}
table.display thead tr .sorting {
        background-color:    #CCCCCC;
        background: url(http://www.eso.org/qc/ALL/img/bg.gif) no-repeat center right;
}
table.display thead tr .sorting_desc {
        background-color:    ;
        background: url(http://www.eso.org/qc/ALL/img/desc.gif) no-repeat center right;
}
table.display thead tr .sorting_asc {
        background-color:    ;
        background: url(http://www.eso.org/qc/ALL/img/asc.gif) no-repeat center right;
}
</style>
</head>

<h2>calSelector results: comparison for $FILE </h2>

<font size=2><b>Setup: $SETUP &nbsp; &nbsp; Comment: $COMMENT</b><p>
EOT

if [ Q$OCA_RULES != Q ]
then
	cat >> $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
<b><font color=red>Local OCA file used: $OCA_RULES</font></b><p>
EOT
fi

if [ $ASSOC_MODE = $CALSEL_MODE ]
then
	cat >> $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
CalSelector mode called: <b>$ASSOC_MODE</b> <br>
Followed by tool: <b>$CALSEL_MODE</b> <br>
EOT
elif [ $CALSEL_MODE = failed ]
then
	cat >> $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
<font color=red>CalSelector mode called: <b>$ASSOC_MODE</b> <br>
Tool: <b>$CALSEL_MODE</b></font> <br>
EOT
else
	cat >> $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
CalSelector mode called: <b>$ASSOC_MODE</b> <br>
<font color=red>Modified by tool: <b>$CALSEL_MODE</b></font> <br>
EOT
fi

if [ "Q$CHECK_DATASET" != Q ]
then
	cat >> $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
<font color=red>No dataset defined for ${DP_ID}</font><br>
EOT
fi

COMPLETE_YN=`grep "^SCIENCE" $TXT | awk '{print $4}' | sort -u`

if [ $COMPLETE_YN != true ] && [ $COMPLETE_YN != false ]
then
	COMPLETE_YN=`grep "^SCIENCE" $TXT | awk '{print $5}' | sort -u`
fi

if [ $COMPLETE_YN = "true" ]
then
	COMPLETE_YN="<font color=green><b>$COMPLETE_YN</b></font>"
elif [ $COMPLETE_YN = "false" ] && [ Q$CHECK_NOOCA = Q ]
then
	COMPLETE_YN="<font color=red><b>$COMPLETE_YN</b></font>"
	MISS=`grep "could not find enough products" $DFO_AB_DIR/CALSELECTOR/calselector.log | sed "s/^.*products of type//" | sed "s/for ab.*//" | sort -u | sed "s/'//g" | tr "\012" "|" | sed "s/|$//" | sed "s/^.*/&\n/"`
	MISS_TAG="<font color=red>Missing file(s): <b>${MISS}</b></font>"
elif [ Q$CHECK_NOOCA = QYES ]
then
	COMPLETE_YN="<font color=red><b>$COMPLETE_YN</b></font>"
	MISS_TAG="<font color=red><b>No OCA rules found</b></font>"
fi

cat >> $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
Complete: $COMPLETE_YN<p>
$MISS_TAG
<p>
This is a comparison of calSelector results and the AB results (as downloaded from ${DFO_WEB_SERVER}).<br>
Effectively this is also a comparison of the CALSELECTOR rules and the DFOS_OPS rules.<br>
To help with the comparison, the table displays time differences (in days).<p>

Note that the tool auto-switches to Raw2Raw if one or more mcalibs are missing. While in Raw2Master only<br>
the mcalibs required for SCIENCE processing are collected, in Raw2Raw all raw calibrations are collected,<br>
including the ones needed for processing other calibrations!<p>

The mcalibs are displayed here by origfile name (the "QC name") rather than by arcfile name as in the<br>
 original txt files. 
<p>

D - difference marker: <br>
<table>
<tr bgcolor=#33FF33><td><font size=2>=</td><td><font size=2>same content in both files (only AB result displayed)</td></tr>
<tr bgcolor=#FF9999><td><font size=2>|</td><td><font size=2>line contained in both files but content differs</td></tr>
<tr bgcolor=#FF6666><td><font size=2>></td><td><font size=2>content is missing in the left (AB) file</td></tr>
<tr bgcolor=#FF6666><td><font size=2><</td><td><font size=2>content is missing in the right (calSelector) file</td></tr>
</table>

<b>CATG:</b> SCIENCE for input and associated (from same template) science files, RASSOC for associated raw files,<br> 
&nbsp;&nbsp;&nbsp;&nbsp; MCALIB for master calibs, CALIB for raw calibrations, VIRTUAL for virtual/uncertified masters;<br>
<b>REQ_1_FND_0</b>: 1 file requested in OCA rule, 0 files found<br>
<b>not_archived</b>: listed MCALIB/MASSOC is not archived <br>
<b>uncertif</b>: raw file is not certified (quality is unknown, not necessarily bad). No entry means 'certified'.

<p>
Empty lines in the calSelector results mean entries are identical to the AB results.<br>
*Time differences (in fractional days) are not part of the comparison (their format could differ).
<p>

<font size=2><a href=./$FILE.log.html>log file</a> ||
<a href=$DFO_AB_DIR/CALSELECTOR/>all files</a> | 
EOT

if [ $AB = UNKNOWN ]
then
	cat >> $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
<b>no AB found for comparison</b> ||
EOT
else
	cat >> $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
<a href=./${AB}>original AB</a> | 
<a href=./${DP_ID}_stat.txt>sof txt file</a> ||
EOT
fi

cat >> $DFO_AB_DIR/CALSELECTOR/${DP_ID}.sdiff.html <<EOT
<a href=./${DP_ID}.txt>calSelector txt file</a> |
<a href=./${XML}>calSelector xml file</a> ||
<a href="http://www.eso.org/qc/ALL/OCA/CALSELECTOR/${DFO_INSTRUMENT}/cascade_${DFO_INSTRUMENT}_ALL.html">CALSELECTOR calib maps</a> | 
<a href="${DFO_LST_DIR}/REPORT/list_${DFO_INSTRUMENT}_${AB_DATE}_data.html">data report</a>
<p>

<!-- sortable table -->
<i><font size=2 color=#666666>
<table id="mainTable" class="display" cellspacing=1 cellpadding=1 border=0>
<thead>
  <tr bgcolor=#CCCCCC>
    <th colspan=6>AB result</th>
    <th>D<font color=#CCCCCC>___</font></th>
    <th colspan=5>calSelector result ($CALSEL_MODE)</th>
  </tr>
  
  <tr bgcolor=#CCCCCC>
    <th nowrap>index <font color=#CCCCCC>__</th>
    <th>CATG<font color=#CCCCCC>__</th>
    <th>ORIGFILE<font color=#CCCCCC>___</th>
    <th>DO_CLASS<font color=#CCCCCC>__</th>
    <th><a title="MCALIB: main; MASSOC: auxiliary">MCALIB/ MASSOC?</a><font color=#CCCCCC>___</th>
    <th width=60><a title="negative: file is earlier than SCIENCE">time</a><br> diff*</th>
    <th>&nbsp;</td>
    <th>CATG<font color=#CCCCCC>__</th>
    <th>ORIGFILE<font color=#CCCCCC>___</th>
    <th>DO_CLASS<font color=#CCCCCC>__</th>
    <th><a title="MCALIB: main; MASSOC: auxiliary">MCALIB/ MASSOC?</a><font color=#CCCCCC>___</th>
    <th width=60><a title="negative: file is earlier than SCIENCE">time</a><br> diff*</th>
  </tr>
</thead>
<tbody>
EOT

# =========================================================================
# 2.3.2 prepare txt file for diff comparison
#     - science
#     - raw calibs
#     - static calibs
#     - TDIFFs
# =========================================================================

# select all data for display 
# 1. science
cat ${TXT} | grep -v "^#" | grep SCIENCE    | awk '{print $1,$2".fits",$3,$5,"TDIFF"}' > ${TXT}_1

# 2. calibs (first: main, then: auxiliary)
if [ $CALSEL_MODE = "Raw2Master" ]
then
	cat ${TXT} | grep -v "^#" | grep CALIB | grep main | awk '{print $1,$2,$3,$5,"TDIFF"}' | sed "s/CALIB/MCALIB/g" | sort -k2,2 > ${TXT}_2
# RASSOC and MASSOC in ${TXT} can only be distinguished by raw file vs. mcalib tag
# RASSOC:
	cat ${TXT} | grep -v "^#" | grep CALIB | grep auxi | grep $DFO_FILE_NAME | awk '{print $1,$2".fits",$3,$5,"TDIFF"}' | sed "s/CALIB/RASSOC/g" | sort -k2,2 >> ${TXT}_2
# MASSOC:
	cat ${TXT} | grep -v "^#" | grep CALIB | grep auxi | grep -v $DFO_FILE_NAME | awk '{print $1,$2,$3,$5,"TDIFF"}' | sort -k2,2 >> ${TXT}_2
else
	cat ${TXT} | grep -v "^#" | grep CALIB | grep main | awk '{print $1,$2,$3,$5,"TDIFF"}' | sort -k2,2 >  ${TXT}_2
	cat ${TXT} | grep -v "^#" | grep CALIB | grep auxi | grep $DFO_FILE_NAME | awk '{print $1,$2,$3,$5,"TDIFF"}' | sed "s/CALIB/RASSOC/g" | sort -k2,2 >> ${TXT}_2
	cat ${TXT} | grep -v "^#" | grep CALIB | grep auxi | grep -v $DFO_FILE_NAME | awk '{print $1,$2,$3,$5,"TDIFF"}' | sort -k2,2 >> ${TXT}_2
fi

# MISSING: collect request/found info
cat ${TXT} | grep -v "^#" | grep CALIB | grep MISSING | awk '{print $1,$2,$3,"...",$5,$6,$7,$8}' | sort -k2,2 >> ${TXT}_2
# ignore multiple occurrences but preserve sequence
PREV_C=""
for C in `cat ${TXT}_2 | awk '{if ($2=="MISSING") {print $3} else {print $2}}'`
do
	if [ Q$C != Q$PREV_C ]
	then
		egrep "CALIB $C|CALIB MISSING $C|ASSOC $C" ${TXT}_2 | head -1 >> ${TXT}_1
		PREV_C=$C
	fi
done

# edit TDIFFs (dynamic) 
if [ -s $TMP_DIR/cs_timediff2 ]
then
	for F in `cat $TMP_DIR/cs_timediff2 | awk '{print $1}'`
	do
		TDIFF=`grep $F $TMP_DIR/cs_timediff2 | awk '{print $2}'`
		sed -i -e "/$F/s/TDIFF/$TDIFF/" ${TXT}_1
	done

	sed "s/TDIFF/\&nbsp;/g" ${TXT}_1 |\
 awk '{if ($2=="MISSING") {print $1,$2,$3,$4,"REQ_"$6"FND_"$8} else {print $0}}' |\
         sed "s/,FND/_FND/" | sed "s/: REQ/ REQ/" > ${TXT}_2
	mv ${TXT}_2 ${TXT}_1

else
	echo "***WARNING: No CALIB files found in dynamic mode, something went wrong."
fi

# =========================================================================
# 2.3.3 AB results: prepare txt file for diff comparison
#     - science
#     - mcalibs
# =========================================================================

if [ -s $AB ]
then
# 1. science
	cat ${AB} | grep "^RAWFILE" | sed "s|/| |g" | awk '{print "SCIENCE",$4,$5,"main","TDIFF"}' > ${STAT_TXT}_1

# 2. RASSOCs
	cat ${AB} | grep "^RASSOC" | grep -v NONE | sed "s|/| |g"  | awk '{print "RASSOC",$4,$5,"auxiliary","TDIFF"}' >> ${STAT_TXT}_1

# 3. mcalibs: REAL ...
	cat ${AB} | grep "^MCALIB" | grep REAL | sed "s|/| |g"  | awk '{print "MCALIB",$5,$6,"main","TDIFF"}'   | sort -k2,2  > ${STAT_TXT}_2
#             VIRTUAL: different cases due to historical errors
	cat ${AB} | grep "^MCALIB" | grep VIRTUAL | grep DFO_CAL_DIR | sed "s|/| |g" | awk '{print "MCALIB",$5,$6,"main","TDIFF"}' | sort -k2,2  >> ${STAT_TXT}_2
	cat ${AB} | grep "^MCALIB" | grep VIRTUAL | grep -v DFO_CAL_DIR | sed "s|/| |g" | awk '{print "MCALIB",$6,$7,"main","TDIFF"}' | sort -k2,2  >> ${STAT_TXT}_2
#             MASSOC
	cat ${AB} | grep "^MASSOC" | grep REAL | sed "s|/| |g"  | awk '{print "MCALIB",$5,$6,"auxiliary","TDIFF"}' | sort -k2,2  >> ${STAT_TXT}_2
	cat ${AB} | grep "^MASSOC" | grep VIRTUAL | grep DFO_CAL_DIR | sed "s|/| |g" | awk '{print "MCALIB",$5,$6,"main","TDIFF"}' | sort -k2,2  >> ${STAT_TXT}_2
	cat ${AB} | grep "^MASSOC" | grep VIRTUAL | grep -v DFO_CAL_DIR | sed "s|/| |g" | awk '{print "MCALIB",$6,$7,"auxiliary","TDIFF"}' | sort -k2,2  >> ${STAT_TXT}_2

# ignore multiple occurrences but preserve sequence
# unclear if MISSING is the proper treatment...
	for C in `cat ${STAT_TXT}_2 | awk '{if ($2=="MISSING") {print $3} else {print $2}}'`
	do
		CHECK_C=`grep "MCALIB $C " ${STAT_TXT}_1 | head -1`
		if [ "Q$CHECK_C" = Q ]
		then
			egrep "MCALIB $C|MCALIB MISSING $C" ${STAT_TXT}_2 | head -1 >> ${STAT_TXT}_1
		fi
	done
	rm -f ${STAT_TXT}_2

# get TDIFFs
	rm -f $TMP_DIR/cm_getMjd_mcal
	cat > $TMP_DIR/cm_getMjd_mcal <<EOT
select
origfile, mjd_obs
from
qc_metadata..qc_products
where
origfile in (
EOT

	cat $AB | egrep "^MCALIB|^MASSOC" | grep -v MISSING | grep VIRTUAL | sed "s|/| |g" | awk '{print $6}' | sort -u > $TMP_DIR/va_AB_filelist
	cat $AB | egrep "^MCALIB|^MASSOC" | grep -v MISSING | grep -v VIRTUAL | sed "s|/| |g" | awk '{print $5}' | sort -u >> $TMP_DIR/va_AB_filelist
	cat $TMP_DIR/va_AB_filelist | sed "s/^.*/\"&\" ,/" | sed "$,$ s/,//" >> $TMP_DIR/cm_getMjd_mcal

	cat >> $TMP_DIR/cm_getMjd_mcal <<EOT
)
go
EOT

	isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $TMP_DIR/cm_getMjd_mcal | sed "1,2 d" | sed "/affected/d" | sed "s/^$//" | awk '{print $1,$2}' | sort -u >> $TMP_DIR/cm_arc_mjd
	MJD_SCI=`grep $DP_ID $TMP_DIR/cm_arc_mjd | awk '{print $2}'`

	rm -f $TMP_DIR/cs_timediff*
	cat $TMP_DIR/cm_arc_mjd | sort -u > $TMP_DIR/cm_arc_mjd1
	mv $TMP_DIR/cm_arc_mjd1 $TMP_DIR/cm_arc_mjd

	for F in `cat $TMP_DIR/cm_arc_mjd | sort -u | awk '{print $1}' | grep -v $DP_ID`
	do
		MJD_CAL=`grep $F $TMP_DIR/cm_arc_mjd | awk '{print $2}'`
		TDIFF=`echo $MJD_CAL $MJD_SCI | awk '{print $1-$2}'`
		echo $F $TDIFF >> $TMP_DIR/cs_timediff2
	done

	cat $TMP_DIR/cs_timediff2 | sort -u > $TMP_DIR/cs_timediff22
	mv $TMP_DIR/cs_timediff22 $TMP_DIR/cs_timediff2

# any file from cm_getMjd_mcal missing in cs_timediff2?
	for F in `cat $TMP_DIR/va_AB_filelist`
	do
		CHECK_EXIST=`grep $F $TMP_DIR/cs_timediff2 | head -1`
		if [ "Q$CHECK_EXIST" = Q ]
		then
			echo "***WARNING: $F	found in AB but not archived!"
			sed -i -e "/$F/s|TDIFF|not_archived|" ${STAT_TXT}_1
		fi
	done

# mark gencalibs: anything older than 1yr (static)
# turned off since useful to compare mjd_obs of gencalibs having different name but same content (old names vs. new standard names starting with MCAL_CODE) 
	#cat $TMP_DIR/cs_timediff2 | awk '{if ($2 < -365 || $2 > 365) {print $1,"GEN"} else {print $0}}' > $TMP_DIR/cs_timediff22
	#mv $TMP_DIR/cs_timediff22 $TMP_DIR/cs_timediff2

# edit TDIFFs
	for F in `cat $TMP_DIR/cs_timediff2 | awk '{print $1}'`
	do
		TDIFF=`grep $F $TMP_DIR/cs_timediff2 | awk '{print $2}'`
		sed -i -e "/$F/s/TDIFF/$TDIFF/" ${STAT_TXT}_1
	done

# remaining TDIFF strings: from SCIENCE file itself, --> 0
	cat ${STAT_TXT}_1 |\
	 sed "s/TDIFF/0.00000/g" ${STAT_TXT}_1 |\
	 awk '{ if ($2=="MISSING") {print $1,$2,$3,$4,"null",$3,"REQ_"$6"FND_"$8,"none"} else {print $0}}' |\
	 sed "s/FND/_FND/" > ${STAT_TXT}_2
	mv ${STAT_TXT}_2 ${STAT_TXT}_1
	cp ${STAT_TXT}_1 ${STAT_TXT}
fi

# =========================================================================
# 2.3.4 do the diff
# =========================================================================

if [ ! -s ${STAT_TXT}_1 ]
then
	touch ${STAT_TXT}_1
fi

rm -f $DP_ID.sdiff1

# a) if no AB exists, we have to add "SCIENCE|RASSOC" files from ${TXT}_1
if [ $AB = UNKNOWN ] && [ -s ${TXT}_1 ]
then
	grep "SCIENCE" ${TXT}_1 | sed "s/^.*/	> &/" > $DP_ID.sdiff1
	grep "RASSOC" ${TXT}_1  | sed "s/^.*/	> &/" >> $DP_ID.sdiff1

else
# b) otherwise: SCIENCE/RASSOC from AB
# sort -u required because of non-standard VCAM ABs with multiple entries for SCIENCE dp_ids
# ${TXT}_1 has .fits for product files, no .fits for raw files
	for F in `egrep "SCIENCE|RASSOC" ${STAT_TXT}_1 | awk '{print $2}' | sort -u`
	do
		F1=`echo $F | sed "s/.fits//"`
		FTYPE=`grep " $F " ${STAT_TXT}_1 | awk '{print $3}' | head -1`
		# was: FSTAT=`grep " $FTYPE " ${TXT}_1  | awk '{print $2}' | head -1`
		FSTAT=`egrep " $F | $F1 " ${TXT}_1  | awk '{print $2}' | head -1`

		ENTRY_STAT=`grep " $F " ${STAT_TXT}_1 | head -1`
		ENTRY_DYN=`egrep " $F | $F1 " ${TXT}_1`
		ENTRY_TYPE_DYN=`grep " $FTYPE " ${TXT}_1 | grep -v MISS | head -1`

		if [ "Q$ENTRY_DYN" != Q ]
		then
			echo "$ENTRY_STAT	=" >> $DP_ID.sdiff1
			sed -i -e "/$F/d" ${TXT}_1
		elif [ "Q$ENTRY_TYPE_DYN" != Q ]
		then
			echo "$ENTRY_STAT	| $ENTRY_TYPE_DYN " >> $DP_ID.sdiff1
			sed -i -e "/$FSTAT/d" ${TXT}_1
		else
			echo "$ENTRY_STAT	<" >> $DP_ID.sdiff1
		fi
	done	
fi

# CALIB from AB:
for F in `grep MCALIB ${STAT_TXT}_1 | awk '{print $2}'`
do
	FTYPE=`grep " $F " ${STAT_TXT}_1 | awk '{print $3}'`
	FSTAT=`grep " $FTYPE " ${TXT}_1  | awk '{print $2}' | head -1`

	ENTRY_STAT=`grep " $F " ${STAT_TXT}_1`
	ENTRY_DYN=`grep  " $F " ${TXT}_1`
	ENTRY_TYPE_DYN=`grep " $FTYPE " ${TXT}_1 | grep -v MISS | head -1`

	if [ "Q$ENTRY_DYN" != Q ]
	then
		echo "$ENTRY_STAT	=" >> $DP_ID.sdiff1
		sed -i -e "/$F/d" ${TXT}_1
	elif [ "Q$ENTRY_TYPE_DYN" != Q ]
	then
		echo "$ENTRY_STAT	| $ENTRY_TYPE_DYN " >> $DP_ID.sdiff1
		sed -i -e "/$FSTAT/d" ${TXT}_1
	else
		echo "$ENTRY_STAT	<" >> $DP_ID.sdiff1
	fi
done	

# b) if remaining entries in ${TXT}_1:
# first check if RASSOC files from ${STAT_TXT}_1 exist in ${TXT}_1; if so, remove them (why?)
rm -f $TMP_DIR/rm_rassoc
grep "^RASSOC" ${STAT_TXT}_1 | awk '{print $2}' | sed "s/.fits//" > $TMP_DIR/rm_rassoc
if [ -s $TMP_DIR/rm_rassoc ]
then
	for R in `cat $TMP_DIR/rm_rassoc`
	do
		sed -i -e "/$R/d" ${TXT}_1
	done
fi

# if no AB exists, or if remaining entries in ${TXT}_1:
if [ -s ${TXT}_1 ]
then
	cat ${TXT}_1 | egrep -v "SCIENCE|RASSOC" | sed "s/^.*/	> &/" >> $DP_ID.sdiff1
fi

# editing sdiff1.html
cat $DP_ID.sdiff1 |\
 awk '{if (NR < 10)  {print "00"NR,$0} else 
       if (NR < 100)  {print "0"NR,$0} else
                         {print NR,$0}}' |\
 tr "\011" " " |\
 sed "s/  */ /g" |\
 sed "s|>|GREATERTHAN|" | sed "s|<|LESSTHAN|" | sed "s|=|EQUAL|" |\
 sed "/|/s| $||" | sed "s/|/<center>|/" | sed "/|/s| |</td><td>|g" | sed "/|/s|^.*|<tr bgcolor=#FF9999><td>&</td></tr>|" |\
 sed "/GREATERTHAN/s/ $//" | sed "/GREATERTHAN/s| |</td><td bgcolor=#ff6666>|"g | sed "s|<td bgcolor=#ff6666>GREATERTHAN|<td></td><td><i>missing</td><td></td><td></td><td></td><td bgcolor=#ff6666>GREATERTHAN</td>|" | sed "/GREATERTHAN/s|^.*|<tr><td>&</td></tr>|" | sed "s/GREATERTHAN/<center>\&gt;/" |\
 sed "s|LESSTHAN.*|LESSTHAN|" | sed "/LESSTHAN/s| |</td><td>|g" | sed "/LESSTHAN/s|^.*|<tr bgcolor=#FF6666><td>&</td><td></td><td><i>missing</td><td>\&nbsp;</td>\&nbsp;</td><td>\&nbsp;</td><td>\&nbsp;</td></tr>|" | sed "s/LESSTHAN/<center>\&lt;/" |\
 sed "s|MISSING|<i>missing</i>|g" |\
 sed "/missing/s/<tr>/<tr bgcolor=#ff6666>/" |\
 sed "/EQUAL/s/ $//" | sed "/EQUAL/s| |</td><td>|g" | sed "/EQUAL/s|^.*|<tr bgcolor=#33FF33><td>&\&nbsp;</td><td bgcolor=white>\&nbsp;</td><td bgcolor=white>\&nbsp;</td><td bgcolor=white>\&nbsp;</td><td bgcolor=white>\&nbsp;</td><td bgcolor=white>\&nbsp;</td><td bgcolor=white>\&nbsp;</td><td bgcolor=white>\&nbsp;</td><td bgcolor=white>\&nbsp;</td></tr>|" | sed "s/EQUAL/<center>=/" |\
 sed "s/level2/<i>\&nbsp;\&nbsp;&<\/i>/g" |\
 sed "s/>SCIENCE<\/td><td>/><b>SCIENCE<\/td><td><b>/" |\
 sed "/auxiliary/s/>CALIB</>MASSOC</" \
>> $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html
echo "</table></html>" >> $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html

# final edits: auxiliary MCALIB files are MASSOC files; mark not archived files
sed -i -e "/auxiliary/s|MCALIB|MASSOC|" $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html
sed -i -e "/MASSOC/s/not_archived/<i>&<\/i>/" $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html
sed -i -e "/MCALIB/s/not_archived/<b>&<\/b>/" $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html

# check for certif flag in raw files
if [ -s $TMP_DIR/va_raw_calib ]
then
	for F in `cat $TMP_DIR/va_raw_calib | awk '{print $1}' | sort -u`
	do
		FLAG=`grep $F $TMP_DIR/va_raw_calib | awk '{print $2}' | sort -u`
		if [ Q$FLAG = Qfalse ]
		then
			sed -i -e "/>$F</s|$F|& FALSE|g" $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html
			sed -i -e "s|$F FALSE.fits|$F.fits FALSE|" $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html
			sed -i -e "s|FALSE|<b>uncertif</b>|g" $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html
		fi
	done
fi

# replace MCALIB by VIRTUAL if r.$DFO_FILE_NAME
sed -i -e "/r.${DFO_FILE_NAME}/s|MCALIB|VIRTUAL|" $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html

# if no AB found, replace 'missing' by 'no AB'
if [ $AB = UNKNOWN ]
then
	sed -i -e "s|<i>missing</td>|<i>no AB found</i></td>|" $DFO_AB_DIR/CALSELECTOR/$DP_ID.sdiff.html
fi

rm -f SDIFF.html ${STAT_TXT}_1 ${TXT}_1 ${TXT}_2 $DP_ID.sdiff1

# =========================================================================
# 2.4 End: create link to $DP_ID.diff.html for easy display
# =========================================================================

echo "
3. Find the following output in $DFO_AB_DIR/CALSELECTOR:
- XML and TXT file of the calSelector association as $DP_ID.xml | .txt
- the association log as $DP_ID.log.html
- the AB content as ${DP_ID}_stat.txt
- the comparison file $DP_ID.sdiff.html
- click 'SDIFF' on the dfoMonitor, or point your browser to $DFO_AB_DIR/CALSELECTOR/SDIFF.html, and inspect the difference file.
"
cp ${DP_ID}.sdiff.html SDIFF.html

exit
