#!/bin/sh
# PURPOSE: 	calChecker: checks the completeness of calibrations 
# AUTHOR:       Reinhard Hanuschik and Wolfgang Hummel / ESO
#		Complete history under http://www.eso.org/~qc/dfos/history/history_calChecker.html
# VERSIONS:     1.0 -- (2006-05-15)                          
#		2.0 -- incremental mode; stabilized against loss of navbar (2010-08-31)
#		3.0 -- include CAL4CAL checks; result pages: tablesorter replacing OB list and tags; wrapper extended to dataclient;  auto-refresh on/off (2011-01-19)
#		4.0 -- implementing LOST case (configured nighttime calibrations after end of night); more compact setup tables (2013-09-19)
#		4.0.1- mark tellTracker issue (unrecoverable tellurics from last night); new TRANSFER_URL (2013-10-10)
#		4.0.2- bug fix for tellTracker issue marker (2013-10-17)
#		4.1 -- stargate1 replaced by $DFO_WEB_SERVER (2013-10-30)
#		4.2 -- memory for open issues beyond 7 days, until N_VCAL_MEM; needs getObInfo v1.2.4 (2013-11-14)
#		4.2.1- N_VCAL_MEM_EX expanded, to include longest validity (for FULL mode) (2013-12-05)
#		4.2.2- minor bug fixed for open-issue memory (2013-12-09)
#		4.2.3- replace 'switch' by 'change' in awk commands (for OS upgrade) (2014-02-17) 
#		4.3 -- PREVIOUS directory now under $DFO_MON_DIR/CALCHECK; dataclient replaced by HdrDownloader (2014-04-28)
#		4.3.1- final migration to HdrDownloader (2014-05-22)
#		4.3.2- bug fix in :4038 [bwolff] (2014-09-23)
#		4.3.3- added ServerAliveInterval to scp parameters (2014-10-21)
#		4.3.4- bug fix in line :5725 (2015-01-12)
#		4.4 -- introducing preliminary comments for the analysis editor (2015-01-26)
#		4.4.1- bug fix for CHECK_C4C=NO (2015-03-05) [BWo]
#		4.4.2- -o Timeout removed; tell_warn.html implemented for the navbar (2015-03-23)
#		4.5 -- new result line for RAWDISP links; link to 'how_to_execute' changed, edit jobs 0.13.4 obsolete (2015-07-23)
#		4.5.1- filtered out PROG_ID=Maintenance (2015-09-21)
#		4.6 -- php script for hiding bad calibrations; forced mode; managing outdated history (2015-11-02)
#		4.6.1- optional PGI_FILTER to fine-tune suppression of pseudo-SCIENCE (2015-11-15)
#		4.6.2- prepared for P100 (0060) (2017-03-06)
#		4.6.3- removed obsolete NLT_APPLICABLE (2017-07-21)
#		4.6.4- updated BANDWIDTH_URL (2017-12-07)
#		4.6.5- CAL added to labels 'product quality' (2017-12-08)
#		4.6.6- modified: :2894 no sourcing of qcrc and dfosrc (2018-06-16)
#		4.6.7- line :5445 'marquee' disabled, incompatible with browser (2018-09-06)
#		4.6.8- line :2371 'grep -m 1' in order to handle case with duplicated files in CAL_BADQUAL (2022-08-03)
#
# PARAMETERS:	none
# CONFIG:	config.calChecker under $DFO_CONFIG_DIR/CALCHECK
#		customized OCA configuration under $DFO_CONFIG_DIR/CALCHECK/OCA
#		config.processQC for RAWDISP_SUPPORT
# TOOLS CALLED:	createAB, ABbuilder; createReport, dfosCron -t HdrDownloader
# OUTPUT:	HTML output under $DFO_MON_DIR/CALCHECK (also exported to internal QC monitor site)
# COMMENTS:	option -n should  be used for testing (no SCP to web)
#		dataTables: java code based on the jQuery library, from http://datatables.net. Find more in 'getStatusAB'.
#		"General news:" come from www.eso.org/ALL/CC_POSTIT. This news file becomes visible on all calChecker
#		interfaces. 
#
#	tags: <!--FILTER--> marks content to be removed for the w1/2 version
#	      <!--QCWEB-->  same, for qcweb version
#	      <!--LOCAL-->  same, for local version
#
# NOTES:	MIRROR_WEB_SERVER: the main pages are scp'ed there, all others passively replicated from $DFO_WEB_SERVER
#		The scp is on a best-effort basis. It has a timeout protection for the network, plus for the server. Error messages are suppressed.
#		If LOST+MISS: col. LOST plus col. 'Calib. action'
#		The tool includes the code for the php scripts that are used on the web server to manage the comment editing (calEditor)
#		and the bad quality marker (calBadQual).
#		0060.A checks removed, this never came.
# ======================================================================
TOOL_VERSION="4.6.8"
TOOL_NAME="calChecker"

# =========================================================================
# 0. Initialize
# 0.1 some hard-coded settings
# =========================================================================

# for createReport wrapper (documented under createWrapper)
SLEEP=10	#check time: every SLEEP sec we check again up to $SLEEP_MAX
SLEEP_MAX=200	#max waiting time: if HdrDownloader takes longer than that, we kill it

# web and URLs
INSTR_DIR="/home/qc/public_html/${DFO_INSTRUMENT}" # on $DFO_WEB_SERVER
QCBIN_DIR="/home/qc/bin"        		   # on $DFO_WEB_SERVER

QC_WEB_SERVER_URL="http://qcweb.hq.eso.org"	   # QC web server
PUB_QC_WEB_SERVER_URL="http://www.eso.org/qc"	   # public web server (HC site)	

# WEB_DIR for direct ingestion into w1/w2
WEB_DIR="/home/qc/qc/${DFO_INSTRUMENT}/reports"		# on $DFO_WEB_SERVER
ALL_DIR="/home/qc/qc/ALL"				# on $DFO_WEB_SERVER
UPDATE_DIR="${ALL_DIR}/common/update"			# on $DFO_WEB_SERVER

MIRROR_WEB_DIR="/home/qc_garching/dfo/quality/${DFO_INSTRUMENT}/reports"	#on $MIRROR_WEB_SERVER
MIRROR_UPDATE_DIR="/home/qc_garching/dfo/quality/ALL/common/update"		#on $MIRROR_WEB_SERVER

REL_URL="/observing/dfo/quality/${DFO_INSTRUMENT}/reports"
QC_URL="/observing/dfo/quality/ALL"	#URL for CALCHECK help site
IMG_URL="http://www.eso.org/observing/dfo/quality/ALL/img" #URL for images
IMG_URL1="/observing/dfo/quality/ALL/img" #same, relative
DFOS_URL="http://www.eso.org/~qc/dfos"	#URL for DFOS tools
HCHELP_URL="${QC_URL}/HC_help.html"	#URL for HC help (needed for the exported result table)
JSCRIPT_URL="http://www.eso.org/observing/dfo/quality/ALL/jscript"      #URL for java scripts

CMAP_URL="/observing/dfo/quality/ALL/OCA/CALCHECKER"  #URL for calibmaps

TRANSFER_URL="http://achmp.hq.eso.org/datatransfermonitor/networktransfer.html?filenamePattern=${DFO_FILE_NAME}&status=all&fileType=fits&nrResults=all"	
								# monitor to follow up status of files

EVALSO_URL="http://monkey.sc.eso.org/mrtg/mrtg2017"
BANDWIDTH_URL="${EVALSO_URL}/10.0.0.11_gigabitethernet0_0_1.html"      # DTS   PAR->VIT #1

NLT_URL="http://www.eso.org/gnlt/qcReport/instrumentReport"     # URL for NLT

HOW_TO_URL="http://sciops.pl.eso.org/wiki/index.php/${DFO_INSTRUMENT}_CalChecker_howtoexecute"	# URL for "how_to_execute" file

TOP="<p><a href=#top title=\"top\"><img src=\"$IMG_URL/arr1red-up.gif\" border=0 height=10 width=10></a>"
BOTTOM="<p><a href=#bottom title=\"bottom\"><img src=\"$IMG_URL/arr1red-down.gif\" border=0 height=10 width=10></a>"

# hard-coded tool configuration
HOUR_DIFF=2		# limit to become outdated
AUTO_REFRESH=300	# browser auto-refresh in sec
DEFAULT_NVCAL=7		# default value for N_VCAL_LIST
UT_LIMIT=11		# after $UT_LIMIT hrs, a missing calibration is considered LOST (see lines with UT_NOW)

HEADER="header_CAL.html"
NAVBAR="navbar_CAL.html"	# edit for quick links

STYLES="${QC_URL}/common/qc_eso.style"

# REPORT_DIR etc.: if on DFO_WEB_SERVER: no interactive functions
REPORT_DIR="$DFO_LST_DIR/REPORT"
REPORT_DIR2="${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/REPORT"

NR_DIR=${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/NIGHTLOG

AB_DIR=${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/logs
MON_DIR=${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/monitor

# HTML colors
GRY="#CCCCCC"	# grey
GRN="#33FF33"   # green

LGRN="#99FF99"	# light-green
RED="#FF6666"	# red
LRED="#FF9999"	# light-red
YLW="#FFFF00"	# yellow
LYLW="#FFFF99"	# light-yellow
LBLU="#66CCFF"	# light-blue

TITLE_COLOR="#006A9D" 	#eso-blue
HDRSORT_COLOR="#8dbdd8" #lightblue

REPORT_COLOR="#FF6600" 	#orange
PRODUCT_COLOR="#66CCFF"	#light-blue

SCI_COLOR="#FFFF80" 	#light-yellow: color for nights with science
NOSCI_COLOR="#CCCCCC" 	#grey: color for nights w/o science
OLD_SCI_COLOR="#FFFFEE" #XXlight-yellow:color for old nights with science issue

SM_COLOR="#006A9D"  	#eso-blue:  color for SM nights
VM_COLOR="#666666"  	#dark_grey: color for VM nights

LOST_COLOR="#3366FF"	#dark_blue:  color for LOST calibs

# watcher
OK_COL="#33FF33"
OK_GIF="ok.gif"
NOK_COL="#FF6666"
NOK_GIF="nok.gif"

# cells 
OK_COLOR=$GRN
NOK_COLOR=$YLW
MISS_COLOR=$RED

# =========================================================================
# 0.2 check for directory variables which should be set globally in .dfosrc
# =========================================================================

CHECK=`printenv | grep DFO`

if [ "Q$CHECK" = "Q" ]
then
        echo "***ERROR: DFO variables not defined. Check ~/.dfosrc and restart."
        exit -1
fi

if [ "Q$DFO_WEB_SERVER" = "Q" ]
then
        echo "*** WARNING: \$DFO_WEB_SERVER not defined, update your .dfosrc."
fi

# =========================================================================
# 0.3 initialization
# 0.3.1 read options and parameters
# =========================================================================

SCP=YES
REPORTS=YES
INCREM=YES
CAL4CAL=NO
REPAIR=NO
COPY_SSH=NO
FORCED_MODE=NO  #hidden option, for php interface

while getopts nsNhvCFId:D: OPTION
do
	case "$OPTION" in
         v ) echo $TOOL_VERSION
             exit 0 ;;
         h ) cat $DFO_DOC_DIR/calChecker.h | more
             exit 0 ;;
	 n ) SCP=NO ;;
	 C ) CAL4CAL=YES; REPORTS=NO ;;
	 N ) REPORTS=NO ;;
	 F ) INCREM=NO ;;
	 D ) N_VCAL_LIST=$OPTARG ;;
	 I ) FORCED_MODE=YES ;;
	 d ) REPAIR=YES; REPAIR_DATE=$OPTARG ;;
	 s ) COPY_SSH=YES ;;
         ? ) cat $DFO_DOC_DIR/calChecker.h | more
             exit 0 ;;
	esac
done

source $HOME/.dfosrc

if [ $INCREM = NO ] && [ $CAL4CAL = NO ]
then
	rm -f $TMP_DIR/cc_counterlog
	PID=$$

	ps -efl | grep calChecker | egrep -v "grep|vi|emacs" | awk '{print $4}' | grep -v $PID  > $TMP_DIR/cc_counterlog

	if [ -s $TMP_DIR/cc_counterlog ]
	then
		TIMESTAMP=`date +%Y-%m-%d" "%H:%M:%S`
		echo "[$TIMESTAMP]	calChecker in FULL mode: another instance(s) found, all killed." 

		for pid in `cat $TMP_DIR/cc_counterlog`
		do
			kill -9 $pid 2>/dev/null
		done
	fi
fi

# =========================================================================
# 0.3.2 initialization of ssh to $MIRROR_WEB_SERVER
# =========================================================================

if [ $COPY_SSH = YES ]
then
	echo "We send your public ssh key to $MIRROR_WEB_SERVER. This will be needed once to make 
your identity known to that machine. On the first execution, you will be prompted
for the password which is stored in $MIRROR_PWD. 
[Make sure that your ~/.ssh/config file has no 'special' entries like 'BatchMode=yes'.]

The execution will take some time since you connect to a Paranal server.
Hit return to continue ..."
	read input

	ssh-copy-id -i $HOME/.ssh/id_dsa.pub $MIRROR_WEB_SERVER

	if [ $? != 0 ]
	then
		echo "*** ERROR: An error occurred, ssh access to $MIRROR_WEB_SERVER is not possible."
	else
		echo "
Your credentials have been submitted successfully. You can now automatically scp files to 
the mirror site $MIRROR_WEB_SERVER. Calling 'calChecker -s' is not needed again.

"
	fi
	exit
fi

# =========================================================================
# 0.3.3 writeLog and scpLog 
#       a) writeLog (adapted from autoDaily)
# =========================================================================
ORIG_TMP_DIR=$TMP_DIR
FORCED_LOG=$ORIG_TMP_DIR/forcedlog
if [ $FORCED_MODE = YES ] && [ $CAL4CAL = NO ] 
then
	rm -f $FORCED_LOG
fi
LOGFILE=$DFO_MON_DIR/calChecker.log
	
writeLog(){
TIMESTAMP=`date -u +%Y-%m-%d" "%H:%M:%S`
echo "$TIMESTAMP	$1" >> $FORCED_LOG
}

# =========================================================================
#       b) scpLog (adapted from autoDaily)
# =========================================================================
scpLog(){
chmod a+w $FORCED_LOG
scp -p $FORCED_LOG ${DFO_WEB_SERVER}:${WEB_DIR}/CAL/php/calchecklog.html 1>/dev/null 2>/dev/null
}

# =========================================================================
# 0.3.4 exitProcs (adapted from autoDaily)
# =========================================================================
exitProcs(){
FLAG="$1 since automatic $TOOL_NAME of $DFO_INSTRUMENT data on $HOSTNAME is already going on.<br>                        Please wait for the results of that automatic processing"
echo "<html> <META HTTP-EQUIV=\"Refresh\" CONTENT=\"10\">  <META HTTP-EQUIV=\"Cache-Control\" CONTENT=\"NO-CACHE\"> <pre>" > $FORCED_LOG
echo "<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30>" >> $FORCED_LOG
# CAL_ENFORCE.lock set by dfosCron
rm -f $ORIG_TMP_DIR/CAL_ENFORCE.lock

writeLog "Forced calChecker ${FLAG}."
echo "</pre> </html>" >> $FORCED_LOG
scpLog

cat $FORCED_LOG | grep -v "^<" >> $LOGFILE
rm -f $ORIG_TMP_DIR/message
}

# =========================================================================
# 0.3.5 check and set variables
# =========================================================================

CHECK_TELLTRACK=`grep "^CHECK_TELLTRACK" $DFO_CONFIG_DIR/CALCHECK/config.calChecker | awk '{print $2}'`
if [ Q$CHECK_TELLTRACK != QYES ]
then
	CHECK_TELLTRACK=NO
fi

# SUPPRESS_ABCHECK
SUPPRESS_ABCHECK=`grep "^SUPPRESS_ABCHECK" $DFO_CONFIG_DIR/CALCHECK/config.calChecker | awk '{print $2}'`
if [ Q$SUPPRESS_ABCHECK != QYES ]
then
	SUPPRESS_ABCHECK=NO
fi

# CALEDIT_PWD: needed to download calEditor.dat; should contain file name!
CALEDIT_PWD=`grep "^CALEDIT_PWD" $DFO_CONFIG_DIR/CALCHECK/config.calChecker | awk '{print $2}'`
if [ Q$CALEDIT_PWD = Q ]
then
	echo "***ERROR: \$CALEDIT_PWD not configured, calEditor.dat cannot be managed." > $TMP_DIR/mail
	cat $TMP_DIR/mail
	mail -s "calChecker error" $OP_ADDRESS <$TMP_DIR/mail 
elif [ ! -s $DFO_BIN_DIR/$CALEDIT_PWD ]
then
	echo "***ERROR: \$CALEDIT_PWD configured but not existing, calEditor.dat cannot be managed." > $TMP_DIR/mail
	cat $TMP_DIR/mail
	mail -s "calChecker error: \$CALEDIT_PWD" $OP_ADDRESS <$TMP_DIR/mail 
else
	CALEDIT_PWD=`cat $DFO_BIN_DIR/$CALEDIT_PWD`
fi	

CHECK_C4C=NO
if [ $CAL4CAL = YES ]
then
	if [ -d $DFO_CONFIG_DIR/CALCHECK ]
	then
		CHECK_C4C=`grep "^CHECK_C4C" $DFO_CONFIG_DIR/CALCHECK/config.calChecker | awk '{print $2}'`
		if [ Q$CHECK_C4C != QYES ]
		then
			CHECK_C4C=NO
		fi
	fi
	if [ $CHECK_C4C = NO ]
	then
		echo "***ERROR: You cannot call mode -C since \$CHECK_C4C is not enabled in config.calChecker. Exit."
		exit -1
	fi
fi

if [ $FORCED_MODE = YES ]
then
	AUTHOR=`cat $TMP_DIR/CAL_ENFORCE | tail -1 | awk '{print $2}'`
	if [ Q$AUTHOR = Q ]
	then
		AUTHOR=XXX
	fi

	cat > $FORCED_LOG <<EOT
<html>
<META HTTP-EQUIV="Refresh" CONTENT="10">
<META HTTP-EQUIV="Cache-Control" CONTENT="NO-CACHE">
<pre>
EOT
fi

HC_REPORT=`cat $DFO_JOB_DIR/JOBS_HEALTH | grep -v "^#" | grep "trendPlotter" | head -1 | sed "s/^.*trendPlotter/trendPlotter/" | sed "s/-f//" | sed "s/-r//" | sed "s/-t//" | sed "s/HEALTH//" | awk '{print $2}'`
if [ "Q$HC_REPORT" = Q ]
then
	echo "***WARNING: no HC_REPORT found. Please check your \$DFO_JOB_DIR/JOBS_HEALTH."
else
	HC_REPORT="trend_report_${HC_REPORT}_HC.html"
fi

# for commissioning or Science Verification: ENABLE_60
ENABLE_60=`grep "^ENABLE_60"	$DFO_CONFIG_DIR/CALCHECK/config.calChecker | awk '{print $2}'`

if [ Q$ENABLE_60 != QYES ]
then
	ENABLE_60=NO
fi

# for additional filtering for pseudo-science
PGI_FILTER=`grep "^PGI_FILTER"	$DFO_CONFIG_DIR/CALCHECK/config.calChecker | awk '{print $2}'`

if [ Q$PGI_FILTER != Q ]
then
	if [ ! -s $DFO_BIN_DIR/$PGI_FILTER ]
	then
		echo "***WARNING: PGI_FILTER $PGI_FILTER configured but not found. Check your $DFO_CONFIG_DIR/CALCHECK/config.calChecker!"
		PGI_FILTER=""
	fi
fi

# for the 'raw' links: need to know if rawdisp support is turned off
RAWDISP_SUPPORT=`grep "^RAWDISP_SUPPORT"	$DFO_CONFIG_DIR/config.processQC | awk '{print $2}'`
if [ Q$RAWDISP_SUPPORT != QNO ]
then
	RAWDISP_SUPPORT=YES
fi

# =========================================================================
# 0.4 generate a new workspace for this tool
#     in order to avoid conflicts with operational OCA and AB creation
# =========================================================================

# in forced mode, we don't wait for the other/earlier instance but exit
if [ $FORCED_MODE = YES ] && [ $CAL4CAL = NO ]
then
	CHECK_OTHER=`ps -wfC ${TOOL_NAME} | grep -v CMD | grep $USER | wc -l`
	if [ $CHECK_OTHER -gt 2 ]
	then
		TIMESTAMP=`date -u +%Y-%m-%d" "%H:%M:%S`
		echo "$TIMESTAMP ***INFO: forced ${TOOL_NAME} call terminated since other job already started for `whoami`@`hostname`." > $TMP_DIR/auto_mail
		echo "           Please wait for results." >> $TMP_DIR/auto_mail
		mail -s "Forced ${TOOL_NAME} call terminated since other job already running" $OP_ADDRESS < $TMP_DIR/auto_mail

# CAL_ENFORCE.lock set by dfosCron
        	rm -f $ORIG_TMP_DIR/CAL_ENFORCE.lock
		exitProcs 
		exit 10
	fi

# same in incremental mode if another instance is found (FORCED)
elif [ $FORCED_MODE = NO ] && [ $INCREM = YES ] && [ $CAL4CAL = NO ]
then
	CHECK_OTHER=`ps -wfC ${TOOL_NAME} | grep -v CMD | grep $USER | wc -l`
	if [ $CHECK_OTHER -gt 2 ]
	then
		TIMESTAMP=`date -u +%Y-%m-%d" "%H:%M:%S`
		echo "$TIMESTAMP ***INFO: ${TOOL_NAME} call terminated since other job (FORCED?) already started for `whoami`@`hostname`." > $TMP_DIR/auto_mail
		echo "           Please wait for results." >> $TMP_DIR/auto_mail
		mail -s "${TOOL_NAME} call terminated since other job already running" $OP_ADDRESS < $TMP_DIR/auto_mail
		exit
	fi
fi

if [ $CAL4CAL = NO ]
then
	echo ""
	case $FORCED_MODE in
 	 "NO"  ) echo "calChecker started: `date -u`" ;;
	 "YES" ) writeLog "calChecker FORCED_MODE started for $DFO_INSTRUMENT by $AUTHOR as `whoami`@`hostname` ..." 
		 writeLog "All times in UT" 
		 ssh -o BatchMode=yes ${DFO_WEB_SERVER} "rm -f ${WEB_DIR}/CAL/php/calchecklog.html" 1>/dev/null 2>/dev/null
		 scpLog ;;
	esac
fi

# define DATA_DISK and DISK_SPACE for security check; from autoDaily
DATA_DISK=`grep "^DATA_DISK"  	$DFO_CONFIG_DIR/config.dfoMonitor |  awk '{print $2}'`
DISK_SPACE=`grep "^DISK_SPACE"	$DFO_CONFIG_DIR/config.autoDaily | awk '{print $2}' | sed "s/\%//"`
if [ Q$DISK_SPACE = Q ]
then
	DISK_SPACE=95
fi

if [ "Q$DATA_DISK" != "Q" ]
then
	DISK_FRAC=`df $DATA_DISK | grep -v File | sed "s/\%//" | awk '{printf"%3s\n",$5}'`
	if [ "$DISK_FRAC" -ge "$DISK_SPACE" ]
	then
		echo "  $DATA_DISK: occupied volume (${DISK_FRAC}%) exceeds configured threshold (${DISK_SPACE}%)."
		TIMESTAMP=`date +%Y-%m-%d" "%H:%M:%S`
		echo "$TIMESTAMP	calChecker stopped." 
		echo "[$TIMESTAMP]	calChecker: occupied volume (${DISK_FRAC}%) on $DATA_DISK exceeds configured threshold (${DISK_SPACE}%). calChecker stopped." > $TMP_DIR/cal_mail
		mail -s "calChecker stopped because of disk space problem" $OP_ADDRESS <$TMP_DIR/cal_mail
		exit	
	fi
fi

# directories
ORIG_CONFIG_DIR=$DFO_CONFIG_DIR/OCA
CONFIG_CALCHECK=$DFO_CONFIG_DIR/CALCHECK #have it here but not further down!

# to have all calChecker configuration separated from daily workflow:
case $CAL4CAL in
 "NO"  ) DFO_CONFIG_DIR=$DFO_CONFIG_DIR/CALCHECK ;;	
 "YES" ) DFO_CONFIG_DIR=$DFO_CONFIG_DIR/CALCHECK/CAL4CAL ;;
esac

if [ ! -d $DFO_CONFIG_DIR ]
then
	mkdir $DFO_CONFIG_DIR
	mkdir $DFO_CONFIG_DIR/OCA
fi

# quick info about files in $DFO_CONFIG_DIR/OCA/
if [ $CAL4CAL = YES ]
then
	cat > $DFO_CONFIG_DIR/AAREADME.txt <<EOT
This directory is used for the CAL4CAL mode of calChecker.
- config.calChecker: is just an automatic copy of the same
  file in $CONFIG_CALCHECK
- info.calChecker: to be edited by you!

This file was created by calChecker.
EOT

	cat > $DFO_CONFIG_DIR/OCA/AAREADME.txt <<EOT
This directory is used for the CAL4CAL mode of calChecker.
It is largely filled automatically. The only files maintained by you are:
- ${DFO_INSTRUMENT}_classification.h
- ${DFO_INSTRUMENT}_organisation.h
- ${DFO_INSTRUMENT}_association.h
Note that in particular long_${DFO_INSTRUMENT}_classification.h is just an
automatic copy of the same file in $CONFIG_CALCHECK/OCA.

This file was created by calChecker.
EOT
fi

# technical (maintenance) calibrations to be monitored (optional)
LONG_CLASS=$CONFIG_CALCHECK/OCA/long_${DFO_INSTRUMENT}_classification.h

if [ $CAL4CAL = YES ]
then
	cp $CONFIG_CALCHECK/config.calChecker $DFO_CONFIG_DIR/
	if [ -s $LONG_CLASS ]
	then
		cp $LONG_CLASS $DFO_CONFIG_DIR/OCA/
	fi
fi

if [ ! -s $DFO_CONFIG_DIR/config.calChecker ]
then
        echo "*** ERROR: No configuration file $DFO_CONFIG_DIR/config.calChecker found. Check and re-start."
        exit -1
fi

ORIG_AB_DIR=$DFO_AB_DIR
case $CAL4CAL in
 "NO"  ) DFO_AB_DIR=$DFO_AB_DIR/CALCHECK ;;
 "YES" ) DFO_AB_DIR=$DFO_AB_DIR/CALCHECK/CAL4CAL ;;
esac

if [ ! -d $DFO_AB_DIR ]
then
	mkdir $DFO_AB_DIR
fi

ORIG_MON_DIR=$DFO_MON_DIR
case $CAL4CAL in
 "NO"  ) export DFO_MON_DIR=$DFO_MON_DIR/CALCHECK ;;
 "YES" ) export DFO_MON_DIR=$DFO_MON_DIR/CALCHECK/CAL4CAL ;;
esac

if [ ! -d $DFO_MON_DIR ]
then
	mkdir $DFO_MON_DIR
fi

PREVIOUS_DIR=$DFO_MON_DIR/PREVIOUS

case $CAL4CAL in
 "NO"  ) export TMP_DIR=$TMP_DIR/CALCHECK ;;
 "YES" ) export TMP_DIR=$TMP_DIR/CALCHECK/CAL4CAL ;;
esac

if [ ! -d $TMP_DIR ]
then
	mkdir $TMP_DIR
fi

case $CAL4CAL in
 "NO"  ) DFO_CAL_DIR=$DFO_CAL_DIR/CALCHECK ;;
 "YES" ) DFO_CAL_DIR=$DFO_CAL_DIR/CALCHECK/CAL4CAL ;;
esac

if [ ! -d $DFO_CAL_DIR ]
then
	mkdir $DFO_CAL_DIR
fi

# final place for daily results
case $CAL4CAL in
 "NO"  ) CALREPORT_DIR=$DFO_LST_DIR/CALCHECK 
	 CALREPORT_DIR2="${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/CALCHECK" ;;
	 
 "YES" ) CALREPORT_DIR=$DFO_LST_DIR/CALCHECK/CAL4CAL
	 CALREPORT_DIR2="${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/CALCHECK/CAL4CAL" ;;
esac

if [ ! -d $CALREPORT_DIR ]
then
	mkdir $CALREPORT_DIR
fi

# log
LOGFILE=$DFO_MON_DIR/calChecker.log
if [ $INCREM = NO ] && [ $CAL4CAL = NO ]
then
	TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
	echo "[$TIMESTAMP] calChecker started" > $LOGFILE
	echo "- INCREM=NO: full run, all $DEFAULT_NVCAL dates are scanned ..." | tee -a $LOGFILE
elif [ $INCREM = YES ] && [ $CAL4CAL = NO ]
then
	TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
	echo "[$TIMESTAMP] calChecker started" > $LOGFILE
	echo "- INCREM=YES: incremental run ..." | tee -a $LOGFILE
	if [ $FORCED_MODE = YES ]
	then
		writeLog "This is an incremental run (we scan only open issues, NOK/MISS dates, and TODAY)."
		scpLog
	fi
else
	TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
	echo "[$TIMESTAMP] calChecker CAL4CAL started" > $LOGFILE
fi

# check for configured MODIF_SETUP keys
CHECK_MODIF=`grep "^MODIF_SETUP[[:space:]]" $DFO_CONFIG_DIR/config.calChecker | sed "s/\&\&$//" | sed "s/^.*\&\&//" | head -1`

if [ "Q$CHECK_MODIF" != "Q" ]
then
	rm -f $TMP_DIR/cc_modif_string
	grep "^MODIF_SETUP[[:space:]]" $DFO_CONFIG_DIR/config.calChecker | sed "s/\&\&$//" | sed "s/^.*\&\&//" | tr "\012" "|" | sed "s/|$//" > $TMP_DIR/cc_modif_string
	chmod u+x $TMP_DIR/cc_modif_string
	APPLY_MODIF=TRUE
	if [ $CAL4CAL = NO ]
	then
		echo "- MODIF_SETUP configured, (some) setup keys will be modified!"
	fi
else
	APPLY_MODIF=FALSE
fi

HTML="$DFO_MON_DIR/calChecker_${DFO_INSTRUMENT}.html"

# =========================================================================
#  0.5 read configuration file config.calChecker
# =========================================================================

# -D <N_VCAL_LIST> can be passed on the command-line, for tests
if [ Q$N_VCAL_LIST = Q ]
then
	N_VCAL_LIST=$DEFAULT_NVCAL

# check that N_VCAL_LIST is a reasonable number (<100)
else
	CHECK_N=`echo $N_VCAL_LIST | cut -c 1-2`
	if [ Q$CHECK_N != Q$N_VCAL_LIST ]
	then
		echo "***ERROR: N_VCAL_LIST = $N_VCAL_LIST doesn't seem to be a reasonable choice, must be a number <100. Exit."
		exit -1
	fi	
fi

if [ $N_VCAL_LIST != $DEFAULT_NVCAL ] && [ $CAL4CAL = NO ]
then
	echo "***WARNING: You have chosen \$N_VCAL_LIST=$N_VCAL_LIST.
For automatic mode, use \$N_VCAL_LIST = $DEFAULT_NVCAL (per QC policy). Anything else should only be used interactively (for testing)." > $TMP_DIR/message
	cat $TMP_DIR/message
	echo ""

	mail -s "calChecker warning" $OP_ADDRESS <$TMP_DIR/message
fi

if [ $SCP = NO ] && [ $CAL4CAL = NO ]
then
	echo "***WARNING: The output will not be transferred to the web servers. Hit return:"
	read input
fi

CHECK_C4C=`grep "^CHECK_C4C"  	$DFO_CONFIG_DIR/config.calChecker | awk '{print $2}'`
if [ "Q$CHECK_C4C" != "QYES" ]
then
	CHECK_C4C=NO
fi

N_VCAL_MEM=`grep "^N_VCAL_MEM" 	$DFO_CONFIG_DIR/config.calChecker | awk '{print $2}'`
if [ "Q$N_VCAL_MEM" = "Q" ] && [ $CAL4CAL = NO ]
then
	N_VCAL_MEM=`echo $N_VCAL_LIST | awk '{print $1*2}'`
	echo "*** WARNING: \$N_VCAL_MEM not defined in $DFO_CONFIG_DIR/config.calChecker. Set to 2*\$N_VCAL_LIST = $N_VCAL_MEM."
fi

PSO_EMAIL=`grep "^PSO_EMAIL"		$DFO_CONFIG_DIR/config.calChecker | awk '{print $2}'`
if [ Q$PSO_EMAIL = Q ] && [ $CAL4CAL = NO ]
then
	echo "***WARNING: no PSO_EMAIL found in configuration file. Pls. upgrade! You can't send emails to PSO without it." > $TMP_DIR/message
	cat $TMP_DIR/message
	echo ""

	mail -s "calChecker warning" $OP_ADDRESS <$TMP_DIR/message
fi

if [ Q$DFO_EDITOR = Q ]
then
	echo "***WARNING: you need to define \$DFO_EDITOR (e.g. vi, vim, emacs) in .dfosrc, then source and start again." > $TMP_DIR/message
	cat $TMP_DIR/message
	echo ""
	mail -s "calChecker warning" $OP_ADDRESS <$TMP_DIR/message
fi

INFO_NAME=`grep "^INFO_NAME"		$DFO_CONFIG_DIR/config.calChecker | awk '{print $2}'`
XTERM_GEOM=`grep XTERM_GEOM             $DFO_CONFIG_DIR/config.calChecker | grep -v "^#" | awk '{print $2}'`

CFG_EDITOR=`eval "echo $DFO_EDITOR $DFO_CONFIG_DIR/config.calChecker"`
POSTIT_EDITOR=`eval "echo $DFO_EDITOR $DFO_MON_DIR/CAL_POSTIT"`
MAIL_EDITOR=`eval "echo $DFO_EDITOR $TMP_DIR/pso_mail"`

# =========================================================================
#  0.6 procedures
#  0.6.1 cfgParser: create table pro.catg vs. raw.type
# =========================================================================

cfgParser(){
rm -f $TMP_DIR/list_actions $TMP_DIR/list_rawpro

# parse $OCA_ASSOC to find actions
cat $OCA_ASSOC |\
awk '{
  if ( $1 == "action" ) { change=1; action=$2 }
  if ( $1 == "product" && change == 1 ) { print action,$2 ; change=0 }
     }' > $TMP_DIR/list_actions

# extract PRO_CATG and RAW_TYPE from $OCA_ORGAN, for later mapping
for A in `cat $TMP_DIR/list_actions | awk '{print $1}'`
do
        PRO_CATG=`grep "^$A[[:space:]]" $TMP_DIR/list_actions | awk '{print $2}'`
        RAW_TYPE=`grep "execute(${A})" $OCA_ORGAN | sed "s/^.*RAW.TYPE==\"//" | sed "s/\"//g"`
        echo "$RAW_TYPE $PRO_CATG" >> $TMP_DIR/list_rawpro
done
}

# =========================================================================
# 0.6.2 squeeze: procedure to remove redundant strings in $1
# and to replace blanks by '|'
# =========================================================================
squeeze(){
CHECK_ACT1=`echo $1 |\
  tr " " "\012" |\
  sed "s/:.*//" |\
  sort -u |\
  tr "\012" "|" |\
  sed "s/|$//"  |\
  sed "s/|/ | /g" |\
  sed "s/^.*/&\n/"`
echo "$CHECK_ACT1"
}

# =========================================================================
# 0.6.3 getRawTypes: extracts config info from $OCA_CLASS, for display 
# =========================================================================

getRawTypes(){
RAWTYPES=rawtypes_${DFO_INSTRUMENT}.txt
rm -f $TMP_DIR/list_rawt* $TMP_DIR/rawtypes*
case $CAL4CAL in
 "NO"  ) LIST_RAWT=$TMP_DIR/list_rawt ;;
 "YES" ) LIST_RAWT=$TMP_DIR/list_rawt1 ;;
esac

# parse $OCA_CLASS, to display raw types and assoc rules in rawtypes_<ins>.txt
# CAL4CAL=YES: copy the depending CALIBs from CALCHECK/$OCA_CLASS
if [ $CAL4CAL = YES ]
then
	cp $TMP_DIR/../list_rawt $TMP_DIR/list_rawt
fi

cat $OCA_CLASS |\
 awk '{
  if ( change == 0 && $2 ~ "DPR.CATG==" ) { change=1; cond=$0 }
  if ( change == 1 && $1 == "RAW.TYPE" )  { change=0; rawt=$3; printf"%15s %30s\n",rawt,cond }
     }' > $LIST_RAWT

# for CAL4CAL: find triggering CALs (the ones with CATG=SCIENCE) and depending CALs (CATG=CALIB)
if [ $CAL4CAL = YES ]
then
	cat $OCA_CLASS |\
 awk '{
  if ( change == 0 && $1 ~ "RAW.TYPE" ) { change=1; cond=$3 }
  if ( change == 1 && $1 == "CATG" )    { change=0; rawt=$3; printf"%15s %30s\n",rawt,cond }
     }' > $TMP_DIR/list_rawt_cal4cal
fi

case $CAL4CAL in
 "NO"  ) CALTEXT="calChecker rules applied for $DFO_INSTRUMENT (calibrations for science)" 
 	 CALTEXT1="1. List of CALIB raw data types, used in the calChecker:"
 	 CALTEXT2="2. List of SCIENCE raw data types, used in the calChecker:" ;;
 "YES" ) CALTEXT="calChecker rules applied for $DFO_INSTRUMENT (calibrations for calibrations)" 
 	 CALTEXT1="1. List of depending CALIB raw data types, used for CAL4CAL:"
 	 CALTEXT2="2. List of triggering CALIB raw data types, used for CAL4CAL:" ;;
esac

# 1. List of CALIBs/depending CALIBs
if [ $CAL4CAL = "NO" ]
then
	cat $TMP_DIR/list_rawt |\
 grep -v "DPR.CATG==\"SCIENCE\"" |\
 sed "s/\"//" |\
 sed "s/\"//" |\
 sed "s/;/:/" |\
 sed "s/then//" |\
 sed "s/if//" > $TMP_DIR/list_rawt1
	cat $TMP_DIR/list_rawt1 >> $TMP_DIR/rawtypes1
else
	rm -f $TMP_DIR/cal4cal2
	cat $TMP_DIR/../list_rawt1 >> $TMP_DIR/rawtypes1

	grep "SCIENCE" $TMP_DIR/list_rawt_cal4cal | sort -u > $TMP_DIR/cal4cal2
	for SCI in `cat $TMP_DIR/cal4cal2 | awk '{print $2}' | sed "s/\"//g" | sed "s/;//"`
	do
		sed -i -e "/${SCI}:/d" $TMP_DIR/rawtypes1
	done
fi

# 2. List of SCIENCE/triggering CALIBs
cat > $TMP_DIR/rawtypes2 <<EOT

$CALTEXT2
=============================================================

EOT

if [ $CAL4CAL = NO ]
then
	cat $LIST_RAWT |\
 grep  "DPR.CATG==\"SCIENCE\"" |\
 sed "s/\"//" |\
 sed "s/\"//" |\
 sed "s/;/:/" |\
 sed "s/then//" |\
 sed "s/if//" >> $TMP_DIR/rawtypes2
else
	cat $LIST_RAWT |\
 sed "s/\"//" |\
 sed "s/\"//" |\
 sed "s/;/:/" |\
 sed "s/then//" |\
 sed "s/if//" > $TMP_DIR/list_rawt2
	for SCI in `cat $TMP_DIR/cal4cal2 | awk '{print $2}' | sed "s/\"//g" | sed "s/;//"`
	do
		grep "$SCI" $TMP_DIR/list_rawt2 >> $TMP_DIR/rawtypes2
	done
fi

# 3. Association rules for SCIENCE/CAL4CAL
if [ $CAL4CAL = NO ]
then
	cat >> $TMP_DIR/rawtypes2 <<EOT

3. Association rules for SCIENCE, used in the calChecker:
 col1: SCIENCE raw data type (defined in 2.)
 col2: CALIB raw data type, required for calibration (defined in 1.)
 col3: validity time range in days used for checking
=============================================================

EOT
else
	cat >> $TMP_DIR/rawtypes2 <<EOT

3. Association rules for CAL4CAL, used in the calChecker:
 col1: triggering CALIB raw data type (defined in 2.)
 col2: depending  CALIB raw data type, required for calibration (defined in 1.)
 col3: validity time range in days used for checking
=============================================================

EOT

fi

case $CAL4CAL in
 "NO"  ) echo "SCIENCE CALIB DeltaT/d" |        awk '{printf"%-10s %-30s %4s\n",$1,$2,$3}'  >> $TMP_DIR/rawtypes2 ;;
 "YES" ) echo "trig_CALIB dep_CALIB DeltaT/d" | awk '{printf"%-10s %-30s %4s\n",$1,$2,$3}'  >> $TMP_DIR/rawtypes2 ;;
esac
echo "--------------------------------------------------" >> $TMP_DIR/rawtypes2

rm -f $TMP_DIR/list_rawt_sci
if [ $CAL4CAL = NO ]
then
	cat $TMP_DIR/list_rawt |\
 grep "DPR.CATG==\"SCIENCE\"" |\
 sed "s/\"//" |\
 sed "s/\"//" |\
 sed "s/;//" |\
 awk '{print $1}' | sort -u > $TMP_DIR/list_rawt_sci
else
	cat $TMP_DIR/cal4cal2 | awk '{print $2}' | sed "s/\"//g" | sed "s/;//" > $TMP_DIR/list_rawt_sci
fi

rm -f $TMP_DIR/list_dep_CALIB $TMP_DIR/list_master_all_cals
for RTSCI in `cat $TMP_DIR/list_rawt_sci`
do
	rm -f $TMP_DIR/list_master_cals
	cat $OCA_ASSOC | grep "^// ACTION_" | awk '{print $2,$3,$4}' | sed "s/ACTION_//" | grep "^$RTSCI " | sed "s/DELTAT_RULE=//" | sort -u > $TMP_DIR/list_master_cals 
	cat $TMP_DIR/list_master_cals >> $TMP_DIR/list_master_all_cals

	for M in `cat $TMP_DIR/list_master_cals | awk '{print $2}'`
	do
		RAWTYPE=`cat $TMP_DIR/list_rawpro | awk '{ if ( $2 == m ) { print $1 }}' m=$M`
		if [ "Q$RAWTYPE" = "Q" ]
		then
			echo "***WARNING: $M has no corresponding raw data type in $OCA_CLASS."
			continue
		fi
		M1=`grep " $M " $TMP_DIR/list_master_cals | awk '{print $1}'`
		M3=`grep " $M " $TMP_DIR/list_master_cals | awk '{print $3}'`
		echo "$M1 $RAWTYPE $M3" | awk '{printf"%-10s %-30s %4s\n",$1,$2,$3}'  >> $TMP_DIR/rawtypes2
		echo "$RAWTYPE"  >> $TMP_DIR/list_dep_CALIB
	done
	echo "" >> $TMP_DIR/rawtypes2
done

# assemble $RAWTYPES
cat > $DFO_MON_DIR/$RAWTYPES <<EOT
$CALTEXT
====================================================================

$CALTEXT1
============================================================

EOT

# consolidate $TMP_DIR/rawtypes1 for the CALIBs actually needed
if [ -s $TMP_DIR/list_dep_CALIB ]
then
	for CAL in `cat $TMP_DIR/list_dep_CALIB | sort -u`
	do
		egrep "^${CAL}:| ${CAL}:" $TMP_DIR/rawtypes1 >> $DFO_MON_DIR/$RAWTYPES
	done
fi
cat $TMP_DIR/rawtypes2 >> $DFO_MON_DIR/$RAWTYPES
}

# =========================================================================
# 0.6.4 procedure jCheckHealth: javascript to calculate elapsed time
#       (from ngasMonitor) 
#       $1: TIMESTAMP of last update
#	$2: switch to have font_size=1 or 2
# =========================================================================

jCheckHealth(){
YEAR_TS=`echo  $1 | cut -c1-4`
MONTH_TS=`echo $1 | cut -c6,7 | awk '{print $1-1}'`
DAY_TS=`echo   $1 | cut -c9,10`
HOUR_TS=`echo  $1 | cut -c12,13`
MIN_TS=`echo   $1 | cut -c15,16`
SEC_TS=`echo   $1 | cut -c18,19`

if [ $2 = 2 ]
then
	GIF_WIDTH=14
	GIF_HEIGHT=15
else
	GIF_WIDTH=10
        GIF_HEIGHT=9
fi

OK_TAG=" <td bgcolor=#CCCCCC width=1%><img src=${IMG_URL}/${OK_GIF}  width=$GIF_WIDTH height=$GIF_HEIGHT border=0></td>"
NOK_TAG="<td bgcolor=#CCCCCC width=1%><img src=${IMG_URL}/${NOK_GIF} width=$GIF_WIDTH height=$GIF_HEIGHT border=0></td>"
OK_LABEL=" <td nowrap width=1% bgcolor=${OK_COL} ><font size=$2 color=${TITLE_COLOR}><b>"
NOK_LABEL="<td nowrap width=1% bgcolor=${NOK_COL}><font size=$2 color=#FFFFFF><b>"

rm -f $TMP_DIR/cal_checkhealth
cat > $TMP_DIR/cal_checkhealth <<EOT

<SCRIPT language="JavaScript"><!--
var today = new Date();
var Now = today.getTime() / 1000 ;
var tzdiff = today.getTimezoneOffset();

var lastTimeStamp = new Date($YEAR_TS, $MONTH_TS, $DAY_TS, $HOUR_TS, $MIN_TS, $SEC_TS);
var TimeStamp = lastTimeStamp.getTime() / 1000 - tzdiff*60;

var dayDiff = Math.floor((Now-TimeStamp) / 86400 );
var hourDiff = Math.floor(((Now-TimeStamp) - dayDiff*86400)/3600);
var hourDiff2 = ((hourDiff < 10) ? "d 0" : "d ");
var minDiff = Math.floor(((Now-TimeStamp) - dayDiff*86400 - hourDiff*3600)/60);
var minDiff2 = ((minDiff < 10) ? "h:0" : "h:");

if ( dayDiff < 1 && hourDiff < $HOUR_DIFF ) {
qc_tag   = "${OK_TAG}" ;
qc_label = "${OK_LABEL}" ;
}
else
{
qc_tag   = "${NOK_TAG}" ;
qc_label = "${NOK_LABEL}" ;
}

document.write(qc_label, "<a title=\"green if last update within $HOUR_DIFF hr\">(", dayDiff, hourDiff2, hourDiff, minDiff2, minDiff,"m ago)</a></b></td>", qc_tag);
//--></SCRIPT>
<NOSCRIPT><font size=1 color=#FF6666>[enable javascript for checking process health!]</font></NOSCRIPT>

EOT
}

# =========================================================================
# 0.6.5 javascript to display complete OB comments
# =========================================================================

cat > $TMP_DIR/cal_obcomments <<EOT
<!-- java script for OB comments -->
<SCRIPT LANGUAGE="JavaScript" TYPE="text/javascript">
<!--
OBcomm = null;

document.onmousemove = updateOBcomm;

function updateOBcomm(e) {
	x = (document.all) ? window.event.x + document.body.scrollLeft : e.pageX;
	y = (document.all) ? window.event.y + document.body.scrollTop  : e.pageY;
	if (OBcomm != null) {
		OBcomm.style.left = (x + 17) + "px";
		OBcomm.style.top  = (y + 17) + "px";
	}
}

function showOBcomm(id) {
	OBcomm = document.getElementById(id);
	OBcomm.style.display = "block"
}

function hideOBcomm() {
	OBcomm.style.display = "none";
}
-->
</script>
EOT

# =========================================================================
# 0.6.6 javascript to control browser refresh (code from getStatusAB)
# =========================================================================

cat > $TMP_DIR/offlink <<EOT
<!-- java script for switching off/on the refresh link-->
<script type="text/javascript">
function OffLink () {
  if (document.getElementById("refresh_link").firstChild.nodeValue = "[page auto-refreshes after $AUTO_REFRESH sec]") {
      document.getElementById("refresh_link").firstChild.nodeValue = "[auto-refresh turned off, click \'on\' to resume] ";
      document.getElementById("refresh_link").className = "off";
} else {
      document.getElementById("refresh_link").firstChild.nodeValue = "on";
      }
}
</script>

<!-- style for switching off the refresh -->
<style type="text/css">
.off  { color:red; }
</style>

EOT

# =========================================================================
# 0.6.7 generate calEditor php scripts
#       calAnalysis.php and calEditor.php
# =========================================================================
createCalEditor(){

# calEditor.php
cat > $TMP_DIR/calEditor.php <<EOT
<html>
<head>
<style type="text/css">
.esocal {
  background-color: ${REPORT_COLOR};
  color: #FFF;
  font-size: large;
}
.nounder {
  text-decoration: none;
}
</style>
</head>
<?php
/*
This php script is created by calChecker v$TOOL_VERSION .
It analyzes the input from calAnalysis.php and updates calEditor.dat.
The analyzed lines are replaced. 
*/

// 1. check if form elements are properly filled:
// 1.1 cases: at least one must be selected (we need to remove \r and \n for reasons not fully understood...)
\$i = 1 ;
\$test_empty=="";
while (\$i <= 100):
  \$case = "case\$i" ;
  \$analysisCase = str_replace("\r","",\$_POST["\$case"]);
  \$analysisCase = str_replace("\n","",\$analysisCase);
  if (\$i == "1")
    \$analysisFlag = \$analysisCase;
  \$test_empty = \$test_empty.\$analysisCase ;
  \$i++;
endwhile;

if (\$test_empty=="") {
  die ('<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30> <font size=2>You have to select at least one <b>case</b>. <br><br><a href="calAnalysis.php" class="nounder"><input type="button" value="back to form" class="back"></a></font>');
}

// 1.2 editedFlag: must be set
if (\$_POST['editedFlag']=='empty') {
  die ('<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30> <font size=2>You have to enter a new analysis <b>flag</b>. <br><br><a href="calAnalysis.php" class="nounder"><input type="button" value="back to form" class="back"></a></font>');
}

// 1.3 valid comment must be provided
\$analysisComment = str_replace("<","&lt;",\$_POST[analysisComment]);
if (\$analysisComment=='&lt;your analysis comment>'||\$analysisComment==''||\$analysisComment==' ') {
  die ('<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30> <font size=2>You have to enter a valid <b>comment</b>. <br><br><a href="calAnalysis.php" class="nounder"><input type="button" value="back to form" class="back"></a></font>');
}

// 1.4 valid email must be provided
\$analysisAuthor = str_replace("<","&lt;",\$_POST[analysisAuthor]);

if (\$analysisAuthor=='&lt;your_email>@eso.org'||\$analysisAuthor=='') {
  die ('<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30> <font size=2>You should enter a valid <b>email address</b>. <br><br><a href="calAnalysis.php" class="nounder"><input type="button" value="back to form" class="back"></a></font>');
}

// 1.5 if flag is NOK, analysis cannot be MISS
\$check_NOK = ereg (":NOK",\$analysisFlag);
\$newFlag = str_replace(":","",\$_POST[editedFlag]);

if (\$newFlag=="MISS"&&\$check_NOK!="")
  die ('<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30> <font size=2>You cannot analyze a NOK case as \'MISS\'; only \'NOK\' or \'OK\' is possible. You could also just enter a comment and leave the analysis for later. <br><a href="calAnalysis.php">back to form</a></font>');

// 1.6 if flag is LOST, analysis can only be OK, or left open (LEAVE)
\$check_LOST = ereg (":LOST",\$analysisFlag);
if ((\$_POST['editedFlag']!="OK")&&(\$_POST['editedFlag']!="LEAVE")&&\$check_LOST!="")
  die ('<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30> <font size=2>You can analyze a LOST case only as \'OK\' since once the night is over the LOST calibration cannot be taken (only possible action is ignoring or re-grading the OB, both meaning \'OK\'). You could also just enter a comment and leave the analysis for later. <br><a href="calAnalysis.php">back to form</a></font>');

// 2. if all fields are valid, we continue
\$time = gmdate("Y-m-d H:i:s");

// 2.1 we write new lines into calEditor_new.dat
\$new_file = "calEditor_new.dat";
if (!\$file_new = fopen(\$new_file,"a"))
 echo "error: can't open \$new_file ...\n";
\$file_new = fopen(\$new_file,"a");

echo "<table><tr style=\"font-size:small;\" bgcolor=#cccccc>";
echo "<td>DATE</td><td nowrap>data type</td><td>setup</td><td>flag</td><td nowrap>missing or outdated calibs</td><td>analysis flag and comment</td>";

\$i = 1 ;
while (\$i <= 100):
  \$case = "case\$i" ;
  \$analysisCase = str_replace("\r","",\$_POST["\$case"]);
  \$analysisCase = str_replace("\n","",\$analysisCase);
  if (\$analysisCase!="") {
    if (\$_POST['editedFlag']=="LEAVE")
      \$data  = \$analysisCase."</td><td>PRELIMINARY [".\$analysisComment." (analyzed by ".\$analysisAuthor.")]\n";
    if (\$_POST['editedFlag']!="LEAVE")
      \$data  = \$analysisCase."</td><td>ANALYZED: ".\$_POST[editedFlag]." [".\$analysisComment." (analyzed by ".\$analysisAuthor.")]\n";
    fwrite(\$file_new, \$data);

    echo "<tr style=\"font-size:small;\" bgcolor=#eeeeee><td>";
    echo \$data;
    echo "</tr>";
  }
  \$i++;
endwhile;
fclose(\$file_new);

// 2.2 we read all entries from calEditor.dat except the new ones, and write them into calEditor_new.dat
\$file_new = fopen(\$new_file,"a");
\$file = fopen("calEditor.dat","r");

\$i = 1;
while (\$i <= 1000):
  \$line = fgets(\$file,4096);		//get a new line for analysis

  \$j = 1;				//check this line against all cases (j=1...100) from form	
  \$match = 0;
  while (\$j <= 100 ):
    \$case = "case\$j" ;
    \$analysisCase = str_replace("\r","",\$_POST["\$case"]);
    \$analysisCase = str_replace("\n","",\$analysisCase);
    if (\$analysisCase!="") {
      if (ereg ("\$analysisCase", \$line)) {
        \$match = 1;
      }
    }
    \$j++;
  endwhile;

  if (\$match == 0) {          		// line does not contain analyzed entry --> write into new file
    fwrite(\$file_new, \$line);
  }
  \$i++;
endwhile;
// end of loops

fclose(\$file);
fclose(\$file_new);

if (!rename("calEditor_new.dat", "calEditor.dat"))
 echo "error: rename failed ...\n";             // have edited calEditor.dat file

echo "<font size=2>Entry updated by ".\$analysisAuthor." on ".\$time." UT<br>";
echo "It will be taken into account by <i>calChecker</i> at the next scheduled execution.<br><br>";
?>

  </table>

<br><br>
<a href="calAnalysis.php" class="nounder"><input type="button" value="OK" class="esocal"></a>
</html>
EOT

# calAnalysis.php
case $CAL4CAL in
 "NO"  ) LABEL="" ;;
 "YES" ) LABEL="(CAL4CAL)" ;;
esac

# php script is created and uploaded each time
cat > $TMP_DIR/calAnalysis.php <<EOT
<?php
/*
This php script is created by calChecker v$TOOL_VERSION .
It creates the cgi interface to catch analysis information about 
recent MISS or NOK cases in calChecker. This information is read
from, and written into, calEditor.dat, from where it is fed into
the calChecker result page upon the next calChecker run.
*/

echo <<<EOT
<html>
<head>
<style type="text/css">
.esocal {
  background-color: ${REPORT_COLOR};
  color: #FFF;
  font-size: large;
}   
.nounder {
  text-decoration: none;
}
</style>

<title>$DFO_INSTRUMENT calChecker: analysis editor $LABEL</title>
</head>
<body>
<table style="border-collapse: collapse;" border="1" bordercolor="#ffffff" cellpadding="0" cellspacing="0">
  <tr align="left" valign="bottom">
    <td bgcolor="$REPORT_COLOR" height="50" width="50"><b><font color="#ffffff" size="3">CAL</font></b></td>
    <td valign="bottom"><font size=+2>$DFO_INSTRUMENT calChecker: analysis editor $LABEL</font></td>
  </tr>
</table>
<p>
<a href="javascript:self.close();" class="nounder"><input type="button" value="close window"></a>
<p>

<table width="750" border="0" cellpadding="0" cellspacing="0">
EOT

if [ $CAL4CAL = NO ] && [ $CHECK_C4C = YES ]
then
	cat >> $TMP_DIR/calAnalysis.php <<EOT
  <tr bgcolor="#FFFFFF" height="3">
     <td align=center bgcolor="#FF9966"></td>
     <td></td>
     <td></td>
     <td></td>
     <td></td>
     <td></td>
     <td></td>
  </tr>

  <tr>
    <td width="3%" align=center bgcolor="#FF9966" nowrap><b><font size="1">&nbsp;MARK_SCIENCE<a href="${REL_URL}/CAL/php/calAnalysis.php" title="analysis editor for science"><font color="#333333">science</font></a>&nbsp;</font></b></td> 
    <td width="1"><font size=1 color=#FFFFFF>.</font></td>
    <td width="3%" align=center bgcolor="#FFD5BF" nowrap><b><font size="1">&nbsp;MARK_CAL4CAL<a href="${REL_URL}/CAL/CAL4CAL/php/calAnalysis.php" title="analysis editor for calibrations"><font color="#333333" size="1">cal4cal</font></a>&nbsp;</b></td> 
    <td width="1"><font size=1 color=#FFFFFF>.</font></td>
    <td width="3%"><font size="1">&nbsp; </td>
    <td width="88%">&nbsp;</td>
  </tr>
EOT

elif [ $CAL4CAL = NO ] && [ $CHECK_C4C = NO ]
then
	cat >> $TMP_DIR/calAnalysis.php <<EOT
 <tr>
   <td width="3%" align=center bgcolor="#FF9966" nowrap><b><font size="1">&nbsp;MARK_SCIENCE<a href="${REL_URL}/CAL/php/calAnalysis.php" title="calibration completeness for science"><font color="#333333">science</font></a>&nbsp;</font></b></td> 
   <td width="3%" align=center bgcolor="#eeeeee" nowrap><font size="1">&nbsp;<font color="#333333"><a title="${DFO_INSTRUMENT} has no checks for calibration completeness for calibrations">no cal4cal</a></font>&nbsp;</font></td>
   <td width="88%">&nbsp;</td>
 </tr>
EOT
else
	cat >> $TMP_DIR/calAnalysis.php <<EOT
  <tr bgcolor="#FFFFFF" height="3">
     <td></td>
     <td></td>
     <td align=center bgcolor="#FF9966"></td>
     <td></td>
     <td></td>
     <td></td>
     <td></td>
  </tr>

 <tr>
   <td width="3%" align=center bgcolor="#FFD5B5" nowrap><b><font size="1">&nbsp;MARK_SCIENCE<a href="${REL_URL}/CAL/php/calAnalysis.php" title="analysis editor for science"><font color="#333333">science</font></a>&nbsp;</font></b></td> 
   <td width="1"><font size=1 color=#FFFFFF>.</font></td>
   <td width="3%" align=center bgcolor="#FF9966" nowrap><b><font size="1">&nbsp;MARK_CAL4CAL<a href="${REL_URL}/CAL/CAL4CAL/php/calAnalysis.php" title="analysis editor for calibrations "><font color="#333333" size="1">cal4cal</font></a>&nbsp;</b></td> <!--CAL4CAL_NAV-->
   <td width="88%">&nbsp;</td>
  </tr>
EOT
fi

cat >> $TMP_DIR/calAnalysis.php <<EOT
  <tr bgcolor="#FF9966" height="3">
     <td colspan="7"></td>
  </tr>
</table> 

<table width=750><tr><td>
<font size=1>Here you can:
<ul>
<li>analyze currently open cases of MISS, LOST (on the 'science' interface) or NOK calibrations, or</li>
<li>provide a preliminary comment and leave the analysis open.</li>
</ul>
<font color=${REPORT_COLOR}>Why?</font> While <i>calChecker</i> finds flags in an automatic way, it is useful to analyse the 
automatic results and find out whether there are real issues.  Your analysis flag will override the automatic <i>calChecker</i> flag. 

<p><font color=${REPORT_COLOR}>How?</font> 
For each offered case, provide an analysis flag (OK/NOK/MISS), a comment (like 
"OB graded C, no calibrations required") and your email address. The analysis result
will be taken up by <i>calChecker</i> upon its next scheduled execution (the
flag will be updated, and your comment appears in the "ANALYSIS NOTES"). <p>
If you prefer to provide a preliminary comment only, choose "&lt;comment only&gt;".
[<a href=${QC_URL}/CC_help_analysis.html target=_top>help</a>]
</font>
</td></tr>
</table>
</p>

EOT;

// 1. Find already finished cases
// start loop 
\$database = "calEditor.dat";
\$file = fopen(\$database,"r");
\$first = 0;
\$i = 1;
while (\$i <= 1000):
  \$line = fgets(\$file,4096);  			//get a new line
  if ((ereg ("ANALYZED", \$line)&&(\$line!=""))||(ereg ("PRELIMINARY", \$line)&&(\$line!=""))) { //line contains ANALYZED?
    \$first++;
    if (\$first == 1) {					//first line: header for table
      echo <<<EOT
<font size=2><b>Already analyzed, or provisionally commented:</b></font><br>
<font size=1><i>(then items also appear under 'open analysis cases')</i></font>
<table width="750">
<tr style="font-size:small;" bgcolor=#cccccc>
  <td>DATE</td>
  <td nowrap>data type</td>
  <td>setup</td>
  <td>flag</td>
  <td nowrap>missing/outdated calibs</td>
  <td>analysis flag and comment</td>
<tr>
EOT;
       }
    echo "<tr style=\"font-size:small;\" bgcolor=#eeeeee><td>".\$line."</td></tr>";
  } 
  \$i++; 
endwhile;
fclose(\$file);// end of loop
echo "</table></p>";

// 2. check if any remaining entry for analysis
\$file = fopen(\$database,"r");
\$i = 1;
\$count = 0;
while (\$i <= 1000):
  \$line = fgets(\$file,4096);  			//get a new line
  if (!ereg ("ANALYZED", \$line)&&(\$line!="")) {	//line does not contain ANALYZED?
    \$count++;
  } 
  \$i++; 
endwhile;

// 2.1 No cases: exit
if (\$count == 0){
  echo <<<EOT
<p><font size="2"><b>No open analysis cases. Entries listed above, if any, 
have been added since calChecker's last execution and <br>
will be taken up upon its next scheduled execution. </b></p>
</b><a href="javascript:self.close();" class="nounder"><input type="button" value="close window"></a>
</body></html>
EOT;
exit;
}

// 2.2 Yes: offer form to analyze open cases
echo <<<EOT
<font size="2"><b>Select from currently open analysis cases $LABEL: </b></font><br>
<font size="1"><i>(multiple selection possible for speed; they all get the same analysis flag and comment) </i></font><br>

<form method="post" action="calEditor.php">
<table cellpadding="2" cellspacing="2" border="0" width="750">
EOT;

// 2.3 start loop 
\$file = fopen(\$database,"r");
\$i = 1;
echo <<<EOF
<tr bgcolor="#cccccc" style="font-size:small;">
    <td><a title="select one or more analysis cases">select</a></td>
    <td>DATE</td>
    <td><a title="data type as used on the main interface">data type</a></td>
    <td><a title="combination of setup keys">setup</a></td>
    <td><a title="LOST: calib missing, cannot be recovered; MISS: calib missing; NOK: file outdated">flag</a></td>
    <td><a title="calibration data to be taken">missing or outdated calibs</a></td>
    <td><a title="possible analysis flags">possible analysis flags</a></td>
  </tr>
EOF;

while (\$i <= 1000):
  \$line = fgets(\$file,4096);  				//get a new line
  if (ereg ("PRELIMINARY", \$line)) 				//for PRELIMINARY comments:
    \$line1 = ereg_replace("<!--CAL2-->.*","<!--CAL2-->",\$line);	//previous comments can be overwritten
  else
    \$line1 = \$line;					
  if (!ereg ("ANALYZED", \$line)&&(\$line!="")) {		//line does not contain ANALYZED?
    if (ereg ("LOST", \$line))
      {\$color = "#66CCFF";
       \$values = "<a title=\"change OB grade, issue cannot be solved by taking calibrations\">OK</a>";}
    else if (ereg ("NOK", \$line))
      {\$color = "#FFFF99";
       \$values = "<a title=\"cannot become worse than NOK\">OK,NOK</a>";}
    else
      {\$color = "#FF9999";
       \$values = "OK,NOK,MISS";}
    echo <<<EOF
  <tr bgcolor="\$color" style="font-size:small;">
    <td><input type="checkbox" name="case\$i" value="\$line1"></td>
    <td>\$line1</td>
    <td><font color=#666666" size=1>\$values</font></td>
  </tr>
EOF;
  } 
  \$i++; 
endwhile;
// end of loop
?>

</table>
<br>
<table cellpadding="2" cellspacing="2" border="0" width="750">
  <tr valign="top" bgcolor="#FF9999">
    <td><font size="2">New analysis flag: <br><i><font size=1>(make sure to assign same or better flag; LOST can only be analyzed 'OK'; you could also provide a preliminary comment only and analyze later)</i></font></td>
    <td><font size="2">Enter comment for the selected cases (max. 80 chars):<br>
         <i><font size=1>(use standard comments as appropriate)</i></font>
    </td>
  </tr>

  <tr bgcolor="#FF9999">
    <td width="150" valign=top>
      <select name="editedFlag">
        <option value="empty" selected>values:</option>
        <option value="OK">OK</option>
        <option value="NOK">NOK</option>
        <option value="MISS">MISS</option>
	<option value="LEAVE">&lt;comment only&gt;</option>
      </select>
    </td>
    <td width="500" nowrap><font size=2>
      <input name="analysisComment" type="radio" value="" checked>enter comment:<br>
      &nbsp; &nbsp; <input name="analysisComment" type="text" value="&lt;your analysis&gt;" size="60" maxlength="80"><br>
EOT

if [ -s $ORIG_TMP_DIR/CALCHECK/CC_STDCOMMENTS ]
then
      	echo "&nbsp; &nbsp; &nbsp; ... or standard comment:<br>" >> $TMP_DIR/calAnalysis.php
	cat $ORIG_TMP_DIR/CALCHECK/CC_STDCOMMENTS | sed "s|^.*|<input name=\"analysisComment\" type=\"radio\" value=\"&\"><tt>&</tt><br>|" >> $TMP_DIR/calAnalysis.php
fi

cat >> $TMP_DIR/calAnalysis.php <<EOT
    </td>
  </tr>

  <tr bgcolor="#FFFFFF">
    <td colspan="2">&nbsp;</td>
  </tr>

  <tr bgcolor="#FF9999">
    <td colspan="2"><font size="2">Your email address <i><font size=1>(e.g. $PSO_EMAIL | $QC_ADDRESS | $OP_ADDRESS):</font></i></td>
  </tr>

  <tr bgcolor="#FF9999">
    <td colspan="2">
      <input name="analysisAuthor" type="text" value="&lt;your_email&gt;@eso.org" size="30" maxlength="30">
    </td>
  </tr>
</table>

    <br>
    <a name="bottom"</a>
    <a href=#top><b><font size=2 color=#006A9D>top<font></a><br><br>
    <input type="submit" value="Submit" class="esocal"> &nbsp; &nbsp;
    <input type="reset"  value="Reset">
    <a href="javascript:self.close();" class="nounder"><input type="button" value="close window"></a>
</p>
</form>

</body>
</html>

EOT
}

# =========================================================================
# 0.6.8 generate php scripts for bad quality marking
#	calBadQual.php is the interface to mark datasets as BAD; 
#     	addBadQual.php takes the output and enters it in list_calBadQual.dat 
#       a) calBadQual.php
# =========================================================================
createCalBadQual(){
#calBadQual.php
cat > $TMP_DIR/calBadQual.php <<EOF
<?php
/*
This php script is created by ${TOOL_NAME} v${TOOL_VERSION}.
It creates the cgi interface to catch information about BAD calibrations
to be hidden for ${TOOL_NAME}. This information is fed into \$DFO_MON_DIR/CALCHECK/BADQUAL
from where it is read by the next ${TOOL_NAME} execution. 
The script can handle up to 1000 input cases.
*/

//------------------------------------------------------------
// 1. Prepare HTML page
echo <<<EOT
<html>
<head>

<title>$DFO_INSTRUMENT calChecker: bad quality marker </title>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery-latest.js"></script>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery.dataTables.js"></script>

<!-- dataTable features, like filtering -->
<script type="text/javascript" charset="utf-8">
        \$(document).ready(function()
        {
        \$("#mainTable").dataTable({
        "aaSorting": [[4,'asc']],
        "aoColumnDefs":
           [{ "bSortable": false, "aTargets": [ 0,1,5 ] } ],
        "oLanguage": {
                "sInfo": "<font color=#666666 size=1><i>Showing _START_ to _END_ of _TOTAL_ datasets</i>",
                "sSearch": "<font color=#666666 size=1><b>Search:</b> <i>Enter a string or a combination of strings.</i>",
                "sInfoEmpty": "<font color=#666666 size=1>Showing 0 to 0 of 0 datasets</font>",
                "sInfoFiltered": "<font color=#666666 size=1>(filtered from _MAX_ total datasets)</font>"
        },
        "bPaginate": false,
        "sDom": '<ift>'
         });
        });
</script>

<style type="text/css">
.esocal {
  background-color: ${REPORT_COLOR};
  color: #FFF;
  font-size: large;
  text-decoration: none;
}

.esocal2 {
  background-color: ${REPORT_COLOR};
  color: #FFF;
  font-size: small;
  text-decoration: none;
}

.nounder {
  text-decoration: none; 
}

table.display {
        text-align:     left;
        font-family:    arial;
        font-size:      small;
        height:         10px;
}
table.display thead tr .header {
        background-color:    #CCCCCC;
        background-image:    url(${IMG_URL}/bg.gif) no-repeat center right;
        height:              30px;
        vertical-align:      middle;
}
table.display thead tr .sorting_desc {
        background-color:    #8dbdd8;
        background: url(${IMG_URL}/desc.gif) no-repeat center right;
}
table.display thead tr .sorting {
        background-color:    #8dbdd8;
        background: url(${IMG_URL}/bg.gif) no-repeat center right;
}
table.display thead tr .sorting_asc {
        background-color:    #8dbdd8;
        background: url(${IMG_URL}/asc.gif) no-repeat center right;
}
</style>
</head>

<body>
<table style="border-collapse: collapse;" border="1" bordercolor="#ffffff" cellpadding="0" cellspacing="0">
  <tr align="left" valign="bottom">
    <td bgcolor="${REPORT_COLOR}" height="50" width="50"><b><font color="#ffffff" size="3">CAL</font></b></td>
    <td valign="bottom"><font size=+2>$DFO_INSTRUMENT calChecker: bad quality marker </font></td>
  </tr>
</table>

<p>
<a href=#bottom><b><font size=1 color=#006A9D>bottom</font></a>&nbsp;
<a href="javascript:self.close();" class="nounder"><input type="button" value="close window"></a>  &nbsp;
EOT;

\$database = "list_calBadQual.dat";
\$file = fopen(\$database,"r");

// link to data list
\$i = 1;
while (\$i <= 1000):
  \$line = fgets(\$file,4096);  	
  if (ereg ("DATAREPORT", \$line)) {
      echo "<a title=\"data report for TODAY\" class=\"nounder\" href=".\$line."<input type=\"button\" value=\"report\" class=\"esocal2\"></a><p>";
}
  \$i++; 
endwhile;
fclose(\$file);

echo <<<EOT
<p>

<table width="750" border="0" cellpadding="0" cellspacing="0">
  <tr bgcolor="#FF9966" height="3">
     <td colspan="7"></td>
  </tr>
</table> 

<table width=750>
<tr>
  <td><font size=1>Here you can mark calibration datasets for bad quality. They will become invisible for <i>calChecker</i> upon its next execution.<p>

<font color=${REPORT_COLOR}>Why?</font> Occasionally there are calibrations which satisfy the formal criteria (metadata) for <i>calChecker</i> but are known to have a quality issue. For instance, a set of flats is under-exposed, or a set of darks was taken while a lamp was shining in the dome. This dialogue hides these calibrations for <i>calChecker</i>, thus flagging a <i>calChecker</i> issue and the need for better calibrations.<p>

<font color=${REPORT_COLOR}>How?</font> Identify the datasets you want to be marked. For logical reasons, only complete datasets (as processed by the pipelines) can be marked. Only the calibrations from today (in the usual <i>calChecker</i> sense) are displayed here.<p>
If you select a dataset, please provide a comment that explains the reason why the dataset should be ignored.
There is no need for (and no value in) hiding datasets that are not triggered by the calObBuilder (i.e. that are not
triggered by science data). <p>

There are three possible flags: <font color=${REPORT_COLOR}>OK</font> is the default (quality OK); <font color=${REPORT_COLOR}>BAD</font> is a dataset just marked by you but not yet catched by <i>calChecker</i>; 
<font color=${REPORT_COLOR}>HIDDEN</font> is a bad dataset that is already known to <i>calChecker</i> and has been taken into account.<p>

- For easy navigation you can sort the table columns, or filter by e.g. setup values.<br>
- You can mark multiple datasets at a time if the same comment applies.
</font>
</td></tr>
</table>
EOT;

//------------------------------------------------------------
// 2. Find already marked cases 
// start loop 
\$database = "list_calBadQual.dat";
\$file = fopen(\$database,"r");
\$first = 0;
\$i = 1;
while (\$i <= 1000):
  \$line = fgets(\$file,4096);  			//get a new line
  if (((ereg ("BAD", \$line))||(ereg ("HIDDEN", \$line)))&&(\$line!="")) { 	//line contains BAD/HIDDEN?
    \$first++;					//counter
    if (\$first == 1) {				//first line: header for table
      echo <<<EOT
<font size=2><b>Already marked BAD or HIDDEN:</b><br> </font>
<font size=1>(raw files in these datasets are hidden for <i>calChecker</i>, i.e. are NOT taken into account!)</font>

<table class="display" cellpadding="2" cellspacing="2" border="0" width="750">
<tr style="font-size:small;" bgcolor=#aaa>
  <td><a title="known to calSelector?">known?</font></td>
  <td><a title="date on this monitor changes at 21:00 UT">DATE</a></td>
  <td><a title="raw data type as defined on the 'products' monitor">data type</a></td>
  <td><a title="instrument setup parameters">setup</a></td>
  <td><a title="name of dataset/processing job">dataset</a></td>
  <td><a title="flag">flag</a></td>
  <td><a title="comment">comment</a></td>
<tr>
EOT;
       }
    if (ereg ("BAD", \$line)) 
      echo "<tr style=\"font-size:small;\" bgcolor=#eee><td align=center><a title=\"waiting for next calChecker execution\">...</a></td><td>".\$line."</td></tr>";
    if (ereg ("HIDDEN", \$line)) 
      echo "<tr style=\"font-size:small;\" bgcolor=#ccc><td align=center><font color=green><b><a title=\"known to calChecker\">yes</a></b></td><td>".\$line."</td></tr>";
  } 
  \$i++; 
endwhile;
fclose(\$file);// end of loop
echo "</table></p>";

//------------------------------------------------------------
// 3. check if any calibs exist to be offered
\$file = fopen(\$database,"r");
\$i = 1;
\$count = 0;

while (\$i <= 1000):
  \$line = fgets(\$file,4096);  			//get a new line
  if ((!ereg ("BAD", \$line))&&(!ereg ("HIDDEN", \$line))&&(\$line!="")) {	//line does not contain BAD/HIDDEN
    \$count++;
  } 
  \$i++; 
endwhile;

// 3.1 No calibs? Exit
if (\$count == 0){
  echo <<<EOT
<p><font size="2" color=${TITLE_COLOR}><b>No current calibrations (or all current calibrations already marked as bad).</b></font></p>
<a href="javascript:self.close();" class="nounder"><input type="button" value="close window"></a>
</body></html>
EOT;
exit;
}

// 3.2 Calibs exist? Offer form to mark bad quality calibrations  
echo <<<EOT

<font size="2">Select dataset for hiding (because of bad quality): </b></font><br>
<font size=1><i>(selected dataset will be removed from list of files analyzed by calChecker)</i></font>

<form method="post" action="addBadQual.php">

<table id="mainTable" class="display" cellpadding="2" cellspacing="2" border="0" width="750">
<thead>
EOT;

// 3.3 start loop 
\$file = fopen(\$database,"r");
\$i = 1;
echo <<<EOF
<tr bgcolor="#cccccc" style="font-size:small;">
    <th><a title="select one or more cases for hiding">select<br>case</a></th>
    <th><a title="date on this monitor changes at 21:00 UT">DATE</a></th>
    <th><a title="raw data type as defined on the 'products' monitor">data type</a></th>
    <th><a title="instrument setup parameters">setup</a></th>
    <th><a title="name of dataset/processing job">dataset</a></th>
    <th><a title="flag (always OK)">flag</a></th>
  </tr>
</thead>
<tbody>
EOF;

\$color = "#F99";
while (\$i <= 1000):
  \$line = fgets(\$file,4096);  			//get a new line
  if ((!ereg ("BAD", \$line))&&(!ereg ("HIDDEN", \$line))&&(!ereg ("DATAREPORT", \$line))&&(\$line!="")) {	//line does not contain BAD/HIDDEN/DATAREPORT?
    echo <<<EOF
  <tr bgcolor="\$color" style="font-size:small;">
    <td><input type="checkbox" name="case\$i" value="\$line"></td>
    <td>\$line</td>
    <td><font color=grey>OK</font></td>
  </tr>
EOF;
  } 
  \$i++; 
endwhile;

// end of loop
//------------------------------------------------------------
// 4. form fields for comment and email (HTML only, end of php)
?>

  </tbody>
</table>
<br>
<table cellpadding="2" cellspacing="2" border="0" width="750">
  <tr valign="top" bgcolor="#FF9999">
    <td><font size="2">Enter mandatory comment for the selected case(s) (max. 100 chars):</font>
    </td>
  </tr>

  <tr bgcolor="#FF9999">
    <td width="500" nowrap><font size=2>
      <input name="badQualComment" type="text" value="&lt;your comment&gt;" size="80" maxlength="100">
    </td>
  </tr>

  <tr bgcolor="#FFFFFF">
    <td colspan="2">&nbsp;</td>
  </tr>

  <tr bgcolor="#FF9999">
    <td colspan="2"><font size="2">Your email address <i><font size=1>(e.g. $PSO_EMAIL | $QC_ADDRESS | $OP_ADDRESS):</font></i></td>
  </tr>

  <tr bgcolor="#FF9999">
    <td colspan="2">
      <input name="badQualAuthor" type="text" value="&lt;your_email&gt;@eso.org" size="30" maxlength="30">
      <input name="badQualFlag" type="hidden" value="BAD">
    </td>
  </tr>
</table>

    <br>
    <a name="bottom"></a><a href=#top><b><font size=1 color=#006A9D>top</font></a>&nbsp;
    <input type="submit" value="Submit" class="esocal"> &nbsp; &nbsp;
    <input type="reset"  value="Reset">
    <a href="javascript:self.close();" class="nounder"><input type="button" value="close window"></a> 
</p>
</form>

</body>
</html>
EOF

#addBadQual.php
cat > $TMP_DIR/addBadQual.php <<EOF
<html>
<style type="text/css">
.esocal {
  background-color: ${REPORT_COLOR};
  color: #FFF;
  font-size: x-large;
}
.nounder {
  text-decoration: none; 
}
</style>
<?php
/*
This php script is created by ${TOOL_NAME} v${TOOL_VERSION}. It reads the input from calBadQual.php 
and adds the marked file(s) together with the comment and email address to the 
list_calBadQual.dat from where it is read by the next calChecker execution. 
The script can handle up to 1000 lines ("cases").
*/

//------------------------------------------------------------
// 1. check if form elements are properly filled:
// 1.1 cases: at least one must be selected (we need to remove \\r and \\n for reasons not fully understood...)
\$i = 1 ;
\$test_empty=="";
while (\$i <= 1000):
  \$case = "case\$i" ;
  \$badQualCase = str_replace("\\r","",\$_POST["\$case"]);
  \$badQualCase = str_replace("\\n","",\$badQualCase);
  if (\$i == "1")
    \$analysisFlag = \$badQualCase;
  \$test_empty = \$test_empty.\$badQualCase ;
  \$i++;
endwhile;

if (\$test_empty=="") {
  die ('<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30> <font size=2>You have to select at least one <b>case</b>. <br>
<br><a href="calBadQual.php" class="nounder"><input type="button" value="back to form" class="back"></a> </font>');
}

// 1.2 valid comment must be provided
\$badQualComment = str_replace("<","&lt;",\$_POST[badQualComment]);
if (\$badQualComment=='&lt;your comment>'||\$badQualComment==''||\$badQualComment==' ') {
  die ('<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30> <font size=2>You have to enter a valid <b>comment</b>. <br>
<br><a href="calBadQual.php" class="nounder"><input type="button" value="back to form" class="back"></a> </font>');
}

// 1.3 valid email must be provided
\$badQualAuthor = str_replace("<","&lt;",\$_POST[badQualAuthor]);
if (\$badQualAuthor=='&lt;your_email>@eso.org'||\$badQualAuthor=='') {
  die ('<img src=/observing/dfo/quality/ALL/img/php-warning.png height=30 width=30> <font size=2>You should enter a valid <b>email address</b>. <br>
<br><a href="calBadQual.php" class="nounder"><input type="button" value="back to form" class="back"></a> </font>');
}

//------------------------------------------------------------
// 2. if all fields are valid, we continue
\$time = gmdate("Y-m-d H:i:s");

// 2.1 we write lines with BAD into calBadQual_new.dat
\$new_file = "calBadQual_new.dat";
if (!\$file_new = fopen(\$new_file,"a"))
 echo "error: can't open \$new_file ...\\n";
\$file_new = fopen(\$new_file,"a");

// header
echo "<table><tr style=\"font-size:small;\" bgcolor=#cccccc>";
echo "<td>DATE</td>
<td nowrap>data type</td>
<td>setup</td>
<td nowrap>bad-quality calibs, now hidden</td>
<td>flag</td>
<td>comment</td>";

// body
\$i = 1 ;
while (\$i <= 1000):
  \$case = "case\$i" ;
  \$badQualCase = str_replace("\\r","",\$_POST["\$case"]);
  \$badQualCase = str_replace("\\n","",\$badQualCase);
  if (\$badQualCase!="") {
    \$data  = \$badQualCase."</td><td>".\$_POST[badQualFlag]."</td><td>".\$badQualComment." (hidden by ".\$badQualAuthor.")\\n";
    fwrite(\$file_new, \$data);

    echo "<tr style=\"font-size:small;\" bgcolor=#eeeeee><td>";
    echo \$data;
    echo "</tr>";
  }
  \$i++;
endwhile;
fclose(\$file_new);
// now any new entry is in calBadQual_new.dat

// 2.2 we read all other entries from list_calBadQual.dat and add them into calBadQual_new.dat
\$file_new = fopen(\$new_file,"a");
\$file = fopen("list_calBadQual.dat","r");

\$i = 1;
while (\$i <= 1000):
  \$line = fgets(\$file,4096);		//get a new line for analysis
  \$j = 1;				//check this line against all cases (j=1...1000) from form	
  \$match = 0;
  while (\$j <= 1000):
    \$case = "case\$j" ;
    \$badQualCase = str_replace("\\r","",\$_POST["\$case"]);
    \$badQualCase = str_replace("\\n","",\$badQualCase);
    if (\$badQualCase!="") {
      if (ereg ("\$badQualCase", \$line)) {
        \$match = 1;
      }
    }
    \$j++;
  endwhile;

  if (\$match == 0) {          		// line does not contain analyzed entry --> write into new file
    fwrite(\$file_new, \$line);
  }
  \$i++;
endwhile;
// end of loops

fclose(\$file);
fclose(\$file_new);

if (!rename("calBadQual_new.dat", "list_calBadQual.dat"))
 echo "error: rename failed ...\\n";             // have edited calBadQual.dat file

echo "<font size=2>Entry updated by ".\$badQualAuthor." on ".\$time." UT<br>";
echo "It will be taken into account by <i>calChecker</i> at the next scheduled execution.<br><br>";
?>

  </table>

<br><br>
<a href="calBadQual.php" class="nounder"><input type="button" value="OK" class="esocal"></a> 
</html>
EOF

# calForcedRefresh.php: create it as an edited copy of hcForcedRefresh.php
CAL_WHY="If you have very recent data for which you want a quick calChecker response, launch here a forced refresh. After termination, this calChecker monitor page will be refreshed. This may take a couple of minutes."
CAL_HOW="Enter your email address below (it will be used to send you a confirmation once the processing is finished), then submit."

scp -p ${DFO_WEB_SERVER}:${WEB_DIR}/php/hcForcedRefresh.php $TMP_DIR/calForcedRefresh.php 1>/dev/null 2>/dev/null
if [ -s $TMP_DIR/calForcedRefresh.php ]
then
	sed -i -e "s/hcForcedRefresh.php/calForcedRefresh.php/g" $TMP_DIR/calForcedRefresh.php
	sed -i -e "s/HC monitor/calChecker/g" $TMP_DIR/calForcedRefresh.php
	sed -i -e "s/HC_ENFORCE/CAL_ENFORCE/g" $TMP_DIR/calForcedRefresh.php
	sed -i -e "/HC_WHY/s|^.*|$CAL_WHY|" $TMP_DIR/calForcedRefresh.php
	sed -i -e "/HC_HOW/s|^.*|$CAL_HOW|" $TMP_DIR/calForcedRefresh.php
	sed -i -e "s/created by autoDaily.*/copied from hcForcedRefresh.php and edited by ${TOOL_NAME} v${TOOL_VERSION}./g" $TMP_DIR/calForcedRefresh.php
	sed -i -e "s/autoDaily .*/${TOOL_NAME} -I/g" $TMP_DIR/calForcedRefresh.php
	sed -i -e "s/HC/CAL/g" $TMP_DIR/calForcedRefresh.php
	sed -i -e "s/${TITLE_COLOR}/${REPORT_COLOR}/g" $TMP_DIR/calForcedRefresh.php
	sed -i -e "s/autolog.html/calchecklog.html/" $TMP_DIR/calForcedRefresh.php

	scp -o BatchMode=yes $TMP_DIR/calForcedRefresh.php ${DFO_WEB_SERVER}:${WEB_DIR}/CAL/php 1>/dev/null
fi
}

# =========================================================================
# 0.7 generate directories if not yet available
# =========================================================================

if [ ! -d $DFO_CAL_DIR ]
then 
	echo "calChecker [INF] generated: $DFO_CAL_DIR"
	mkdir $DFO_CAL_DIR
fi

if [ ! -d $DFO_CAL_DIR/MCAL ] 
then
	mkdir $DFO_CAL_DIR/MCAL
	echo "calChecker [INF] generated: $DFO_CAL_DIR/MCAL"
fi

if [ ! -d $DFO_CAL_DIR/VCAL ]
then 
	mkdir $DFO_CAL_DIR/VCAL
	echo "calChecker [INF] generated: $DFO_CAL_DIR/VCAL"
fi

if [ ! -d $DFO_MON_DIR ]
then 
	mkdir $DFO_MON_DIR
	echo "calChecker [INF] generated: $DFO_MON_DIR"
fi

# =========================================================================
# 0.8 copy config files for createAB if not there      
# =========================================================================

OCA_CLASS=$DFO_CONFIG_DIR/OCA/${DFO_INSTRUMENT}_classification.h
OCA_ORGAN=$DFO_CONFIG_DIR/OCA/${DFO_INSTRUMENT}_organisation.h
OCA_ASSOC=$DFO_CONFIG_DIR/OCA/${DFO_INSTRUMENT}_association.h

cp $ORIG_CONFIG_DIR/${DFO_INSTRUMENT}_macro.h	$DFO_CONFIG_DIR/OCA
cp $ORIG_CONFIG_DIR/${DFO_INSTRUMENT}.h        	$DFO_CONFIG_DIR/OCA
cp $ORIG_CONFIG_DIR/OCA_macro.h                	$DFO_CONFIG_DIR/OCA
cp $ORIG_CONFIG_DIR/log4j.cf                   	$DFO_CONFIG_DIR/OCA

# some checks, to avoid having no OCA files at all
if [ ! -f $OCA_CLASS ]
then
	echo "calChecker [ERR] missing: $OCA_CLASS"
	exit 1
fi

if [ ! -f $OCA_ORGAN ]
then
	echo "calChecker [ERR] missing: $OCA_ORGAN"
	exit 1
fi

if [ ! -f $OCA_ASSOC ]
then
	echo "calChecker [ERR] missing: $OCA_ASSOC"
	exit 1
fi

# prepare for measuring exec_time and load
if [ $CAL4CAL = NO ]
then
	PROCESS_LOAD=`uptime | sed "s/^.*load average://" | awk '{print $1}' | sed "s/,//"`
	echo "- create \$DFO_CONFIG_DIR/OCA/config.createAB_CALCHECK ... " | tee -a $LOGFILE
fi

# create minimum version of config.createAB, with some info taken from the operational one
rm -f $DFO_CONFIG_DIR/OCA/config.createAB_CALCHECK
echo "#*** NOTE: this config file is created by calChecker. 
#*** Any modifications here will be overwritten by the tool!
" > $DFO_CONFIG_DIR/OCA/config.createAB_CALCHECK

grep "^CONFIG_VERSION" $ORIG_CONFIG_DIR/config.createAB >> $DFO_CONFIG_DIR/OCA/config.createAB_CALCHECK
echo "TOOL_MODE		AUTO
FILTER_RAW	NO
SCI_VIRT_MCAL	NO
SCI_VIRT_MASS	NO
N_MCAL_LIST	0
N_VCAL_LIST	$N_VCAL_LIST
GEN_CALDIR	NONE
DRS_TYPE	CON
" >> $DFO_CONFIG_DIR/OCA/config.createAB_CALCHECK
chmod u-w $DFO_CONFIG_DIR/OCA/config.createAB_CALCHECK

# =========================================================================
# 0.9 create table $TMP_DIR/list_rawpro to match pro.catg with raw.types
#     in CAL4CAL, we cp list_rawpro from CALCHECK to CALCHECK/CAL4CAL
# =========================================================================
if [ $CAL4CAL = NO ]
then
	echo  "- parse OCA rules to match pro.catg with raw.types "
	cfgParser
else
	cp $TMP_DIR/../list_rawpro $TMP_DIR/list_rawpro
fi

# =========================================================================
# 0.10 create list $RAWTYPES and get list of sorted $RAWTYPES if configured
# =========================================================================
getRawTypes 

# maximum validity time, needed for INCREM=NO
MAX_VALTIME=`cat $TMP_DIR/list_master_all_cals | awk '{print $3}' | sort -k1,1 | tail -1 | awk '{printf"%1.0f\n",$1}'`
export N_VCAL_LIST_EX=`echo $N_VCAL_MEM $MAX_VALTIME | awk '{print $1+$2}'`	# for the VCAL memory, being managed by createAB

if [ $INCREM = NO ] && [ $CAL4CAL = NO ]
then
# for the issue memory, we need to scan N_VCAL_MEM days plus longest validity
	echo "- memory filled for $N_VCAL_LIST_EX days (N_VCAL_MEM=$N_VCAL_MEM plus longest validity $MAX_VALTIME) ..." | tee -a $LOGFILE
fi

# check for explicit sorting by raw_types
# we do this only for CAL4CAL=NO; otherwise it gets too complex
rm -f $TMP_DIR/cc_sorted_rawtypes
CHECK_SORTING=`grep "^SORTING[[:space:]]" $DFO_CONFIG_DIR/config.calChecker | head -1`

if [ $CAL4CAL = YES ]
then
	cat $TMP_DIR/list_rawt1 |\
 awk '{print $1}' |\
 sed "s/\"//g" |\
 sed "s/;//" > $TMP_DIR/cc_sorted_rawtypes
elif [ "Q$CHECK_SORTING" = "Q" ]
then
	cat $TMP_DIR/list_rawt |\
 grep "DPR.CATG==\"SCIENCE\"" |\
 awk '{print $1}' |\
 sed "s/\"//g" |\
 sed "s/;//" > $TMP_DIR/cc_sorted_rawtypes
else
	grep "^SORTING[[:space:]]" $DFO_CONFIG_DIR/config.calChecker | awk '{print $2}' > $TMP_DIR/cc_sorted_rawtypes
fi

# =========================================================================
# 0.11 create gui esh scripts
# =========================================================================
# edit_calChecker.esh
cat > $DFO_GUI_DIR/edit_calChecker.esh <<EOT
#!/bin/sh

$CFG_EDITOR
exit
EOT
chmod u+x $DFO_GUI_DIR/edit_calChecker.esh

# refresh_calChecker.esh
cat > $DFO_GUI_DIR/refresh_calChecker.esh <<EOT
#!/bin/sh
echo "refresh calChecker ..."
calChecker 
exit
EOT
chmod u+x $DFO_GUI_DIR/refresh_calChecker.esh

# ===================================================================================
# 0.12 getAnalysis: check for entries in $DFO_MON_DIR/CAL_ANALYZED
# fully enabled: auto-detecting wrong analysis flags, correcting, info to qc
# ===================================================================================

getAnalysis(){
ANALYSIS_CONTENT=`grep "^$D $RT $S_MODIF " $DFO_MON_DIR/CAL_ANALYZED`

# check for inconsistent cases ':LOST ANALYZED: NOK' or ':LOST ANALYZED: MISS'
ANALYSIS_CHECK1=`echo $ANALYSIS_CONTENT	| grep ":LOST" | grep "ANALYZED:" | egrep "MISS|NOK"`
ANALYSIS_CHECK2=`echo $ANALYSIS_CONTENT	| grep ":NOK " | grep "ANALYZED: MISS"`
if [ "Q$ANALYSIS_CHECK1" != Q ]
then
	cat >  $TMP_DIR/inconsist_mail <<EOT
calChecker for `whoami`@`hostname`: issue with ANALYSIS for LOST case incorrectly set to MISS/NOK, forced to OK:
WAS:
$ANALYSIS_CONTENT
NOW:
EOT

	sed -i -e "/^$D $RT $S_MODIF/s|ANALYZED: MISS \[|ANALYZED: OK \[flag forced by calChecker; |" $DFO_MON_DIR/CAL_ANALYZED
	sed -i -e "/^$D $RT $S_MODIF/s|ANALYZED: NOK \[|ANALYZED: OK \[flag forced by calChecker; |"  $DFO_MON_DIR/CAL_ANALYZED
	ANALYSIS_CONTENT=`grep "^$D $RT $S_MODIF " $DFO_MON_DIR/CAL_ANALYZED`
	echo "$ANALYSIS_CONTENT" >> $TMP_DIR/inconsist_mail
fi

if [ "Q$ANALYSIS_CHECK2" != Q ]
then
	cat >  $TMP_DIR/inconsist_mail <<EOT
calChecker for `whoami`@`hostname`: issue with ANALYSIS for NOK case incorrectly set to MISS, forced to NOK:
WAS:
$ANALYSIS_CONTENT
NOW:
EOT
	sed -i -e "/^$D $RT $S_MODIF/s|ANALYZED: MISS \[|ANALYZED: NOK \[flag forced by calChecker; |" $DFO_MON_DIR/CAL_ANALYZED
	ANALYSIS_CONTENT=`grep "^$D $RT $S_MODIF " $DFO_MON_DIR/CAL_ANALYZED`
	echo "$ANALYSIS_CONTENT" >> $TMP_DIR/inconsist_mail
fi

if [ "Q$ANALYSIS_CHECK1" != Q ] || [ "Q$ANALYSIS_CHECK2" != Q ]
then
	mail -s "calChecker warning" $OP_ADDRESS <$TMP_DIR/inconsist_mail
	echo "***WARNING: CAL_ANALYZED modified:"
	cat $TMP_DIR/inconsist_mail | sed "s/^.*/	&/"
fi
 
# consistent cases 
ANALYSIS=`echo $ANALYSIS_CONTENT	| egrep ":LOST|:MISS|:NOK" | grep ANALYZED: | sed "s/^.*\[//" | sed "s/\]//"`
ANALYSIS_TAG=`echo $ANALYSIS_CONTENT	| egrep ":LOST|:MISS|:NOK" | grep ANALYZED: | sed "s/^.*ANALYZED: //" | sed "s/\[.*//" | awk '{print $1}'`
PRELIMINARY_TAG=`echo $ANALYSIS_CONTENT	| egrep ":LOST|:MISS|:NOK" | grep PRELIMINARY | sed "s/^.*\[//" | sed "s/\]//"`

if [ "Q$PRELIMINARY_TAG" != Q ]
then
	ANALYSIS=`echo $ANALYSIS_CONTENT     | egrep ":LOST|:MISS|:NOK" | grep PRELIMINARY | sed "s/^.*\[//" | sed "s/\]//"`
	ANALYSIS_TAG=`echo $ANALYSIS_CONTENT | awk '{print $4}' | sed "s/://"`
fi

if [ "Q$ANALYSIS_TAG" = "QMISS" ]
then
	ANALYSIS_COL="$LRED"
elif [ "Q$ANALYSIS_TAG" = "QNOK" ]
then
	ANALYSIS_COL="$LYLW"
elif [ "Q$ANALYSIS_TAG" = "QLOST" ]
then
	ANALYSIS_COL="$LBLU"
else
	ANALYSIS_COL="$LGRN"
fi

if [ "Q$ANALYSIS" = Q ]
then
	ANALYSIS="<font size=1 color=${TITLE_COLOR}><i><blink>[not yet analyzed]</blink></i></font>"

	if [ $D =  $TODAY ]
	then
		ACTION=""
# for the normal calibs the b/eocal signal matters ...
		if [ $ENTRY != LOST ]
		then	
		 	case $DAYCAL in
		 	 0 ) EOCAL="[daycals not yet started]" ;
				ANALYSIS="<font size=1 color=${TITLE_COLOR}><i>$EOCAL</i></font>" ;;
		 	 1 ) EOCAL="[daycals ongoing]"  ;
				ANALYSIS="<font size=1 color=${TITLE_COLOR}><i>$EOCAL</i></font>" ;;
		 	 2 ) EOCAL="[daycals finished; problem not yet analyzed]" ;
        			ANALYSIS="<font size=1 color=${TITLE_COLOR}><i><blink>$EOCAL</blink></i></font>" 
				echo "$D BLINK" >> $TMP_DIR/list_blink ;;
		 	esac
# ... but for LOST calibs not
		else	
			EOCAL="[night finished, calib lost, problem not yet analyzed]" 
       			ANALYSIS="<font size=1 color=${TITLE_COLOR}><i><blink>$EOCAL</blink></i></font>" 
			echo "$D BLINK" >> $TMP_DIR/list_blink 
		fi	
	else
		echo "$D BLINK" >> $TMP_DIR/list_blink
	fi

else
	if [ "Q$PRELIMINARY_TAG" != Q ]
	then
		echo "<tr bgcolor=${ANALYSIS_COL}><td><font size=2><b>[${ANALYSIS_INDEX}]</b></td><td><font size=1>$RT</td><td nowrap><font size=1>$S_MODIF</td><td nowrap><font size=1>$D</td><td><font size=1>${ANALYSIS_TAG}</td><td><font size=1><a style=\"background-color:#CCCCCC;\">preliminary comment:</a> $ANALYSIS</td></tr>" >> $TMP_DIR/ANALYSIS.html
		ANALYSIS="<font size=1><font color=${TITLE_COLOR}><i>preliminary comment:</i></font> [${ANALYSIS_INDEX}]</font>"
	else
		echo "<tr bgcolor=${ANALYSIS_COL}><td><font size=2><b>[${ANALYSIS_INDEX}]</b></td><td><font size=1>$RT</td><td nowrap><font size=1>$S_MODIF</td><td nowrap><font size=1>$D</td><td><font size=1>${ANALYSIS_TAG}</td><td><font size=1>$ANALYSIS</td></tr>" >> $TMP_DIR/ANALYSIS.html
		ANALYSIS="<font size=1><font color=${TITLE_COLOR}><i>analyzed:</i></font> [${ANALYSIS_INDEX}]</font>"
	fi
	ANALYSIS_INDEX=`echo $ANALYSIS_INDEX | awk '{print $1+1}'`
fi
}

# ===================================================================================
# 0.13 edit jobs
# 0.13.1 edit_CAL_POSTIT.esh; the master is on ${WEB_DIR}/CAL/CAL_POSTIT
# ===================================================================================
case $CAL4CAL in
 "NO"  ) EDIT_POSTIT=edit_CAL_POSTIT.esh     ;;
 "YES" ) EDIT_POSTIT=edit_CAL4CAL_POSTIT.esh ;;
esac

cat > $DFO_GUI_DIR/${EDIT_POSTIT} <<EOT
#!/bin/sh
# download from $DFO_WEB_SERVER
echo "We download the CAL_POSTIT file from ${PUB_QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/reports/CAL.
If none exists, a new file is created.
The file is then downloaded for local editing, and then uploaded again, to become visible immediately.
Hit return:"
read input

rm -f CAL_POSTIT
ssh -o BatchMode=yes ${DFO_WEB_SERVER} "if [ ! -s ${WEB_DIR}/CAL/CAL_POSTIT ]; then touch ${WEB_DIR}/CAL/CAL_POSTIT; fi" 1>/dev/null
eval "wget -q ${PUB_QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/reports/CAL/CAL_POSTIT -T 5 -t 1"
if [ ! -s CAL_POSTIT ]
then
	touch CAL_POSTIT
fi
mv CAL_POSTIT $DFO_MON_DIR/

$POSTIT_EDITOR
if [ ! -s $DFO_MON_DIR/CAL_POSTIT ]
then
	touch $DFO_MON_DIR/CAL_POSTIT
fi

# We clean CAL_POSTIT for '/' entries since otherwise sed fails and the whole HTML page is lost
CAL_POSTIT1=\`cat $DFO_MON_DIR/CAL_POSTIT | sed "s|/| |g" | tr "\\012" " " | sed "s/^.*/&\\n/"\`
echo \$CAL_POSTIT1 > $DFO_MON_DIR/CAL_POSTIT

echo "Now make changes visible (scp to ${DFO_WEB_SERVER} and ${MIRROR_WEB_SERVER}) (hit return):"
read REFRESH_YN

scp    -o BatchMode=yes $DFO_MON_DIR/CAL_POSTIT ${DFO_WEB_SERVER}:${WEB_DIR}/CAL/
scp -q -o BatchMode=yes $DFO_MON_DIR/CAL_POSTIT ${MIRROR_WEB_SERVER}:${MIRROR_WEB_DIR}/CAL &

sed -i -e "/<!--CAL_POSTIT-->/s|^.*|\${CAL_POSTIT1} <!--CAL_POSTIT-->|" $DFO_MON_DIR/calChecker.html
cp $DFO_MON_DIR/calChecker.html $HTML

echo "
... done, output updated. Hit return:"
read input
exit
EOT
chmod u+x $DFO_GUI_DIR/$EDIT_POSTIT

# ===================================================================================
# 0.13.2 edit_CAL_ANALYSIS.esh (simplified; for bulk editing only!)
# ===================================================================================

case $CAL4CAL in
 "NO"  ) EDIT_ANALYSIS=edit_CAL_ANALYSIS.esh ; EDIT_TEXT="for SCIENCE" ;;
 "YES" ) EDIT_ANALYSIS=edit_CAL4CAL_ANALYSIS.esh ; EDIT_TEXT="for CAL4CAL" ;;
esac

ANALYSIS_EDITOR=`echo $POSTIT_EDITOR | sed "s/CAL_POSTIT/CAL_ANALYZED/"`
cat > $DFO_GUI_DIR/$EDIT_ANALYSIS <<EOT
#!/bin/sh
echo "For normal editing of the analysis case, use the analysis interface (\"analyze issues\").
Here you can directly edit the analysis file, but this makes sense only in exceptional cases, 
e.g. if you have many similar cases. Note that this approach is error prone, and that any analysis
entered in the meantime on the analysis interface (by you or a PSO astronomer) will override your editing.
"

echo "Edit the CAL_ANALYZED file $EDIT_TEXT directly (Y/N) (Y)?"
read EDIT_YN
if [ Q\$EDIT_YN = QN ]
then
	echo "Think about it."
	exit
fi

$ANALYSIS_EDITOR

echo "You need to re-run the tool to make any changes visible. Refresh now (Y/N) (Y)?"
read input
if [ "Q\$input" != "QN" ]
then	
	calChecker -N
	exit
else
	echo "Changes become visible next time calChecker is executed. Hit return:"
	read input
	exit
fi
EOT

chmod u+x $DFO_GUI_DIR/$EDIT_ANALYSIS

# ===================================================================================
# 0.13.3 edit_BADQUAL.esh
# ===================================================================================

case $CAL4CAL in
 "NO"  ) EDIT_BADQUAL=edit_BADQUAL.esh ; EDIT_TEXT="for SCIENCE" ;;
 "YES" ) EDIT_BADQUAL=edit_CAL4CAL_BADQUAL.esh ; EDIT_TEXT="for CAL4CAL" ;;
esac

BADQUAL_EDITOR=`echo $POSTIT_EDITOR | sed "s/CAL_POSTIT/CAL_BADQUAL/"`
cat > $DFO_GUI_DIR/$EDIT_BADQUAL <<EOT
#!/bin/sh
echo "Edit the CAL_BADQUAL file $EDIT_TEXT (Y/N) (it stores the calibrations with bad quality) (Y)?"
read EDIT_YN
if [ Q\$EDIT_YN = QN ]
then
        echo "Think about it."
        exit
else
        echo "Follow the instructions. Remember: if the bad-quality file comes from a template, 
you need to add *all* raw files here, otherwise the association would go to the next in the sequence. Hit return:"
	read input
fi

$BADQUAL_EDITOR

echo "
Now we clean up the VCAL directory. Hit return:"
read input
for F  in \`grep -v "^#" $DFO_MON_DIR/CAL_BADQUAL | awk '{print \$2}' | sed "s/.fits//" | sed "s/.hdr//"\`
do
	rm -f $DFO_CAL_DIR/VCAL/r.\${F}*
	DATE_NOK=\`grep -v "^#" $DFO_MON_DIR/CAL_BADQUAL | grep -m 1 \$F | awk '{print \$1}'\`
	if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ]
	then
		sed -i -e "/^\${DATE_NOK}/d" $PREVIOUS_DIR/cc_list_dates_OKK
	fi
done

echo "
You need to re-run the tool to make any changes visible. 
- If only the current date is affected, the incremental call is sufficient. It is offered at the end of this dialogue.
- If a SCIENCE date in the past is affected, call 'calChecker -d <date>'. 
- If more than one SCIENCE date is affected, or you don't know for sure, call 'calChecker -F' (full mode) but be careful not to crash into a cronjob-triggered instance.

Launch incremental refresh now (Y/N) (N)?"
read input

if [ "Q\$input" = "QY" ]
then
        calChecker -N 
        exit
else
        echo "calChecker not up-to-date. Hit return:"
        read input
        exit
fi
EOT

chmod u+x $DFO_GUI_DIR/$EDIT_BADQUAL

# ===================================================================================
# 0.13.4 view_calChecker_log.esh
# ===================================================================================
if [ $CAL4CAL = NO ]
then
	cat > $DFO_GUI_DIR/view_calChecker_log.esh <<EOT
#!/bin/sh

$DFO_EDITOR $DFO_MON_DIR/calChecker.log
exit
EOT
	chmod u+x $DFO_GUI_DIR/view_calChecker_log.esh
fi

# ===================================================================================
# 0.14 launch_calmail.esh
# ===================================================================================
cat > $DFO_GUI_DIR/launch_calmail.esh <<EOT
#!/bin/sh
echo "Create here an email to Paranal Science Operations about a $DFO_INSTRUMENT calibration issue.
This will be read by the shift coordinator and distributed properly. Your name will be auto-added at the end.

Edit the message with your configured editor ($DFO_EDITOR). Hit return to continue:"
read CONTI

echo "PSO_REQUEST ($DFO_INSTRUMENT calibrations):" > $TMP_DIR/pso_mail
xterm -fg white -bg grey40 -fn fixed -T "PSO_REQUEST" -geom $XTERM_GEOM -e $MAIL_EDITOR

echo "
Reply to: $OP_ADDRESS, $DFO_INSTRUMENT QC operations" >> $TMP_DIR/pso_mail

echo "You have created the following mail:"
echo ""
echo "----"
cat $TMP_DIR/pso_mail
echo "----"
echo ""

echo "Send to $PSO_EMAIL (Y/N) (N) ?"
read SEND_YN
if [ Q\$SEND_YN = QN ]
then
	echo "Think about it."
	exit
else
	nail -s "PSO_REQUEST: ${DFO_INSTRUMENT} calChecker mail" -c $OP_ADDRESS -r $QC_ADDRESS $PSO_EMAIL < $TMP_DIR/pso_mail
	echo "Mail sent to: $PSO_EMAIL; cc: $OP_ADDRESS; return address: $QC_ADDRESS"
	read input
fi
EOT
chmod u+x $DFO_GUI_DIR/launch_calmail.esh

# =========================================================================
# 0.15 create styles
# =========================================================================

includeStyles(){
cat >> $HTML <<EOT
<style TYPE="text/css">
<!--
body.page {
        font-family: sans-serif;
        font-size: smaller;
        background-color: #FFFFFF;
}
a.dfos_item:link {
        font-size: small;
        color: #FFFFFF;
}
a.dfos_item:visited {
        font-size: small;
        color: #FFFFFF;
}
.esocal {
  background-color: ${REPORT_COLOR};
  color: #FFF;
  font-size: large;
}

.esocal2 {
  font-size: small;
  text-decoration: none;
}

.hclabel {
  background-color: $TITLE_COLOR;
  color: #FFF;
  font-size: medium;
  text-decoration: none;
  font-weight: bold;
  vertical-align: bottom;
  text-align: left;
}

.nounder {
  text-decoration: none;
}
-->
</style>

<link rel="StyleSheet" href="$STYLES" type="text/css">
EOT
}

# =========================================================================
# 0.16 get SM_VM flag from the raw data report ($D: date)
# =========================================================================
getFlag(){
if [ -s $DFO_LST_DIR/list_${DFO_INSTRUMENT}_$D.txt ]
then
	rm -f $TMP_DIR/list_flag
        cat $DFO_LST_DIR/list_${DFO_INSTRUMENT}_$D.txt |\
        awk '{
 if ( change == 1 && $1 != "Number" && $2 != "" )  { print date, $2 }
 if ( $1 == "OBS_PROG_ID" && $2 == "SM/VM" ) { change=1 }
 if ( $1 == "Number" ) { change=0 }
             }' change=0 date=$D | sort -u > $TMP_DIR/list_flag
	cat $TMP_DIR/list_flag | awk '{print $2}' | sort -u |\
 tr "\012" " " |\
 sed "s/s v/<font color=${SM_COLOR}><b>SM <font color=${VM_COLOR}>VM<\/b><\/font>/" |\
 sed "s/v s/<font color=${SM_COLOR}><b>SM <font color=${VM_COLOR}>VM<\/b><\/font>/" |\
 sed "s/s/<font color=${SM_COLOR}><b>SM<\/b><\/font>/" |\
 sed "s/v/<font color=${VM_COLOR}>VM<\/font>/"
else
	echo ""
fi
}

# =========================================================================
# 0.17 getSCI_NUM (apply filters to number of found SCIENCE files, per $D)
# =========================================================================
getSCI_NUM(){
case $ENABLE_60 in
 NO  ) SCI_NUM=`dfits *hdr | fitsort -d dpr.catg obs.prog.id | grep -v "60\.A" | grep -v Maintenance | awk '{print $2}' | grep SCIENCE | wc -l` ;;
 YES ) SCI_NUM=`dfits *hdr | fitsort -d dpr.catg obs.prog.id | grep -v Maintenance | awk '{print $2}' | grep SCIENCE | wc -l` ;;
esac

# apply PGI_FILTER if defined & found
if [ $SCI_NUM -gt 0 ] && [ "Q$PGI_FILTER" != Q ]
then
	SCI_NUM=`$DFO_BIN_DIR/$PGI_FILTER`
fi

if [ $SCI_NUM -gt 0 ]
then
	SCI_YN=Y
fi
}

# =========================================================================
# 0.18 beocal: evaluate begin and end-of-calib signal
#      special case UVES: evaluate also the GIRAF eocal signal since this is 
#      the one for FLAMES
# =========================================================================
beocal(){
case $DFO_INSTRUMENT in
 "UVES" ) WHERE_INSTRUMENT="instrument in (\"UVES\",\"GIRAF\")" 
 	  MESSAGE="- get beocal signal (UVES or GIRAF) ...";;
 *      ) WHERE_INSTRUMENT="instrument = \"$DFO_FILE_NAME\"" 
 	  MESSAGE="- get beocal signal ...";;
esac

echo $MESSAGE
cat > $TMP_DIR/beocal_query <<EOT
SELECT
        convert (char(10), timestamp , 102) ,
        convert (char(5), timestamp , 108) ,
        instrument,
        status
from
        observations..calibration_queues
where
        $WHERE_INSTRUMENT
order by 
	timestamp
go
EOT

rm -f $TMP_DIR/beocal_results
isql -Uqc -S$ARCH_SERVER -w999 -P`cat $ARCH_PWD` -i $TMP_DIR/beocal_query |\
 grep -v affected | sed "1,2 d" | sed "/^$/d"  |\
 sed "s/\./-/g" \
 > $TMP_DIR/beocal_results

# find first and last EOCAL entries for $TODAY1 (all times in UT) 
FIRST_BOCAL=`grep $TODAY1 $TMP_DIR/beocal_results | grep $DFO_FILE_NAME | grep BOCAL | head -1 | awk '{print $2}'`
LAST_EOCAL=`grep  $TODAY1 $TMP_DIR/beocal_results | grep $DFO_FILE_NAME | grep EOCAL | tail -1 | awk '{print $2}'`

if [ Q$FIRST_BOCAL = Q ]
then
	DAYCAL=0
elif [ Q$FIRST_BOCAL != Q ] && [ Q$LAST_EOCAL = Q ]
then
	DAYCAL=1
elif [ Q$LAST_EOCAL != Q ]
then
	DAYCAL=2
fi

# UVES: special
if [ $DFO_INSTRUMENT = UVES ]
then
	FIRST_BOCAL2=`grep $TODAY1 $TMP_DIR/beocal_results | grep GIRAF | grep BOCAL | head -1 | awk '{print $2}'`
	LAST_EOCAL2=`grep  $TODAY1 $TMP_DIR/beocal_results | grep GIRAF | grep EOCAL | tail -1 | awk '{print $2}'`
	
# neither UVES nor GIRAF started
	if [ Q$FIRST_BOCAL = Q ] && [ Q$FIRST_BOCAL2 = Q ]
	then
		DAYCAL=0
# GIRAF+UVES started but GIRAF not yet finished
	elif [ Q$FIRST_BOCAL != Q ] && [ Q$FIRST_BOCAL2 != Q ] && [ Q$LAST_EOCAL2 = Q ]
	then
		DAYCAL=1
# no GIRAF; UVES started but not yet finished
	elif [ Q$FIRST_BOCAL != Q ] && [ Q$FIRST_BOCAL2 = Q ] && [ Q$LAST_EOCAL = Q ]
	then
		DAYCAL=1
# GIRAF finished, or no GIRAF: UVES finished
	elif [ Q$LAST_EOCAL2 != Q ] 
	then
		DAYCAL=2
		LAST_EOCAL=$LAST_EOCAL2
	elif [ Q$LAST_EOCAL != Q ] && [ Q$FIRST_BOCAL2 = Q ] 
	then
		DAYCAL=2
	fi
fi
}

# =========================================================================
# 0.19 techCals: evaluate validity of technical calibrations
# =========================================================================

techCals(){
echo "- check for technical/maintenance calibrations ..."
rm -f $TMP_DIR/cc_list_all_tech $DFO_MON_DIR/CAL_TECH $DFO_MON_DIR/CAL_TECH.html $TMP_DIR/cc_tech.html*
if [ ! -s $LONG_CLASS ]
then
	echo "  ... none defined."
	cat > $TMP_DIR/cc_tech.html1 <<EOT
    <td ROWSPAN=2 width=40% bgcolor=#CCCCCC align=right valign=top>
<!-- white border table-->
<table BORDER=0 width="100%" cellpadding="0" cellspacing="0" bgcolor=#FFFFFF>
  <tr>
    <td>

<!-- result table -->
<table border="0" width="100%" align=left cellpadding="1" cellspacing="1">
  <tr bgcolor=#CCCCCC>
    <td COLSPAN=99 align=left nowrap><font size=2 color=${TITLE_COLOR}><b>Long-term calibrations:</b></font><font size=2 color=#666666> <i>none defined</i></font></td>
  </tr>
</table>
<!-- end of result table -->

    </td>
  </tr>
</table>
<!-- end of white-border table -->
EOT
	return
fi

grep "^#TECHNICAL" $LONG_CLASS | awk '{print $2,$3,$4}' > $TMP_DIR/cc_list_all_tech

cat > $DFO_MON_DIR/CAL_TECH.html <<EOT
<html>
<title>${DFO_INSTRUMENT} long-term calibrations monitor</title>
<table width=600>
<tr><td>
<h2>${DFO_INSTRUMENT} long-term calibrations monitor</h2>
<p><font size=2>
<a href="${HOW_TO_URL}" title="how to execute long-term and maintenance calibrations (internal link)" style="background-color:#FFD5BF; color:#333333";>how to execute</a>
</font></p>

<p><font size=2>
This is an overview of the current age of ${DFO_INSTRUMENT} long-term calibrations. These are calibrations 
which are not triggered by science data but by the maintenance part of the calibration plan.
This list includes maintenance, technical and Health Check calibrations.</font></p>

<table width=600>
  <tr bgcolor=#CCCCCC style="font-size:small";>
    <td valign=top><b><a title="short tag defining the type of calibration">type of calibration</a></b></td>
    <td valign=top><b><a title="description of type of calibration">description</a></b></td>
    <td valign=top><b><a title="validity of calibration (per calibration plan)">validity (days)</a></b></td>
    <td valign=top><b><a title="age of last calibration found, in days">current age of last calibration (days)</a></b></td>
    <td valign=top><b><a title="recommended action, see below for details">evaluation</a></b></td>
  </tr>
EOT

# for each TECH (type of file), we start with current data and go backwards in time until we find a file; max 1 yr
for TECH in `cat $TMP_DIR/cc_list_all_tech | awk '{print $1}'`
do
	if [ $TECH = HRULER ]
	then
		echo "<tr><td COLSPAN=99><hr noshade height=1></td></tr>" >> $DFO_MON_DIR/CAL_TECH.html
		continue
	fi
		
        VALID=`grep "^${TECH} " $TMP_DIR/cc_list_all_tech | awk '{print $2}'`
        PATTERN=`grep "^${TECH}[[:space:]]" $TMP_DIR/cc_list_all_tech | awk '{print $3}' | sed "s/\&/ | egrep /"g | sed "s/^.*/egrep &/"`
	DESCRIPTION=`grep "^#TECHNICAL" $LONG_CLASS | grep "[[:space:]]${TECH}[[:space:]]" | sed "s/\&\&$//" | sed "s/^.*\&\&//" | grep -v "^#TECHNICAL"`
        RESULT=""
        YYMM=`date +%Y-%m`
        YYMM2=`date +%Y%m`
        YYMM_min=`echo $YYMM2 | awk '{print $1-100}'` #one earlier

        while [ Q$RESULT = Q ] && [ $YYMM2 -ge $YYMM_min ]
        do

# we scan this month's listings; if we find results, take the last one; isolate the ARCFILE name and don't confuse it with the ORIGFILE name
		CHECK_EXIST=`ls $DFO_LST_DIR | grep list_${DFO_INSTRUMENT} | grep $YYMM | head -1`
		if [ Q$CHECK_EXIST != Q ]
		then	
                	RESULT=`eval "cat $DFO_LST_DIR/list_${DFO_INSTRUMENT}_$YYMM*.txt | $PATTERN" | tail -1 | sed "s/T[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9].fits/&PATTERN/" | sed "s/PATTERN.*//" | sed "s/^.*${DFO_FILE_NAME}./${DFO_FILE_NAME}./" | awk '{print $2,$1}' | awk '{print $1}'`
		else
			RESULT=""
		fi

# RESULT found: exit
               	if [ Q$RESULT != Q ]
               	then
               		break
               	else

# no RESULT found: check month before
                        YYMM1=`qcdate $YYMM-01 -1`
                        YYMM=`echo $YYMM1 | cut -c1-7`
                        YYMM2=`echo $YYMM | sed "s/-//"`
                fi
        done

        if [ Q$RESULT != Q ]
        then
                HDR=`echo  $RESULT | sed "s/fits/hdr/"`
                HH=`echo   $RESULT | sed "s/^.*[0-9]T//" | cut -c1,2`
                DATE=`echo $RESULT | sed "s/${DFO_FILE_NAME}.//" | sed "s/T.*//"`
		TURNOVER=`echo $DFO_OFFSET | awk '{print $1+12}'`

                if [ $HH -lt $TURNOVER ]
                then
                        DATE=`qcdate $DATE -1`
                fi

                if [ -s $DFO_HDR_DIR/$DATE/$HDR ]
                then
                        MJD=`dfits $DFO_HDR_DIR/$DATE/$HDR | fitsort -d MJD-OBS | awk '{printf"%5.3f\n", $2}'`
                else
                        MJD=`qcdate $DATE | awk '{print $1+0.5}'`
                fi

		TODAY_MJD2=`echo $TODAY_MJD $TODAY_HH $TODAY_MM | awk '{printf"%10.6f\n", $1+($2+$3/60)/24}'`

# AGE is evaluated against configured VAL +/- 33%
                AGE=`echo $TODAY_MJD2 $MJD  | awk '{printf"%5.1f\n", $1-$2}'`
                AGE1=`echo $VALID           | awk '{printf"%5.1f\n", $1*0.70}'`
                AGE2=`echo $VALID           | awk '{printf"%5.1f\n", $1*1.30}'`
		ACTION=`echo $AGE $AGE2     | awk '{printf"%i\n", $2-$1}'`

                FLAG=`echo $AGE $AGE1 $AGE2 |\
		 awk '{
			if ($1 < $2)      {print grn}
			else if ($1 < $3) {print ylw}
			else              {print red }
		      }' grn=$LGRN ylw=$LYLW red=$LRED`
                EVAL=`echo $AGE $AGE1 $AGE2 $VALID |\
		 awk '{
			if      ($1 < $2) 	{print "OK"}
			else if ($1 < $4) 	{print "soft REMINDER: to be taken within next", action, "day(s)"}
			else if ($1 < $3)	{print "REMINDER: take as soon as possible, outdated"}
			else 			{print "<blink>REMINDER: take as soon as possible, outdated by >30%!</blink>"}
		      }' action="${ACTION}"`
                EVAL1=`echo $AGE $AGE1 $AGE2 $VALID |\
		 awk '{
			if      ($1 < $2) 	{print "OK"}
			else if ($1 < $4) 	{print "soft REMINDER: next " action " day(s)"}
			else if ($1 < $3)	{print "REMINDER: take as soon as possible"}
			else 			{print "<blink>REMINDER: take as soon as possible!</blink>"}
		      }' action="${ACTION}"`

		echo "<tr bgcolor=$FLAG valign=top><td><font size=2><b>$TECH</b></font></td><td><font size=1>$DESCRIPTION</font></td><td align=center><font size=2><b>$VALID</b></font></td><td><font size=2>$AGE</font></td><td><font size=2>$EVAL</font></td></tr>" >> $DFO_MON_DIR/CAL_TECH.html
		if [ "$EVAL" != "OK" ]
		then
			echo "<tr bgcolor=$FLAG valign=top><td><font size=1><b>$TECH</b></font></td><td align=center><font size=1><b>$VALID</b></font></td><td><font size=1>$AGE</font></td><td><font size=1>$EVAL1</font></td></tr>" >> $TMP_DIR/cc_tech.html
		fi
	
        else
		echo "<tr bgcolor=$LRED valign=top><td><font size=2><b>$TECH</b></font></td><td><font size=1>$DESCRIPTION</font></td><td align=center><font size=2><b>$VALID</b></font></td><td COLSPAN=99><font size=2>nothing found in last 12 months!</font></td></tr>" >> $DFO_MON_DIR/CAL_TECH.html
		echo "<tr bgcolor=$LRED valign=top><td><font size=1><b>$TECH</b></font></td><td align=center><font size=1><b>$VALID</b></font></td><td COLSPAN=99><font size=1>nothing found in last 12 months!</font></td></tr>" >> $TMP_DIR/cc_tech.html
		rm -f $TMP_DIR/check_message
		grep "calChecker for `whoami`@`hostname`: $TODAY - bad configuration for long-term calibration $TECH" $DFO_MON_DIR/DFO_MESSAGE > $TMP_DIR/check_message
		if [ ! -s $TMP_DIR/check_message ]
		then
			echo "***WARNING: Long-term calibration $TECH is configured but has not been taken since 12 months. This is most likely a configuration error, please check!" > $TMP_DIR/message 
			echo "calChecker for `whoami`@`hostname`: Bad configuration for long-term calibration $TECH?" >> $DFO_MON_DIR/DFO_MESSAGE
			mail -s "calChecker warning" $OP_ADDRESS <$TMP_DIR/message
			echo "calChecker for `whoami`@`hostname`: $TODAY - bad configuration for long-term calibration $TECH" >> $DFO_MON_DIR/DFO_MESSAGE
		fi
        fi
done
echo "</table>
<p>
<table>
<tr bgcolor=$GRY style="font-size:small";>
  <td align=right valign=top><b>type of calibration:</b> </td>
  <td align=left valign=top>tag defining the type of calibration</td>
</tr>
<tr bgcolor=$GRY>
  <td align=right valign=top><font size=2><b>description:</b></td><td align=left valign=top><font size=2>description of data</font></td>
<tr bgcolor=$GRY style="font-size:small";>
  <td align=right valign=top><b>validity:</b></td><td align=left valign=top><font size=2>as configured (follows the definition in the calibration plan)</td>
</tr>
<tr bgcolor=$GRY style="font-size:small";>
  <td align=right valign=top><b>current age of last calibration</b></td><td align=left valign=top><font size=2>age of last calibration found, in days</td>
</tr>
<tr bgcolor=$GRY style="font-size:small";>
  <td align=right valign=top><b>evaluation:</b></td><td align=left valign=top><font size=2>recommended action <br> OK/green: within 30% of validity, no need for action; <br>
soft REMINDER/yellow: within validity but less than 30%, refresh on next occasion; <br>
REMINDER/yellow: outdated but less than 30%, refresh on next occasion; <br>
REMINDER/red: outdated by more than 30%, refresh as soon as possible<br>
nothing found in last 12 months: most likely a configuration error
</td>
</tr>
</table>
</p>

</td</tr></table>
</html>" >> $DFO_MON_DIR/CAL_TECH.html

# create the table for the main page
cat > $TMP_DIR/cc_tech.html1 <<EOT
    <td ROWSPAN=2 width=40% bgcolor=#CCCCCC align=right valign=top>

<!-- white border table-->
<table BORDER=0 width="100%" cellpadding="0" cellspacing="0" bgcolor=#FFFFFF>
  <tr>
    <td>

<!-- result table-->
<table border="0" width="100%" align=left cellpadding="1" cellspacing="1">
  <tr bgcolor=#eee>
    <td COLSPAN=99 align=left nowrap>
      <font size=2 color=${TITLE_COLOR}><b>Long-term calibrations and maintenance</b></font><font size=1><i> <a href="javascript:openWin('./CAL_TECH.html');" title="check for status of long-term and maintenance calibrations">complete overview</a> | <a href="javascript:openWin('${HOW_TO_URL}');" title="how to execute long-term and maintenance calibrations (internal link)" style="background-color:#FFD5BF; color:#333333";>how to execute</a></i> 
      &nbsp; &nbsp; <font color=#666666><b>[<a href="javascript:openWin('${QC_URL}/CC_help_long.html');" title="info about long-term calibrations">?</a>]</b></font></font>
    </td>
  </tr>
EOT

if [ -s $TMP_DIR/cc_tech.html ]
then
	cat >> $TMP_DIR/cc_tech.html1 <<EOT
  <tr bgcolor=#eee style="font-size:x-small";>
    <td valign=top nowrap><b><a title="short tag defining the type of calibration">type of calibration</a></b></td>
    <td valign=top nowrap><b><a title="validity of calibration (per calibration plan)">validity (days)</a></b></td>
    <td valign=top nowrap><b><a title="age of last calibration found, in days">age (days)</a></b></td>
    <td valign=top><b><a title="recommended action, see detail page for more ">evaluation</a></b></td>
  </tr>
EOT
	cat $TMP_DIR/cc_tech.html >> $TMP_DIR/cc_tech.html1
else
	cat >> $TMP_DIR/cc_tech.html1 <<EOT	
  <tr bgcolor=$LGRN>
    <td valign=top><font size=1><b>all long-term calibrations within validity range</font></td>
  </tr>
EOT
fi

cat >> $TMP_DIR/cc_tech.html1 <<EOT
</table>
<!-- end of result table -->

    </td>
  </tr>
</table>
<!-- end of white-border table -->
EOT
}

# =========================================================================
# 0.20 createWrapper: create a wrapper around HdrDownloader and createReport
# We create a wrapper around HdrDownloader/createReport and call it in the background, to protect 
# ourselves against database connection issues.
# There is a timeout parameter which skips HdrDownloader and createReport in case
# of troubles, hoping that both are available again upon next execution of calChecker.
# SLEEP:     fundamental beat for checking of activity 
# SLEEP_MAX: timeout after which job is killed (interpreted as being hanging)
# We loop over SLEEP to minimize waiting time for successful execution.
# Since v4.6 we don't call createReport in the background since this
# could crash with the bad quality calib management.
# =========================================================================
createWrapper(){
cat > $TMP_DIR/wrappedCreateReport <<EOT 
#!/bin/sh
#source \$HOME/.qcrc
#source \$HOME/.dfosrc

dfosCron -t HdrDownloader & 

PID=\$!
ALREADY_SLEPT=0
sleep 3

while [ \$ALREADY_SLEPT -lt $SLEEP_MAX ]
do
	CHECK_PID=\`ps \$PID | grep -v PID | awk '{print \$6}'\`
	if [ Q\${CHECK_PID} != Q ]
	then
		ALREADY_SLEPT=\`echo \$ALREADY_SLEPT $SLEEP | awk '{print \$1+\$2}'\`
		TIMESTAMP=\`date -u +%Y-%m-%d"T"%H:%M:%S\`
		echo "[\$TIMESTAMP] echo ALREADY_SLEPT: \$ALREADY_SLEPT"
		sleep $SLEEP
	else
		break
	fi
done

CHECK_PID=\`ps \$PID | grep -v PID | awk '{print \$6}'\`
if [ Q\${CHECK_PID} != Q ]
then
	kill -9 \$PID 2>&1 1>/dev/null
	echo "TIMEOUT: HdrDownloader killed after $SLEEP_MAX secs."
else
	echo "--------------"
	echo ""
	echo "createReport -d $TODAY"
	createReport -d $TODAY
fi
EOT
chmod u+x $TMP_DIR/wrappedCreateReport
}

# =========================================================================
# 0.21   manage calibration files with bad quality
# 0.21.1 moveBADQUAL: hide files with bad quality 
#  $1: DATE
# =========================================================================
moveBADQUAL(){
if [ ! -d $DFO_HDR_DIR/CAL_HIDE ]
then
	mkdir $DFO_HDR_DIR/CAL_HIDE
fi

if [ -s $DFO_MON_DIR/CAL_BADQUAL ]
then
	rm -f $TMP_DIR/cc_list_hidden_headers
	grep "^$1[[:space:]]" $DFO_MON_DIR/CAL_BADQUAL | awk '{print $2}' > $TMP_DIR/cc_list_hidden_headers

	if [ -s $TMP_DIR/cc_list_hidden_headers ]
	then
		echo "  createAB -m CALIB -d $D -c config.createAB_CALCHECK (files in CAL_BADQUAL found, do not interrupt!)" 
		if [ ! -d $DFO_HDR_DIR/CAL_HIDE/$1 ]
		then
			mkdir $DFO_HDR_DIR/CAL_HIDE/$1
		fi
	
		for H in `cat $TMP_DIR/cc_list_hidden_headers`
		do
			H1=`echo $H  | sed "s/fits/hdr/"`
			mv $DFO_HDR_DIR/$1/$H1 $DFO_HDR_DIR/CAL_HIDE/$1/
		done
	else
		echo "  createAB -m CALIB -d $D -c config.createAB_CALCHECK " 
	fi
fi	
}

# =========================================================================
# 0.21.2 getbackBADQUAL: move them back (to ensure completeness of header directories)
#  $1: DATE
# =========================================================================
getbackBADQUAL(){
if [ -s $DFO_MON_DIR/CAL_BADQUAL ]
then
	rm -f $TMP_DIR/cc_list_hidden_headers
	grep "^$1[[:space:]]" $DFO_MON_DIR/CAL_BADQUAL | awk '{print $2}' > $TMP_DIR/cc_list_hidden_headers
	if [ -s $TMP_DIR/cc_list_hidden_headers ]
	then
		for H in `cat $TMP_DIR/cc_list_hidden_headers`
		do
			H1=`echo $H  | sed "s/fits/hdr/"`
			mv $DFO_HDR_DIR/CAL_HIDE/$1/$H1 $DFO_HDR_DIR/$1
		done
	fi
fi
}

# =========================================================================
# 0.21.3 createBADQUAL: create CAL_BADQUAL file
# =========================================================================
createBADQUAL(){
if [ ! -s $DFO_MON_DIR/CAL_BADQUAL ]
then
	cat > $DFO_MON_DIR/CAL_BADQUAL <<EOT
# This is the current list of files which are known to have bad quality and
# should not be seen by calChecker. You can delete outdated entries which would
# not be seen by calChecker anymore.
#
# WORKFLOW for new entry:
# - enter DATE and file name(s) (extensions .fits or .hdr, both accepted; see example below)
# - run calChecker again, either in incremental mode, or for the affected date, or in full mode (depending on circumstances)
# - now the result box reflects new associations; it might display "miss" or "nok"
# - if so, enter proper ANALYSIS note 
#
# extensions .fits or .hdr (both accepted)
#DATE      FILENAME
#=================================================
#example (real entries without #!)
#2009-07-24 GIRAF.2009-07-25T12:16:27.765.hdr
EOT
	echo "***INFO: $DFO_MON_DIR/CAL_BADQUAL file created."
else
# maintenance: delete entries older than one (calendar) year
	YEAR=`echo $TODAY | cut -c1-4`
	if [ Q$YEAR = Q ]
	then
		echo "***ERROR: something went wrong with the date calculation."
	else
		YEAR_1=`echo $YEAR | awk '{print $1-1}'`
		egrep "^#|^${YEAR}|^${YEAR_1}" $DFO_MON_DIR/CAL_BADQUAL > $TMP_DIR/CAL_BADQUAL1
		mv $TMP_DIR/CAL_BADQUAL1 $DFO_MON_DIR/CAL_BADQUAL
	fi
fi
}

# =========================================================================
# 0.22 createTomorrow: empty page for tomorrow
# =========================================================================
createTomorrow(){
cat > $TMP_DIR/calChecker_tomorrow.html <<EOT
<html>
This date is currently in the future. <p>
[<a href=calResults_${DFO_INSTRUMENT}_${TODAY}.html>back</a>]
</html>
EOT
}

# =========================================================================
# START MAIN PROGRAM
# 1.  create CALIB ABs for set of dates, to know available calibs
# 1.1 define list of dates and do other preparations
# =========================================================================

if [ ! -d $PREVIOUS_DIR ] || [ $INCREM = NO ]
then
	if [ $INCREM = NO ]
	then
		rm -rf $PREVIOUS_DIR
	else
		INCREM=NO
		echo "***INFO: INCREM=NO (full execution) enforced, no directory $PREVIOUS_DIR found." | tee -a $LOGFILE
	fi
	mkdir $PREVIOUS_DIR
fi

# if DATE for REPAIR set: edit cc_list_dates_OKK and call tool for the
# repaired list; we do this for both SCIENCE and CAL4CAL
if [ $REPAIR = YES ]
then
	echo "- REPAIR mode: we remove $REPAIR_DATE from the lists of OK dates and then call the tool.
   This will refresh the specified date plus all other NOK dates. The specified date must be in the
   current default range of dates ($DEFAULT_NVCAL). Hit return to continue, or E to exit:"
	read CONTI_YN
	if [ Q$CONTI_YN = QE ]
	then
		exit
	fi

	if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ]
	then
		sed -i -e "/^${REPAIR_DATE}/d" $PREVIOUS_DIR/cc_list_dates_OKK
	fi

	if [ -s $DFO_MON_DIR/CAL4CAL/PREVIOUS/cc_list_dates_OKK ]
	then
		sed -i -e "/^${REPAIR_DATE}/d" $DFO_MON_DIR/CAL4CAL/PREVIOUS/cc_list_dates_OKK
	fi
fi

# apply DFO_OFFSET here
TODAY=`date -u +%Y-%m-%d`
TODAY_HH=`date -u +%H`
TODAY_MM=`date -u +%M`
TODAY_MJD=`qcdate $TODAY`

# check for $DFO_MON_DIR/CAL_BADQUAL
createBADQUAL

# find correct last date (improved precision 5.3f)
TODAY_MJD1=`echo $TODAY_MJD $TODAY_HH $TODAY_MM $DFO_OFFSET | awk '{printf"%5.3f\n", $1+($2+$3/60-$4)/24 -0.5}'`
START_MJD=`echo $TODAY_MJD ${N_VCAL_LIST} | awk '{print $1-$2}'`
if [ $INCREM = NO ]
then
	START_MJD=`echo $TODAY_MJD ${N_VCAL_MEM} | awk '{print $1-$2}'`
fi
START_MJD2=`echo $TODAY_MJD ${N_VCAL_LIST} | awk '{print $1-$2}'`
START_MJD3=`echo $TODAY_MJD ${N_VCAL_MEM} | awk '{print $1-$2}'`
START_MJD4=`echo $TODAY_MJD ${N_VCAL_MEM} ${MAX_VALTIME} | awk '{print $1-$2-$3}'`

# fill the dates for FULL mode before the memory zone 
MJD1=$START_MJD4
rm -f $TMP_DIR/cc_list_dates_before_mem

if [ $INCREM = NO ]
then
	while [ $MJD1 -lt $START_MJD3 ]
	do
		DATE=`qcdate $MJD1`
		echo $DATE >> $TMP_DIR/cc_list_dates_before_mem
		MJD1=`echo $MJD1 | awk '{print $1+1}'`	
	done
fi

# check if open issues exist outside the 7day zone
MJD=$START_MJD3
MJD1=$MJD

rm -f $TMP_DIR/cc_list_open_issues $TMP_DIR/cc_list_past_oks

if [ -s $DFO_MON_DIR/CAL_ANALYZED ]
then
	while [ $MJD1 -lt $START_MJD2 ]
	do
		DATE=`qcdate $MJD1`

		if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ]
		then
			CHECK_LISTED=`grep "^$DATE" $PREVIOUS_DIR/cc_list_dates_OKK`
		fi	

# entry MISS/NOK in CAL_ANALYZED? That date is a candidate for issues in memory range, entered into cc_list_open_issues
# later it is checked if analyzed to OK, or obsolete because calibs exist in the meantime
# we ignore LOST issues since LOST should be irrelevant after 7 days
		CHECK_ISSUE=`grep "^$DATE " $DFO_MON_DIR/CAL_ANALYZED | egrep ":MISS|:NOK" | grep -v "ANALYZED: OK" | awk '{print $1}' | head -1`

		if [ Q$CHECK_LIST = Q$DATE ]
		then
			echo $DATE >> $TMP_DIR/cc_list_past_oks
		elif [ Q$CHECK_ISSUE != Q ]
		then
			echo $DATE >> $TMP_DIR/cc_list_open_issues
# send info that open issue will disappear from calChecker!
			if [ $MJD1 = $START_MJD3 ] 
			then
				if [ -s $DFO_MON_DIR/DFO_MESSAGE ]
				then
					CHECK_DFO_STATUS=`grep "date $DATE having an issue will disappear from" $DFO_MON_DIR/DFO_MESSAGE  | head  -1`
				fi

				if [ "Q$CHECK_DFO_STATUS" = Q ]
				then
					echo "***WARNING: Date $DATE having an open issue will disappear from calChecker tomorrow. 
Make sure to take the right  measures." > $TMP_DIR/message
					echo "calChecker for `whoami`@`hostname`: date $DATE having an issue will disappear from calChecker tomorrow" >> $DFO_MON_DIR/DFO_MESSAGE
					mail -s "calChecker warning" $OP_ADDRESS <$TMP_DIR/message
				fi
			fi
		else
			echo $DATE >> $TMP_DIR/cc_list_past_oks
		fi
		MJD1=`echo $MJD1 | awk '{print $1+1}'`	
	done
fi

rm -f $TMP_DIR/cc_list_dates $TMP_DIR/cc_list_dates2 $TMP_DIR/cc_list_dates4

# for the CALIB memory
while [ $MJD -le $TODAY_MJD ]
do
	DATE=`qcdate $MJD`
	echo $MJD $TODAY_MJD1 $DATE | awk '{if ($1 <= $2) { print $3} }' >> $TMP_DIR/cc_list_dates
	MJD=`echo $MJD | awk '{print $1+1}'`
done

# for the science/cal4cal loop: normal scan range
MJD=$START_MJD2
while [ $MJD -le $TODAY_MJD ]
do
	DATE=`qcdate $MJD`
	echo $MJD $TODAY_MJD1 $DATE | awk '{if ($1 <= $2) { print $3} }' >> $TMP_DIR/cc_list_dates2
	MJD=`echo $MJD | awk '{print $1+1}'`
done

# for getObInfo: maximum scan range
MJD=$START_MJD3
rm -f $TMP_DIR/cc_list_dates3
while [ $MJD -le $TODAY_MJD ]
do
	DATE=`qcdate $MJD`
	echo $MJD $TODAY_MJD1 $DATE | awk '{if ($1 <= $2) { print $3} }' >> $TMP_DIR/cc_list_dates3
	MJD=`echo $MJD | awk '{print $1+1}'`
done

# correct $TODAY for $DFO_OFFSET
TODAY=`cat $TMP_DIR/cc_list_dates | tail -1`	#correct $TODAY for $DFO_OFFSET
TODAY1=`date -u +%Y-%m-%d`			#correct $TODAY for the beocal signal

# correct MJD for exec_time
TODAY_MJD2=`echo $TODAY_MJD $TODAY_HH $TODAY_MM | awk '{printf"%10.6f\n", $1+($2+$3/60)/24}'`

# TOMORROW
TOMORROW=`qcdate $TODAY +1`

# update report for $TODAY
if [ $REPORTS = YES ]
then
	echo "- update raw data report for $TODAY ..."
	echo "  calling HdrDownloader and createReport ..."

	if [ $FORCED_MODE = YES ]
	then
		writeLog "1. Refresh headers and data report for $TODAY ..."
		scpLog
	fi

# create the wrapper ...
	rm -f $TMP_DIR/out_CreateReport
	createWrapper

# ... and fire it (HdrDownloader in the foreground, all  others in the bkg
	$TMP_DIR/wrappedCreateReport 2>&1 1>$TMP_DIR/out_CreateReport
fi

if [ $REPORTS = NO ] && [ $CAL4CAL = NO ]
then
	echo "- report creation: skipped."
fi

# download CC_STDCOMMENTS from $QC_URL
if [ $CAL4CAL = NO ]
then
	echo "- download CC_STDCOMMENTS ..."
	rm -f CC_STDCOMMENTS.*
	eval "wget -q http://www.eso.org${QC_URL}/CC_STDCOMMENTS -T 5 -t 1"
	if [ -s CC_STDCOMMENTS ]
	then
		mv CC_STDCOMMENTS $TMP_DIR/
	fi
fi

# =========================================================================
# 1.2 call createAB -m CALIB
# 1.2.1 Prepare: set CALCHECK_ENVIRON, prepare cal_list_dates
# =========================================================================

# CALCHECK_ENVIRON: to tell createAB that calChecker has called it
# traditionally this caused createAB to not create product directories which would mess up the operational workflow
# but now it is used for more (see createAB code, search for CALCHECK_ENVIRON)

export CALCHECK_ENVIRON=YES

# requires continuous feeding of headers since we don't download so many headers every hour
if [ $CAL4CAL = NO ]
then
	echo "" | tee -a $LOGFILE
	echo "- create ABs for CALIB (to find VCALs for SCIENCE association) ..."  | tee -a $LOGFILE
	echo "  list of DATEs to be scanned:" >> $LOGFILE
	if [ $INCREM = NO ]
	then
		cat $TMP_DIR/cc_list_dates_before_mem >> $LOGFILE
	fi

	cat $TMP_DIR/cc_list_dates >> $LOGFILE
	echo "" >> $LOGFILE

	rm -f $TMP_DIR/cal_list_dates
	if [ $INCREM = NO ]
	then
		cat $TMP_DIR/cc_list_dates_before_mem > $TMP_DIR/cal_list_dates
		cat $TMP_DIR/cc_list_dates >> $TMP_DIR/cal_list_dates
		echo "  INCREM=NO: We create CALIB ABs for all $N_VCAL_MEM dates, plus for longest validity range before (full refresh of CALIB memory) ..." | tee -a $LOGFILE
	else
# REPAIR=YES: add $DATE_REPAIR and $TODAY; otherwise only $TODAY
		if [ $REPAIR = YES ]
		then
			echo $REPAIR_DATE > $TMP_DIR/cal_list_dates
			echo $TODAY >> $TMP_DIR/cal_list_dates
			echo "  INCREM=$INCREM: We update CALIB ABs for REPAIR=$REPAIR_DATE and TODAY=$TODAY only ..." | tee -a $LOGFILE
		else
			echo $TODAY >> $TMP_DIR/cal_list_dates
			echo "  INCREM=$INCREM: We update CALIB ABs for TODAY=$TODAY only ..." | tee -a $LOGFILE
		fi
	fi

# =========================================================================
# 1.2.2 Management of BADQUAL calibrations 
#       Download/upload list_calBadQual.dat from/to $QCWEB_URL,
#       for calBadQual.php interface. We catch a new BAD flag and enter the 
#	corresponding calibrations in CAL_BADQUAL. Then we update the flag 
#	to HIDDEN and upload the list. New ABs are added by 'autoDaily'.
#       The update in CAL_BADQUAL is done only once, when the BAD signal is found.
#	If later on that entry is commented out/removed, this overrides the entry
#	on the interface list.
# =========================================================================

	if [ $FORCED_MODE = YES ]
	then
		writeLog "2. Check for BADQUAL entries for $TODAY ..."
		scpLog
	fi

	rm -f $TMP_DIR/list_calBadQual.dat* $TMP_DIR/list_calBadQual_down.dat $TMP_DIR/cc_calBadQual $TMP_DIR/cc_calBadQual_files
	touch $TMP_DIR/list_calBadQual.dat

# check out if list_calBadQual.dat on qcweb has new BAD entries; if so, enter them in BADQUAL
	scp -o BatchMode=yes ${DFO_WEB_SERVER}:${WEB_DIR}/CAL/php/list_calBadQual.dat $TMP_DIR/list_calBadQual_down.dat 1>/dev/null 2>/dev/null

	if [ -s $TMP_DIR/list_calBadQual_down.dat ]
	then
		sed -i -e "/DATAREPORT/d" $TMP_DIR/list_calBadQual_down.dat
		grep "<td>BAD</td>" $TMP_DIR/list_calBadQual_down.dat | sed "s|</td><td>| |g" | sed "s/>/ /g" | sed "s/</ /g" | awk '{print $6}' > $TMP_DIR/cc_calBadQual
		if [ -s $TMP_DIR/cc_calBadQual ]
		then
			for AB in `cat $TMP_DIR/cc_calBadQual`
			do
# check if existing in CALCHECK $DFO_AB_DIR, then in DFO_OPS $DFO_AB_DIR
				BAD_COMMENT=`grep $AB $TMP_DIR/list_calBadQual_down.dat | sed "s|^.*<td>BAD</td><td>||"`
				if [ -s $DFO_AB_DIR/$AB ]
				then
					grep ^RAWFILE $DFO_AB_DIR/$AB | awk '{print $2,badcomm}' badcomm="$BAD_COMMENT" >> $TMP_DIR/cc_calBadQual_files
# $DFO_AB_DIR/../ is the DFOS_OPS DFO_AB_DIR
				elif [ -s $DFO_AB_DIR/../$AB ]
				then
					grep ^RAWFILE $DFO_AB_DIR/../$AB | awk '{print $2,badcomm}' badcomm="$BAD_COMMENT" >> $TMP_DIR/cc_calBadQual_files
				else
					echo "  $AB not found in $DFO_AB_DIR or $DFO_AB_DIR/../ "
					F1=`echo $AB | sed "s/_.*//" | sed "s/.ab//" | sed "s/^.*/&.fits/"`
					echo "$F1 $BAD_COMMENT" >> $TMP_DIR/cc_calBadQual_files
				fi
			done

# send mail to $OP_ADDRESS and to originator
			TIMESTAMP=`date -u +%Y-%m-%d" "%H:%M:%S`
			echo "[$TIMESTAMP UT] 	${TOOL_NAME}: BADQUAL entry found for $DFO_INSTRUMENT, check out \$DFO_MON_DIR/CAL_BADQUAL." > $TMP_DIR/auto_mail
			echo "" >> $TMP_DIR/auto_mail
			cat $TMP_DIR/cc_calBadQual_files | sed "s/^.*${DFO_FILE_NAME}/${DFO_FILE_NAME}/" >> $TMP_DIR/auto_mail
			mail -s "${TOOL_NAME}: BADQUAL entry found for $DFO_INSTRUMENT and $TODAY" $OP_ADDRESS < $TMP_DIR/auto_mail

			echo "[$TIMESTAMP UT] 	new BADQUAL entry for $DFO_INSTRUMENT, file(s) will be hidden for ${TOOL_NAME}." > $TMP_DIR/auto_mail
			echo "" >> $TMP_DIR/auto_mail
			cat $TMP_DIR/cc_calBadQual_files | sed "s/^.*${DFO_FILE_NAME}/${DFO_FILE_NAME}/" >> $TMP_DIR/auto_mail
			ORIGINATOR=`cat $TMP_DIR/cc_calBadQual_files | sed "s/^.*hidden by//" | sed "s/)//" | tail -1 | awk '{print $1}'`
			if [ "$ORIGINATOR" != $OP_ADDRESS ]
			then
				mail -s "${TOOL_NAME}: BADQUAL entry found for $DFO_INSTRUMENT and $TODAY" $ORIGINATOR < $TMP_DIR/auto_mail
			fi

# now check for existence in $DFO_MON_DIR/CAL_BADQUAL
			if [ -s $TMP_DIR/cc_calBadQual_files ]
			then
				for RF in `cat $TMP_DIR/cc_calBadQual_files | awk '{print $1}'`
				do
					RF1=`basename $RF`
					CHECK_EXIST=`grep $RF1 $DFO_MON_DIR/CAL_BADQUAL | grep -v "^#"`
# entry not yet existing?
					if [ "Q$CHECK_EXIST" = Q ]
					then
# --> enter
						BAD_COMMENT=`grep $RF1 $TMP_DIR/cc_calBadQual_files | sed "s/^.*$RF1//"`
						echo "$TODAY $RF1 $BAD_COMMENT" >> $DFO_MON_DIR/CAL_BADQUAL
					fi
				done
			fi

			echo "  BAD entries found for $TODAY, email sent, VCALs cleaned up ..."
		fi

# we need to remove the corresponding VCALs (taken from edit_BADQUAL.esh dialogue)
# since this is used for the current date only, no need for maintaining the cc_list_dates_OKK list
		for F  in `grep -v "^#" $DFO_MON_DIR/CAL_BADQUAL | awk '{print $2}' | sed "s/.fits//" | sed "s/.hdr//"`
		do
			rm -f $DFO_CAL_DIR/VCAL/r.${F}*
			rm -f $DFO_CAL_DIR/CAL4CAL/VCAL/r.${F}*
		done

# replace all BAD ABs with HIDDEN (this is different from autoDaily where we add new ABs) (only if still the same date)
		echo "${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/REPORT/list_${DFO_INSTRUMENT}_${TODAY}_data.html <!--DATAREPORT-->" > $TMP_DIR/list_calBadQual.dat1

		grep -v ">BAD<" $TMP_DIR/list_calBadQual_down.dat >> $TMP_DIR/list_calBadQual.dat1
		grep ">BAD<" $TMP_DIR/list_calBadQual_down.dat | sed "s|>BAD<|>HIDDEN<|"  >> $TMP_DIR/list_calBadQual.dat1
		cat $TMP_DIR/list_calBadQual.dat1 | sort -k8,8 > $TMP_DIR/list_calBadQual.dat

# check if any of the HIDDEN ABs should be unhidden because the CAL_BADQUAL entry is commented out
		grep "<td>HIDDEN</td>" $TMP_DIR/list_calBadQual.dat | sed "s|</td><td>| |g" | sed "s/>/ /g" | sed "s/</ /g" | awk '{print $6}' > $TMP_DIR/list_calBadQual.dat2
		if [ -s $TMP_DIR/list_calBadQual.dat2 ]
		then
			for AB_ROOT in `cat $TMP_DIR/list_calBadQual.dat2 | sed "s/_.*//" | sed "s/\.ab//"`
			do
				CHECK_UNHIDE=`grep "^#" $DFO_MON_DIR/CAL_BADQUAL | grep $AB_ROOT | head -1`
				if [ "Q$CHECK_UNHIDE" != Q ]
				then
					sed -i -e "/${AB_ROOT}/s|</td><td>HIDDEN.*||" $TMP_DIR/list_calBadQual.dat
					echo "*** INFO: ${AB_ROOT} unhidden since entry in $DFO_MON_DIR/CAL_BADQUAL was commented out." > $TMP_DIR/auto_mail
					cat $TMP_DIR/auto_mail
					mail -s "${TOOL_NAME}: BADQUAL entry removed for $DFO_INSTRUMENT and $TODAY" $OP_ADDRESS < $TMP_DIR/auto_mail
				fi
			done
		fi

# upload
		if [ -s $TMP_DIR/list_calBadQual.dat ]
		then
			ssh -o BatchMode=yes $DFO_WEB_SERVER "rm -f ${WEB_DIR}/CAL/php/list_calBadQual.dat"
			chmod a+w $TMP_DIR/list_calBadQual.dat 
			scp -o BatchMode=yes $TMP_DIR/list_calBadQual.dat ${DFO_WEB_SERVER}:${WEB_DIR}/CAL/php/ 1>/dev/null 2>/dev/null
		fi
	fi

# =========================================================================
# 1.2.3 Loop over selected dates for createAB
# =========================================================================

	if [ $FORCED_MODE = YES ]
	then
		writeLog "3. Create new CALIB ABs for SCIENCE association ..."
		scpLog
	fi

	for D in `cat $TMP_DIR/cal_list_dates`
	do
		echo "" >> $LOGFILE
		TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
		if [ -s $DFO_HDR_DIR/$D ]
		then
			moveBADQUAL $D		#to ensure that files with known bad quality are not used to create ABs
			echo "[$TIMESTAMP] createAB -m CALIB -d $D -c config.createAB_CALCHECK " >> $LOGFILE
			createAB -m CALIB -d $D -c config.createAB_CALCHECK >> $LOGFILE
    			rm -f $DFO_MON_DIR/AB_list_CALIB_$D
			getbackBADQUAL $D	#to ensure that $DFO_HDR_DIR is complete again
		else
			echo "    no data available for $D, skipped." | tee -a $LOGFILE
		fi

		if [ $FORCED_MODE = YES ]
		then
			writeLog "   $D ..."
			scpLog
		fi
	done 
fi

# ======================================================================
# 2.0 CAL4CAL=YES: check calibrations for calibrations
# 2.1 switch for CAL4CAL: we call calChecker for CAL4CAL here
# ======================================================================

if [ $CHECK_C4C = YES ] && [ $CAL4CAL = NO ] 
then
	echo ""
	echo "- check calibrations for calibrations (CAL4CAL) ..."
	export N_VCAL_LIST=$N_VCAL_LIST
	case $SCP in
	 NO  ) SCP_FLAG="-n" ;;
	 YES ) SCP_FLAG="" ;;
	esac

	if [ $FORCED_MODE = YES ]
	then
		writeLog "4. CAL4CAL branch ..."
		scpLog
	fi

	if [ $INCREM = NO ]
	then
		calChecker -C $SCP_FLAG -F | sed "s/^.*/[CAL4CAL]   &/"
	elif [ $INCREM = YES ]
	then
		calChecker -C $SCP_FLAG    | sed "s/^.*/[CAL4CAL]   &/"
	fi
fi

# ======================================================================
# 2.2 Fill $DFO_CAL_DIR/VCAL (all calibs already known)
# ======================================================================

if [ $CHECK_C4C = YES ] && [ $CAL4CAL = YES ]
then
	cp $DFO_CAL_DIR/../VCAL/* $DFO_CAL_DIR/VCAL/
	
	echo "- create ABs for CAL4CAL CALIBs ..." | tee -a $LOGFILE

	case $INCREM in
	  "YES" ) echo "  INCREM=$INCREM: We scan only NOK/MISS dates and TODAY ..." | tee -a $LOGFILE ;;
	  "NO"  ) echo "  INCREM=$INCREM: We scan all $N_VCAL_LIST dates ..."        | tee -a $LOGFILE ;;
	esac

# ======================================================================
# 2.3 Call createAB -m CALIB for CAL4CAL configuration
#     Create CALIB ABs for later completeness checks
# ======================================================================

# always suppress completeness check in createAB for CAL4CAL
	export CREATEAB_NOTCHECK=TRUE

# loop over dates: memory zone (N_VCAL_MEM...7days)
	if [ -s $TMP_DIR/cc_list_open_issues ]
	then
		for D in `cat $TMP_DIR/cc_list_open_issues`
		do
			CHECK_OKK=""
			if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ]
			then
				CHECK_OKK=`grep "^$D" $PREVIOUS_DIR/cc_list_dates_OKK | sort -u`
			fi

# if analyzed previously OKK, then skip
			if [ Q$CHECK_OKK = Q$D ]
			then
				echo "    CAL4CAL files for $D all OK, skipped." | tee -a $LOGFILE
				continue
			fi

			if [ -s $DFO_HDR_DIR/$D ]
			then
				cd $DFO_HDR_DIR/$D
				CHECK_EXIST=`ls | grep hdr | head -1`
				if [ "Q$CHECK_EXIST" = "Q" ]
				then
					echo "    no CAL4CAL files found for $D, skipped." | tee -a $LOGFILE
	                        	continue
				fi
				TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
				echo "  createAB -m SCIENCE -d $D -c config.createAB_CALCHECK (open issue) ..." | tee -a $LOGFILE
				echo "" >> $LOGFILE
				echo "[$TIMESTAMP]  createAB -m SCIENCE -d $D -c config.createAB_CALCHECK" >> $LOGFILE

       				createAB -m SCIENCE -d $D -c config.createAB_CALCHECK >> $LOGFILE
			else
				echo "    no data available for $D, skipped." | tee -a $LOGFILE
			fi
		done
	fi

# loop over normal range
	for D in `cat $TMP_DIR/cc_list_dates2`
	do
		if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ] && [ $INCREM = YES ] && [ $D != $TODAY ]
		then
			CHECK_OKK=`grep "$D" $PREVIOUS_DIR/cc_list_dates_OKK | sort -u`
			if [ Q$CHECK_OKK = Q$D ]
			then
				echo "    CAL4CAL files for $D all OK, skipped." | tee -a $LOGFILE
				continue
			fi
		fi

		if [ -s $DFO_HDR_DIR/$D ]
		then
			cd $DFO_HDR_DIR/$D
			CHECK_EXIST=`ls | grep hdr | head -1`
			if [ "Q$CHECK_EXIST" = "Q" ]
			then
				echo "    no CAL4CAL files found for $D, skipped." | tee -a $LOGFILE
	                        continue
			fi
			TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
			echo "    CAL4CAL files for $D ... (normal range)" | tee -a $LOGFILE
			echo "" >> $LOGFILE
			echo "[$TIMESTAMP]  createAB -m SCIENCE -d $D -c config.createAB_CALCHECK" >> $LOGFILE

       			createAB -m SCIENCE -d $D -c config.createAB_CALCHECK >> $LOGFILE
		else
			echo "    no data available for $D, skipped." | tee -a $LOGFILE
		fi
	done 
fi

# ======================================================================
# 3.0 CAL4CAL = NO: create SCIENCE ABs 
#     Create SCIENCE ABs for later completeness checks
# ======================================================================

if [ $CAL4CAL = NO ]
then
	echo "" | tee -a $LOGFILE
	if [ $SUPPRESS_ABCHECK = YES ]
	then
		export CREATEAB_NOTCHECK=TRUE
		echo "- create ABs for SCIENCE (no completeness checks!) ..." | tee -a $LOGFILE
	else
		echo "- create ABs for SCIENCE (completeness is checked) ..." | tee -a $LOGFILE
	fi

	if [ $FORCED_MODE = YES ]
	then
		writeLog "5. SCIENCE branch ..."
		scpLog
	fi

	case $INCREM in
	  "YES" ) echo "  INCREM=$INCREM: We scan only open issues, NOK/MISS dates, and TODAY ..." | tee -a $LOGFILE ;;
	  "NO"  ) echo "  INCREM=$INCREM: We scan all $N_VCAL_LIST dates ..."        | tee -a $LOGFILE ;;
	esac

# loop over dates: memory zone (N_VCAL_MEM...7days)
	if [ -s $TMP_DIR/cc_list_open_issues ]
	then
		for D in `cat $TMP_DIR/cc_list_open_issues`
		do
			CHECK_OKK=""
			if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ]
			then
				CHECK_OKK=`grep "^$D" $PREVIOUS_DIR/cc_list_dates_OKK | sort -u`
			fi

# if analyzed previously OKK, then skip
			if [ Q$CHECK_OKK = Q$D ]
			then
				echo "    SCIENCE files for $D all OK, skipped." | tee -a $LOGFILE
				echo "    SCIENCE files for $D all OK, skipped." >> $FORCED_LOG
				continue
			fi

			if [ -s $DFO_HDR_DIR/$D ]
			then
				cd $DFO_HDR_DIR/$D
				CHECK_EXIST=`ls | grep hdr | head -1`
				if [ "Q$CHECK_EXIST" = "Q" ]
				then
					CHECK_EXIST=`grep SCIENCE *hdr | head -1`
				fi
				
				if [ "Q$CHECK_EXIST" = "Q" ]
				then
					echo "    no SCIENCE files found for $D, skipped." | tee -a $LOGFILE
					echo "    no SCIENCE files found for $D, skipped." >> $FORCED_LOG
	                        	continue
				fi

# otherwise filter out 60/060 SCIENCE (unless ENABLE_60=YES)
                        	dfits *hdr | fitsort -d dpr.catg obs.prog.id > $TMP_DIR/cc_catg_prog

				if [ $ENABLE_60 = NO ]
	                        then
       	                        	CHECK_SCI60=`cat $TMP_DIR/cc_catg_prog | grep SCIENCE | awk '{print $1,$2,$3}' | egrep " 60\.| 060\."`
                                	if [ "Q$CHECK_SCI60" != "Q" ]
                                	then
                                       		echo "    $D: SCIENCE files found with run_id starting with 60/060, will be skipped." | tee -a $LOGFILE
                                       		echo "    $D: SCIENCE files found with run_id starting with 60/060, will be skipped." >> $FORCED_LOG
                                	fi

                                	CHECK_SCI=`cat $TMP_DIR/cc_catg_prog | grep SCIENCE | awk '{print $1,$2,$3}' | egrep -v " 60\.| 060\."`
                        	else
                               		CHECK_SCI=`cat $TMP_DIR/cc_catg_prog | grep SCIENCE | awk '{print $1,$2,$3}'`
                        	fi

				if [ "Q$CHECK_SCI" = "Q" ]
                        	then
                               		echo "    no SCIENCE files found for $D, skipped." | tee -a $LOGFILE
                               		echo "    no SCIENCE files found for $D, skipped." >> $FORCED_LOG
                                	continue
                        	else
                                	TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
                        	fi

				echo "  createAB -m SCIENCE -d $D -c config.createAB_CALCHECK (open issue) ..." 
				echo "    calChecking $D (open issue) ..." >> $FORCED_LOG
				echo "" >> $LOGFILE
				echo "[$TIMESTAMP]  createAB -m SCIENCE -d $D -c config.createAB_CALCHECK" >> $LOGFILE

       				createAB -m SCIENCE -d $D -c config.createAB_CALCHECK >> $LOGFILE
			else
				echo "    no data available for $D, skipped." | tee -a $LOGFILE
				echo "    no data available for $D, skipped." >> $FORCED_LOG
			fi
		done
	fi

# normal date range	
	for D in `cat $TMP_DIR/cc_list_dates2`
	do
		if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ] && [ $INCREM = YES ] && [ $D != $TODAY ]
		then
			CHECK_OKK=`grep "$D" $PREVIOUS_DIR/cc_list_dates_OKK | sort -u`
			if [ Q$CHECK_OKK = Q$D ]
			then
				echo "    SCIENCE files for $D all OK, skipped." | tee -a $LOGFILE
				echo "    SCIENCE files for $D all OK, skipped." >> $FORCED_LOG
				continue
			fi
		fi
	
# INCREM=NO, or $D not OKK
		if [ -s $DFO_HDR_DIR/$D ]
		then
			cd $DFO_HDR_DIR/$D
			CHECK_EXIST=`ls | grep hdr | head -1`
			if [ Q$CHECK_EXIST != Q ]
			then
				CHECK_EXIST=`grep SCIENCE *hdr | head -1`
			fi

# CHECK_EXIST="": no SCIENCE hdr exist
			if [ "Q$CHECK_EXIST" = "Q" ]
			then
				echo "    no SCIENCE files found for $D, skipped." | tee -a $LOGFILE
				echo "    no SCIENCE files found for $D, skipped." >> $FORCED_LOG
	                        continue
			fi

# otherwise filter out 60/060 SCIENCE (unless ENABLE_60=YES)
			dfits *hdr | fitsort -d dpr.catg obs.prog.id > $TMP_DIR/cc_catg_prog

			if [ $ENABLE_60 = NO ]
			then
				CHECK_SCI60=`cat $TMP_DIR/cc_catg_prog | grep SCIENCE | awk '{print $1,$2,$3}' | egrep " 60\.| 060\."`
				if [ "Q$CHECK_SCI60" != "Q" ]
				then
					echo "    $D: SCIENCE files found with run_id starting with 60/060, will be skipped." | tee -a $LOGFILE
					echo "    $D: SCIENCE files found with run_id starting with 60/060, will be skipped." >> $FORCED_LOG
				fi
	
				CHECK_SCI=`cat $TMP_DIR/cc_catg_prog | grep SCIENCE | awk '{print $1,$2,$3}' | egrep -v " 60\.| 060\."`
			else
				CHECK_SCI=`cat $TMP_DIR/cc_catg_prog | grep SCIENCE | awk '{print $1,$2,$3}'`
			fi

			if [ "Q$CHECK_SCI" = "Q" ]
			then
				echo "    no SCIENCE files found for $D, skipped." | tee -a $LOGFILE
				echo "    no SCIENCE files found for $D, skipped." >> $FORCED_LOG
				continue
			else
				TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
			fi
			echo "  createAB -m SCIENCE -d $D -c config.createAB_CALCHECK (normal range)"
			echo "    calChecking $D (open issue) ..." >> $FORCED_LOG
			echo "" >> $LOGFILE
			echo "[$TIMESTAMP]  createAB -m SCIENCE -d $D -c config.createAB_CALCHECK" >> $LOGFILE
	
	       		createAB -m SCIENCE -d $D -c config.createAB_CALCHECK >> $LOGFILE
		else
			echo "    no data available for $D, skipped." | tee -a $LOGFILE
			echo "    no data available for $D, skipped." >> $FORCED_LOG
		fi
	done
fi

if [ $FORCED_MODE = YES ]
then
	writeLog "   ... finished."
	scpLog
fi

cd $TMP_DIR

# ======================================================================
# 4.0 Results
# 4.1 Generate list (we generate and analyze AB_list_SCIENCE even if it
#     results from CAL4CAL)
# ======================================================================

echo "" | tee -a $LOGFILE
case $INCREM in
  "YES" ) echo "- create result list (incremental mode) ..." | tee -a $LOGFILE ;;
  "NO"  ) echo "- create result list (full mode) ..."        | tee -a $LOGFILE ;;
esac

if [ $FORCED_MODE = YES ]
then
	writeLog "6. Analyse the results ..."
	scpLog
fi

RESULTS="$DFO_MON_DIR/calResults_${DFO_INSTRUMENT}"
RAW_RESULTS="${RESULTS}_raw"
RESULTS_NAME=`basename $RESULTS`

rm -f $RESULTS $RAW_RESULTS
rm -f $TMP_DIR/cc_list_comments 

# The OB grades cost time, we do them only once; either in CAL4CAL (and then skip in SCIENCE),
# or in SCIENCE if no result file ob_isql_out found
if [ $CHECK_C4C = YES ] && [ $CAL4CAL = YES ] 
then
	getObInfo -C
else
	if [ -s $TMP_DIR/CAL4CAL/ob_isql_out ]
	then
		cp $TMP_DIR/CAL4CAL/ob_isql_out* $TMP_DIR/
		echo "- get OB grades and comments: skipped, already done."
	else
		getObInfo -C
	fi
fi

# here we concatenate cc_list_dates2 (normal date range) and cc_list_open_issues (open issues from the memory zone)
# into $TMP_DIR/cc_list_dates4 and suppress any error resulting from non-existence of cc_list_open_issues (which would be legal)

rm -f $PREVIOUS_DIR/cc_list_memory_OKK
cat $TMP_DIR/cc_list_open_issues 2>/dev/null $TMP_DIR/cc_list_dates2 | sort -u > $TMP_DIR/cc_list_dates4 2>/dev/null 
for D in `cat $TMP_DIR/cc_list_dates4`
do
	CHECK_MEMORY=`grep $D $TMP_DIR/cc_list_open_issues 2>/dev/null`

	if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ] && [ $INCREM = YES ] && [ $D != $TODAY ]
	then
# check if $D is included in cc_list_dates_OKK; if YES, then ...
		CHECK_OKK=`grep "$D" $PREVIOUS_DIR/cc_list_dates_OKK | sort -u`
		if [ Q$CHECK_OKK = Q$D ] && [ -s $PREVIOUS_DIR/calResults_${DFO_INSTRUMENT} ] && [ -s $PREVIOUS_DIR/calResults_${DFO_INSTRUMENT}_raw ]
		then
# all OK, part of memory: no display!
			if [ Q$CHECK_MEMORY != Q ]
			then
				echo $D >> $PREVIOUS_DIR/cc_list_memory_OKK
				continue
			fi

# ... fill $RESULTS and $RAW_RESULTS for incremental mode
			grep "^$D" $PREVIOUS_DIR/calResults_${DFO_INSTRUMENT}     >> $RESULTS
			grep "^$D" $PREVIOUS_DIR/calResults_${DFO_INSTRUMENT}_raw >> $RAW_RESULTS

# ... fill OB comments for incremental mode
			if [ -s $PREVIOUS_DIR/cc_list_comments ]
			then
				grep " $D " $PREVIOUS_DIR/cc_list_comments >> $TMP_DIR/cc_list_comments
			fi

# ... fill alog files into $DFO_AB_DIR for incremental mode
			DATE1=`qcdate $D +1`
			rm -f $TMP_DIR/list_alogs
			ls $PREVIOUS_DIR | egrep "${DFO_FILE_NAME}.${D}T2[1-3]|${DFO_FILE_NAME}.${DATE1}T[0-1]|${DFO_FILE_NAME}.${DATE1}T20" | grep alog > $TMP_DIR/list_alogs
			if [ -s $TMP_DIR/list_alogs ]
			then
				for ALOG in `cat $TMP_DIR/list_alogs`
				do
					cp $PREVIOUS_DIR/$ALOG $DFO_AB_DIR/
				done	
			fi
# ... and leave the loop ...
			continue
		fi
	fi

# ... otherwise do it from scratch (i.e. for NOK dates and always for $TODAY):
    	if [ ! -s $DFO_MON_DIR/AB_list_SCIENCE_$D ]
       	then 
		continue
	fi

       	for AB in `cat $DFO_MON_DIR/AB_list_SCIENCE_$D | awk '{print $1}'`
	do
           	ALOG=`echo  $AB | sed "s/\.ab/.alog/"`
           	AFITS=`echo $AB | sed "s/_.*ab/.fits/" | sed "s/.ab/.fits/"`
		OBS_PROG_ID=`grep "^OBS_PROG_ID" 	$DFO_AB_DIR/$AB | awk '{print $2}'`

# CAL4CAL=NO: filter out OBS_PROG_IDs starting with 60./060., unless $ENABLE_60=YES
		if [ $CAL4CAL = NO ] && [ $ENABLE_60 = NO ]
		then
			OBS_PROG_ID1=`echo $OBS_PROG_ID | sed "s/060\..*//" | sed "s/60\..*//"`
			if [ Q$OBS_PROG_ID1 = Q ]
			then
				continue
			fi
		fi

		OBS_ID=`grep "^OBS_ID" 			$DFO_AB_DIR/$AB | awk '{print $2}'`
		OBS_MODE=`grep "^OBS_MODE" 		$DFO_AB_DIR/$AB | awk '{print $2}'`
		case $OBS_MODE in
		 "SM" ) OBS_MODE="SM_COLOR" ;;
		 "VM" ) OBS_MODE="VM_COLOR" ;;
		esac

# current alogs have sometimes a 2line header, sometimes not
		CHECK_HEAD=`grep "COMPLETE?" $DFO_AB_DIR/$ALOG`
		if [ "Q$CHECK_HEAD" != "Q" ]
		then
           		RT=`cat 		$DFO_AB_DIR/$ALOG | head -3 | tail -1 | awk '{print $2}'`
           		SETUP=`cat	 	$DFO_AB_DIR/$ALOG | head -3 | tail -1 | awk '{print $3}'`
                        if [ $APPLY_MODIF = TRUE ]
                        then
                                S_MODIF=`eval "echo $SETUP | $TMP_DIR/cc_modif_string"`
                        else
                                S_MODIF=$SETUP
                        fi
                        SETUP=$S_MODIF
           		COMP=`cat 		$DFO_AB_DIR/$ALOG | head -3 | tail -1 | awk '{print $4}'`
		else
           		RT=`cat 		$DFO_AB_DIR/$ALOG | head -1 | awk '{print $2}'`
           		SETUP=`cat	 	$DFO_AB_DIR/$ALOG | head -1 | awk '{print $3}'`

                        if [ $APPLY_MODIF = TRUE ]
                        then
                                S_MODIF=`eval "echo $SETUP | $TMP_DIR/cc_modif_string"`
                        else
                                S_MODIF=$SETUP
                        fi
                        SETUP=$S_MODIF
           		COMP=`cat 		$DFO_AB_DIR/$ALOG | head -1 | awk '{print $4}'`
		fi

# analyze COMPLETENESS and validity (OKK/NOK in association logs)
		rm -f $TMP_DIR/list_compl

# MISS (distinction MISS/LOST done later)
		if [ $COMP = INCOMPLETE ]
		then
			cat $DFO_AB_DIR/$ALOG | awk '{
			 if ( change == 1 ) { print $1":MISS" }
			 if ( $2 == "Missing:" ) { change=1 }
			}' change=0 |\
			sed "s/^.*/\*&\*/" >> $TMP_DIR/list_compl
		fi

# OKK/NOK
		egrep "^OK|^*NOK" $DFO_AB_DIR/$ALOG | awk '{if ($1 == "OK") {printf"%3s%1.2f%3s\n",$5":",$6,":OKK"} else {printf"%3s%1.2f%3s\n",$5":",$6,":NOK"}}' | sed "s/^.*/\*&\*/" >>  $TMP_DIR/list_compl

		ALL_MASTER_LIST=`cat $TMP_DIR/list_compl | tr "\012" "*"`
		for M in `cat $TMP_DIR/list_compl | sed "s/:/ /" | sed "s/\*//" | awk '{print $1}'`
		do
			RAW=`grep " ${M}$" $TMP_DIR/list_rawpro | awk '{print $1}'`
			sed -i -e "s/$M/$RAW/" $TMP_DIR/list_compl
		done

		ALL_RAW_LIST=`cat $TMP_DIR/list_compl | tr "\012" "*"`
		GRADE=`grep   " $OBS_ID " $TMP_DIR/ob_isql_out | grep "^ $D " | awk '{print $4}' | grep -v "_" | tail -1 | cut -c1,1`
		if [ "Q$GRADE" = Q ]
		then
			GRADE=Y
		fi

# same colors as in createReport	
		case $GRADE in
                  "A"|"B" ) OBCOLOR=$LGRN ;;
                  "C"|"D" ) OBCOLOR=$LRED ;;
                  "X"|"Y" ) OBCOLOR=$GRY  ;;
                esac

# cf. createReport "OB_COMMENT" for justification of the following; policy for multiple OB executions: display all comments, with UT_start
		OB_COMMENT=`grep " $OBS_ID " $TMP_DIR/ob_isql_out2 | grep " $D " | grep -v " NOCOMMENT" | sed "s/^.*UT_START//"g | sed "s/COMMENT//"g | sed "s/    //g" | sed "2,$ s/^.*/|| &/" | tr "\012" " " | tr "/" "_" | sed "s/\"/ldquo;/g"`

		if [ "Q$OB_COMMENT" != "Q" ]
		then
			echo "$OBS_ID $D $OB_COMMENT" >> $TMP_DIR/cc_list_comments
               	fi

# PROGID, OBGRADE, OBCOLOR, IREF, JREF are placeholders to be replaced later
                echo "$D PROGID$OBS_PROG_ID $OBS_MODE $OBS_ID ${OBCOLOR}OBCOLOR${GRADE}OBGRADE HREF${DFO_AB_DIR}/${ALOG}IREF${AFITS}JREF $RT $SETUP $ALL_MASTER_LIST" >> $RESULTS
                echo "$D PROGID$OBS_PROG_ID $OBS_MODE $OBS_ID ${OBCOLOR}OBCOLOR${GRADE}OBGRADE HREF${DFO_AB_DIR}/${ALOG}IREF${AFITS}JREF $RT $SETUP $ALL_RAW_LIST" >> $RAW_RESULTS
	done
done

# see createReport for justification of this (look for ldquo) (we clean for sed-incompatible strings like /&")
if [ -s $TMP_DIR/cc_list_comments ]
then
	cat $TMP_DIR/cc_list_comments | sort -u | tr "/" "_" | tr "&" "_" | sed "s/\"/ldquo;/g"  > $TMP_DIR/cc_list_comments1
	mv $TMP_DIR/cc_list_comments1 $TMP_DIR/cc_list_comments
fi

# ======================================================================
# 4.1.2 Clean the $RESULT file for OKK dates from the memory zone
# ======================================================================

# cc_list_open_issues: from CAL_ANALYSIS, memory zone
if [ -s $TMP_DIR/cc_list_open_issues ]
then
	for D in `cat $TMP_DIR/cc_list_open_issues`
	do
# [do TESTS for memory here]
		CHECK_NOK=`grep "^$D" $RESULTS | egrep ":NOK|:MISS|:LOST" | head -1`
		if [ "Q$CHECK_NOK" = Q ]
		then
# no issue: add $D to PREVIOUS/cc_list_memory_OKK for next run
			echo $D >> $PREVIOUS_DIR/cc_list_memory_OKK
# ... and remove it from cc_list_open_issues 
			sed -i -e "/^$D/d" $TMP_DIR/cc_list_open_issues
		fi
	done
fi

# ======================================================================
# 4.2 Consolidation: one result per setup and $DATE
# ======================================================================

echo "- consolidate result list ..." | tee -a $LOGFILE
CONSOL="$TMP_DIR/calChecker.consol"
rm -f $CONSOL 

# CAL_ANALYZED: file with analyzed NOK/MISS cases
if [ ! -s $DFO_MON_DIR/CAL_ANALYZED ]
then
	touch $DFO_MON_DIR/CAL_ANALYZED
fi

if [ -s $RESULTS ]
then
	rm -f ${RESULTS}_*.html
	for D in `cat $TMP_DIR/cc_list_dates4`
	do
# if more columns added in $RESULTS, then change awk part correspondingly! #COLUMNS
		grep "^$D" $RESULTS | awk '{print $1,$7,$8,$9}' | sort -u >> $CONSOL	
	done
fi

# ======================================================================
# 4.3 Create daily output tables
# 4.3.1 Header
# ======================================================================

echo "- create daily calResults pages ..."

# info text, to appear on top of the daily calResults pages; text differs for science and CAL4CAL output
case $ENABLE_60 in
 "NO"  ) TEXT_60="- All science data with PROG_ID starting with 60./060. are ignored.<br>" ;;
 "YES" ) TEXT_60="- Science data with PROG_ID starting with 60./060. are included.<br>" ;;
esac

if [ $CAL4CAL = NO ]
then
	cat > $TMP_DIR/result_text <<EOT
- This is the detailed calChecker report about the calibrations for all science OBs for the indicated date.<br>
$TEXT_60
- This report flags calibrations that are </i>formally<i> missing (marked in yellow or red). This formal result may be overridden by the analysis (as indicated in the ANALYSIS notes). This analysis result, being displayed on the main calChecker interface but not here, is the final word. <br>
EOT
	cat > $TMP_DIR/result_text2 <<EOT
- OB comments are truncated after 40 characters. Point your mouse on the comment field to read the full comment, or check the nightlog ('NLT').<br>
- Multiple comments for the same OB in the same night are all displayed, separated by '||'.<br>
- The table can be sorted and filtered.<br>
- Press Shift key for multiple column sorting (default is DATA_TYPE, SETUP and RAW_FILE).
EOT
else
	cat > $TMP_DIR/result_text <<EOT
- This is the detailed calChecker report about calibrations for calibrations (CAL4CAL) for the indicated date. 
Note that this report lists only the triggering calibrations (the ones for which other calibrations should exist).<br>
- This report flags dependent calibrations that are </i>formally<i> missing (marked in yellow or red). This formal result may be overridden by the analysis (as indicated in the ANALYSIS notes). This analysis result, being displayed on the main calChecker interface but not here, is the final word. <br>
EOT
	cat > $TMP_DIR/result_text2 <<EOT
- The table can be sorted and filtered.<br>
- Press Shift key for multiple column sorting (default is DATA_TYPE, SETUP and RAW_FILE).<br>
EOT
fi

# for navigation:
LAST_D=`cat $TMP_DIR/cc_list_dates2 | tail -1`

if [ $FORCED_MODE = YES ]
then
	writeLog "7. Create output HTML table ..."
	scpLog
fi

# loop over $D
for D in `cat $TMP_DIR/cc_list_dates4`
do
	if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ] && [ $INCREM = YES ] && [ $D != $TODAY ]
	then
		CHECK_OKK=`grep "$D" $PREVIOUS_DIR/cc_list_dates_OKK | sort -u`
		if [ Q$CHECK_OKK = Q$D ]
		then
			echo "   $D: all OK, skipped."
			continue
		fi
	fi

	echo "   $D ..."

# for navigation
	DATE_minus=`qcdate $D -1`	
	DATE_plus=`qcdate  $D +1`	
	PREV_HTMLTAG="<a href=calResults_${DFO_INSTRUMENT}_${DATE_minus}.html><img src=${IMG_URL}/left.gif width=16 height=17 border=0 title=\"previous night\"></a>"
	NEXT_HTMLTAG="<a href=calResults_${DFO_INSTRUMENT}_${DATE_plus}.html><img src=${IMG_URL}/right.gif width=16 height=17 border=0 title=\"next night\"></a>"

	if [ -s $RESULTS ]
	then
		CHECK_D=`grep "^$D" $RESULTS`
	else
		CHECK_D=""
	fi

	case $CAL4CAL in
	 "NO"  ) TITLE="$DFO_INSTRUMENT calChecker: Calibration report for all science files, date $D" ;;
	 "YES" ) TITLE="$DFO_INSTRUMENT calChecker: Completeness report for all calibration files, date $D" ;;
	esac
	
	cat > $TMP_DIR/core.html <<EOT
<html>
<head>
<title>${TITLE}</title>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery-latest.js"></script>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery.dataTables.js"></script>
<script type="text/javascript">
\$(document).ready(function() 
{ 
	\$("#mainTable").dataTable({
//index starts with zero!
         "aaSorting": [[6,'asc'],[7,'asc'],[8,'asc']],
         "aoColumnDefs":
           [ { "bSortable": false, "aTargets": [ 0,5,9,10,11 ] } ],
         "oLanguage": {
                "sInfo": "<i><font color=#666666 size=1>Showing _START_ to _END_ of _TOTAL_ entries</i>",
                "sSearch": "<font color=#666666 size=1><i>Search (case-insensitive):</i>",
                "sInfoEmpty": "<font color=#666666 size=1><i>Search: Showing 0 to 0 of 0 entries",
                "sInfoFiltered": "(filtered from _MAX_ total entries)</font>"
        },
        "bPaginate": false,
        "sDom": '<fitif>'
	});
	});
</script>

EOT
	cat $TMP_DIR/cal_obcomments >> $TMP_DIR/core.html

	cat >> $TMP_DIR/core.html <<EOT

<link rel="StyleSheet" href="$STYLES" type="text/css">
<!-- styles for datatable-->
<style type="text/css">
table.display {
        text-align:     left;
        font-family:    arial;
        font-size:      small;
        height:         10px;
}
table.display thead tr .header {
        background-color:    ${HDRSORT_COLOR};
        background-image:    url(${IMG_URL}/bg.gif) no-repeat center right;
        height:              30px;
        vertical-align:      middle;
}
table.display thead tr .sorting_desc {
        background-color:    ${HDRSORT_COLOR};
        background:          url(${IMG_URL}/desc.gif) no-repeat center right;
}
table.display thead tr .sorting {
        background-color:    ${HDRSORT_COLOR};
        background:          url(${IMG_URL}/bg.gif) no-repeat center right;
}
table.display thead tr .sorting_asc {
        background-color:    ${HDRSORT_COLOR};
        background:    	     url(${IMG_URL}/asc.gif) no-repeat center right;
}
</style>
</head>

<body>
EOT

# ======================================================================
# 4.3.2 no OBs --> create empty page with navigation only
# ======================================================================

	if [ "Q$CHECK_D" = "Q" ]
	then
		case $CAL4CAL in
	 	 "NO"  ) NO_SCIENCE="- no science files found, nothing to report." ;;
	 	 "YES" ) NO_SCIENCE="- no calibration files to check, nothing to report." ;;
		esac
		
		cat $TMP_DIR/core.html > ${RESULTS}_$D.html
		cat >> ${RESULTS}_$D.html <<EOT

<table>
  <tr>
    <td COLSPAN=100 bgcolor=#CCCCCC>

      <table cellpadding="0" cellspacing="0" border=0>
        <tr align="left" valign="bottom">
         <td>

           <table style="border-collapse: collapse;" border="1" bordercolor="#ffffff" cellpadding="0" cellspacing="0">
             <tr>
               <td bgcolor="${REPORT_COLOR}" height="40" width="10" align="left" valign="bottom"><b><font color="#ffffff" size="3">
                 <a title="detailed overview of calibration completeness">CAL</a></font></b>
               </td>
             </tr>
           </table>

         </td>
         <td valign="bottom" nowrap><b><font color=${REPORT_COLOR} size=+2>${TITLE}</font></b></td>
        </tr>
      </table>

    </td>
  </tr>
  <tr>		<!--LOCAL-->
    <td>$PREV_HTMLTAG $NEXT_HTMLTAG <font size=2><a href="javascript:self.close();">close window</a> | <a href="./${RESULTS_NAME}_setup.html">setup table</a> | <a href="./${RESULTS_NAME}_setup_all.html">search full table</a> </font></td> <!--LOCAL-->
  </tr>		<!--LOCAL-->
  <tr>
    <td><font size=2>${NO_SCIENCE}</td>
  </tr>
</table>
</body>
</html>
EOT
		mv ${RESULTS}_$D.html ${CALREPORT_DIR}/calResults_${DFO_INSTRUMENT}_$D.html
		continue
	fi

# ======================================================================
# 4.3.3 OBs --> continue with results
# ======================================================================

	cat $TMP_DIR/core.html > ${RESULTS}_$D.html

# report existing? Link! (only for qcweb and local version, not for HC monitor, since too much info)
	if [ -s $DFO_LST_DIR/REPORT/list_${DFO_INSTRUMENT}_${D}_data.html ]
	then
      		REPORT="<b><a href=${REPORT_DIR}/list_${DFO_INSTRUMENT}_${D}_data.html target=_blank title=\"raw data report [internal link]\"><font color=${REPORT_COLOR} size=1>report</font></a> <!--REPORT-->"
	else
		REPORT="<font color=#666666 size=1><a title=\"no data taken\">no data</a></font>"
	fi

# NR link (only for qcweb and local version)
	NR="<a href=\"${NLT_URL}?date=${D}&instrument=$PANL_INSTRUMENT\" title=\"instrument night report from NLT [internal link, password required]\" target=\"_blank\"><font size=2 color=${REPORT_COLOR}>NLT</font></a>&nbsp;"

# fill in the navigation, text etc.
	cat >> ${RESULTS}_$D.html <<EOT

<table width=1050> 
  <tr>
    <td COLSPAN=100 bgcolor=#CCCCCC>
      <table cellpadding="0" cellspacing="0" border=0>
        <tr align="left" valign="bottom">
         <td>

           <table style="border-collapse: collapse;" border="1" bordercolor="#ffffff" cellpadding="0" cellspacing="0">
             <tr>
               <td bgcolor="${REPORT_COLOR}" height="40" width="10" align="left" valign="bottom"><b><font color="#ffffff" size="3">
                 <a title="detailed overview of calibration completeness">CAL</a></font></b>
               </td>
             </tr>
           </table>

         </td>
         <td valign="bottom" nowrap><b><font color=${REPORT_COLOR} size=+2>${TITLE}</font></b></td>
        </tr>
      </table>

    </td>
  </tr>
  <tr>	<!--LOCAL-->
    <td>$PREV_HTMLTAG $NEXT_HTMLTAG <font size=2><a href="javascript:self.close();">close window</a> | <a href="./${RESULTS_NAME}_setup.html">setup table</a> | <a href="./${RESULTS_NAME}_setup_all.html">search full table</a> </font></td> <!--LOCAL-->
  </tr>	<!--LOCAL-->
  <tr><td colspan=99>
    <table width=1050>
      <tr>
        <td width=525 valign=top align=left bgcolor=#eeeeee><font size=2 color=#666666><i>
EOT
	cat $TMP_DIR/result_text >> ${RESULTS}_$D.html

	cat >> ${RESULTS}_$D.html <<EOT
        </i></font></td>
        <td width=525 valign=top align=left bgcolor=#eeeeee><font size=2 color=#666666><i>
EOT
	cat $TMP_DIR/result_text2 >> ${RESULTS}_$D.html

	cat >> ${RESULTS}_$D.html <<EOT
      </i></font></td>
     </tr>
    </table>
  </td></tr>

  <tr><td nowrap COLSPAN=99><font size=2>
<a href=#bottom title="jump to bottom of page"><font color=${REPORT_COLOR}><b>bottom<b></font></a> &nbsp; <font color=${REPORT_COLOR}>|</font> &nbsp;
$REPORT &nbsp; <font color=${REPORT_COLOR}>|</font> &nbsp; 
$NR
    </font></td></tr>
EOT

# start the sortable table
	cat >> ${RESULTS}_$D.html <<EOT
    <tr><td colspan=99>
      <table width=100% cellpadding=0 cellspacing=0>
        <tr>
          <td align=left width=50%><font size=1>OBs: 
EOT

# list of OBs (used for OB navigation)
	cat $RAW_RESULTS |\
 grep "^$D " |\
 awk '{print $4}' |\
 sort -u |\
 sed "s|^.*|<a href=#&>&</a>\&nbsp;|" >> ${RESULTS}_$D.html

	cat >> ${RESULTS}_$D.html <<EOT
  </font></td>

  <td align=left>
    <font size=1 color=#666666><i>&nbsp; &nbsp;The table can be searched and filtered. Restore default content with the browser refresh button.</i></font>
  </td></tr></table>
  </td>
</tr>

  <tr bgcolor=#CCCCCC>
    <td COLSPAN=99 bgcolor=#FFFFFF>

<!-- begin of sortable table -->
<table id="mainTable" class="display" BORDER=0>
  <thead>
    <tr bgcolor=#cccccc>
      <th><font size=2><a title="Date (defined to change at 21:00 UT)">DATE</a></th>
      <th><font size=2><a title="run ID as read from the header">PROG_ID</a></th>
      <th><font size=2><a title="ServiceMode or VisitorMode">MODE</a></th>
      <th><font size=2><a title="OB ID as read from the header">OBS_ID</a></th>
      <th><font size=2><a title="OB grade (A|B|C|D or X) as read from the nightlog database">GRD</a></th>
      <th nowrap width=240><font size=2><a title="OB comment as read from the nightlog database">OB Comm.</a></th>
      <th><font size=2><a title="ARCFILE name of the first raw file produced by this OB">(first) RAW FILE</a></th>
      <th><font size=2><a title="data type as defined in the ASSOC-RULES and used on the main result page">DATA TYPE</a></th>
      <th><font size=2><a title="setup as used on the main result page">SETUP</a></th>
      <td bgcolor=${HDRSORT_COLOR}><font size=1>&nbsp;</td>
      <th COLSPAN=20 align=left><font size=2><a title="list of calibration data types and delays in days, if any">CALIBRATIONS</a></th>
    </tr>
    </thead>

    <tbody>
EOT

# modify setup strings (key 4) if configured; do it in a loop since otherwise you may edit more than just key 4
	rm -f $TMP_DIR/setups1
	cat $CONSOL | grep "$D" | awk '{print $3,$2}' | sort -u > $TMP_DIR/setups1

	for S in `cat $TMP_DIR/setups1 | awk '{print $1}'`
	do
		if [ $APPLY_MODIF = TRUE ]
		then
			S_MODIF=`eval "echo $S | $TMP_DIR/cc_modif_string"`
			sed -i -e ":$S:s!$S!${S_MODIF}!" $RAW_RESULTS
		fi
	done

# the following is shaky grounds ... (line-by-line loop would be cleaner but would cost performance)
	cat $RAW_RESULTS | grep "^$D " |\
 sed "s/ \*/SEPARATOR\*/" |\
 sed "s/\*\*\*/\*/g" |\
 sed "s/\*\*/\*<\/tr>/g" |\
 sed "s/ $//g" |\
 sed "s/ /<\/td><td nowrap><font size=2>/g" |\
 sed "s/^.*/<tr bgcolor=#CCCCCC><td><font size=2><b>&/" |\
 tr "*" "\012" |\
 sed "s/^.*OKK/<td bgcolor=${OK_COLOR} nowrap><font size=1>&/g"    | sed "s/:OKK//g"  | sed "s/[A-Z]:/& /"|\
 sed "s/^.*NOK/<td bgcolor=${NOK_COLOR} nowrap><font size=1>&/g"   | sed "s/:NOK//g"  | sed "s/[A-Z]:/& /"|\
 sed "s/^.*MISS/<td bgcolor=${MISS_COLOR} nowrap><font size=1>&/g" | sed "s/:MISS//g" | sed "s/[A-Z]:/& /"|\
 sed "s/^.*/&<\/td>/" |\
 sed "s/<\/tr><\/td>/<\/tr>/" |\
 sed "s/^.*aname/<a name/" |\
 sed "s/ahref/a href/"g |\
 sed "s|SETUP1|top|" |\
 sed "/NEWLINE:/s|^.*|<tr><td COLSPAN=20 bgcolor=#FFFFFF>&</td></tr>|" |\
 sed "s/NEWLINE:.*:NEWLINE//" |\
 sed "s/<\/td><hr noshade/<hr noshade/" |\
 sed "s/VM_COLOR/<font color=${VM_COLOR}><b>VM<\/font>/" |\
 sed "s/SM_COLOR/<font color=${SM_COLOR}><b>SM<\/font>/" |\
 sed "s|NONE|<font color=#666666>n/a</font>|" |\
 sed "s/HREF/<a href=/" |\
 sed "s/IREF/ target=_blank>/" |\
 sed "s/JREF/<\/a>/" |\
 sed "s/SEPARATOR/<\/td><td bgcolor=${HDRSORT_COLOR}> /" |\
 sed "s/aname=/a name=/" \
>> ${RESULTS}_$D.html

	TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
	cat >> ${RESULTS}_$D.html <<EOT
  </tbody>
  
  <tfoot>
    <tr bgcolor=#cccccc>
      <th><font size=2><a title="Date (defined to change at 21:00 UT)">DATE</a></th>
      <th><font size=2><a title="run ID as read from the header">PROG_ID</a></th>
      <th><font size=2><a title="ServiceMode or VisitorMode">MODE</a><font color=#cccccc>__</font></font></th>
      <th><font size=2><a title="OB ID as read from the header">OBS_ID</a><font color=#cccccc>____</font></font></th>
      <th><font size=2><a title="OB grade (A|B|C|D or X) as read from the nightlog database">GRD</a><font color=#cccccc>__</font></font></th>
      <th nowrap><font size=2><a title="OB comment as read from the nightlog database">OB Comm.</a></font></th>
      <th><font size=2><a title="ARCFILE name of the first raw file produced by this OB">(first) RAW FILE</a></font></th>
      <th nowrap><font size=2><a title="data type as defined in the ASSOC-RULES and used on the main result page">DATA TYPE</a><font color=#cccccc>__</font></font></th>
      <th><font size=2><a title="setup as used on the main result page">SETUP</a><font color=#cccccc>____</font></font></th>
      <td bgcolor=${HDRSORT_COLOR}><font size=1>&nbsp;</td>
      <th COLSPAN=20 align=left><font size=2><a title="list of calibration data types and delays in days, if any">CALIBRATIONS</a></th>
  </tr>
  <tr>
    <td bgcolor=#CCCCCC><a name="bottom"><a href="#top" title="jump to top of page"><font size=2 color=${REPORT_COLOR}><b>top</b></a></td>
    <td COLSPAN=99 bgcolor=#CCCCCC align=right><font size=2 color=#FFFFFF>last update: $TIMESTAMP (UT) &nbsp; &nbsp; &nbsp; <b>powered by QC </b>[calChecker v${TOOL_VERSION}]</td>
  </tr>
  </tfoot>
</table>
<!-- end of sortable table>

  </tr>
 </table>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
</body>
</html>
EOT

# OBs (SCIENCE same as CAL4CAL)
	for OB in `grep "^ $D " $TMP_DIR/ob_isql_out | awk '{print $2}' | sort -u`
	do
		rm -f $TMP_DIR/cc_affected_files
		grep " $OB " ${RAW_RESULTS} | grep "^$D " | awk '{print $6}' | sed "s/^.*IREF//" | sed "s/JREF//" > $TMP_DIR/cc_affected_files

		if [ -s $TMP_DIR/cc_list_comments ]
		then
			CHECK_COMMENT=`grep "^$OB " $TMP_DIR/cc_list_comments | grep " $D "  | tail -1`
		fi

		for F in `cat $TMP_DIR/cc_affected_files`
		do
			if [ "Q$CHECK_COMMENT" = "Q" ]
			then
				OB_COMMENT=" "
			else
				OB_COMMENT=`grep "^$OB " $TMP_DIR/cc_list_comments | grep " $D " | sed "s/$OB $D//"`
			fi

# 'ldquo;' is sed-proof for 'double quote'; replaced later
			LENGTH=`echo "$OB_COMMENT" | sed "s/ldquo;/\"/g" | wc -c`

# shorten and clean for '/'
			if [ $LENGTH -gt 40 ]
			then
				SHORT_COMM=`echo "$OB_COMMENT" | sed "s|/| |g" | sed "s/ldquo;/\"/g" | cut -c 1-40 | sed "s/^.*/&\.\.\./" | sed "s/\"/ldquo;/g"`
			else
				SHORT_COMM=`echo "$OB_COMMENT" | sed "s|/| |g"`
			fi

# insert comments from nightlog (using OBCOLOR.OBGRADE as marker)
# the onMouseOver function relies on the entry OBcomm in the $STYLES file, make sure it exists
# doesn't work, crashes with tablesorter:
# sed "/$F/s/OBCOLOR.OBGRADE/&<\/td><td><font size=1><a onMouseOver=\"showOBcomm('${F}')\" onMouseOut=\"hideOBcomm()\">${SHORT_COMM}<\/a>/" |\
# sed "/$F/s/^.*/<div class=\"OBcomm\" id=\"$F\">${OB_COMMENT}<\/div>&/" |\

			cat ${RESULTS}_$D.html |\
 sed "/$F/s/OBCOLOR.OBGRADE/&<\/td><td><font size=1><a title=\"${OB_COMMENT}\">${SHORT_COMM}<\/a>/" |\
 sed "/$F/s/<td>/&<a name=\"${OB}\"><\/a>/" |\
 sed "/$F/s/OBCOLORYOBGRADE/OBCOLOR<font color=#666666>n\/a<\/font>/" |\
 sed "/$F/s/OBGRADE//g" |\
 sed "/$F/s/><font size=2>PROGID/ nowrap><font size=2>/" |\
 sed "/$F/s/><font size=2>#......OBCOLOR/ bgcolor=&/" |\
 sed "/$F/s/><font size=2>#/#/" |\
 sed "/$F/s/OBCOLOR/><font size=2>/" |\
 sed "/$F/s/font size=2/font size=1/g" |\
 sed "/$F/s/ldquo/\&ldquo/g" \
  > ${RESULTS}_$D.html1
# the second last line (2->1) is done in v4.0 at the end to make the output more compact
			mv ${RESULTS}_$D.html1 ${RESULTS}_$D.html
		done
	done

# just in case a file has no grade (OB has no entry in ob_isql_out --> no entry in affected_files)
        cat ${RESULTS}_$D.html |\
 sed "s/OBCOLOR.OBGRADE/&<\/td><td><font size=1> /" |\
 sed "s/OBCOLORYOBGRADE/OBCOLOR<font color=#666666>n\/a<\/font>/" |\
 sed "s/PROGID//g" | sed "s/OBGRADE//g" |\
 sed "s/><font size=2>#......OBCOLOR/ bgcolor=&/" |\
 sed "s/><font size=2>#/#/" |\
 sed "s/OBCOLOR/><font size=2>/" > ${RESULTS}_$D.html1
        mv ${RESULTS}_$D.html1 ${RESULTS}_$D.html

# final place for ${RESULTS}_$D.html: ${CALREPORT_DIR}
	mv ${RESULTS}_$D.html ${CALREPORT_DIR}/calResults_${DFO_INSTRUMENT}_$D.html
done

# ======================================================================
# 4.3.4 setup table; links back ("back to main") to public server page
# ======================================================================

echo "- create calResults setup table ..."

# this string is encoded in one line since it needs to be replicated several times
THEAD="<thead><tr bgcolor=#CCCCCC><th><a title=\"Date (defined to change at 21:00 UT)\">DATE</a></th><th><a title=\"run ID as read from the header\">PROG_ID</a></th><th><a title=\"Service Mode or Visitor Mode\">MODE<font color=#CCCCCC>___</font></a></th><th><a title=\"OB ID as read from the header\">OBS_ID<font color=#CCCCCC>___</font></a></th><th><a title=\"OB grade (A,B,C,D or X) as read from the nightlog database\">GRD<font color=#CCCCCC>___</font></a></th><th><a title=\"ARCFILE name of the first raw file produced by this OB\">(first) RAW FILE</a></th><th><a title=\"data type as defined in the ASSOC-RULES and used on the main page\">DATA TYPE<font color=#CCCCCC>___</font></a></th><th><a title=\"setup as used on the main result page\">SETUP</a></th><th> </th><th COLSPAN=20 align=left><a title=\"list of calibration data types and delays in days, if any\">CALIBRATIONS</a></th> </tr> </thead>"

if [ $CAL4CAL = NO ]
then
	cat > $TMP_DIR/result_setup_text <<EOT
This is the detailed calChecker report about the calibrations for all science OBs and all dates, sorted by setups.<br>
$TEXT_60
- This report flags calibrations that are </i>formally<i> missing (marked in yellow or red). This formal result may be overridden by the analysis (as indicated in the ANALYSIS notes). This analysis result, being displayed on the main calChecker interface but not here, is the final word. <br>
- The tables can be sorted and filtered.<br>
- Restore defaults with the browser refresh button. 

EOT
else
	cat > $TMP_DIR/result_setup_text <<EOT
This is the detailed calChecker report about calibrations for calibrations (CAL4CAL), sorted by setups.<br>
- This report flags calibrations that are </i>formally<i> missing (marked in yellow or red). This formal result may be overridden by the analysis (as indicated in the ANALYSIS notes). This analysis result, being displayed on the main calChecker interface but not here, is the final word. <br>
- The tables can be sorted and filtered.<br>
- Restore defaults with the browser refresh button. 
EOT
fi

case $CAL4CAL in
 "NO"  ) TITLE="$DFO_INSTRUMENT calChecker: Calibration report for all science files, sorted by setups" ;;
 "YES" ) TITLE="$DFO_INSTRUMENT calChecker: Completeness report for all calibration files, sorted by setups" ;;
esac

if [ -s $CONSOL ]
then
	cat > ${RESULTS}_setup.html <<EOT
<html>
<head>
<title>${TITLE}</title><a name=top>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery-latest.js"></script>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery.dataTables.js"></script>
<script type="text/javascript">
\$(document).ready(function() 
{ 
	\$('.dataTable').dataTable({
         "aaSorting": [[5,'asc'],[6,'asc'],[7,'asc']],
         "aoColumnDefs":
           [ { "bSortable": false, "aTargets": [ 8,9 ] } ],
         "oLanguage": {
                "sInfo": "<i><font color=#666666 size=1>Showing _START_ to _END_ of _TOTAL_ entries</i>",
                "sSearch": "<font color=#666666 size=1><i>Search (case-insensitive):</i>",
                "sInfoEmpty": "<font color=#666666 size=1><i>Search: Showing 0 to 0 of 0 entries",
                "sInfoFiltered": "(filtered from _MAX_ total entries)</font>"
        },
        "bPaginate": false,
        "sDom": '<itf>'
	});

	});
</script>

<link rel="StyleSheet" href="$STYLES" type="text/css">
<!-- styles for datatable-->
<style type="text/css">
table.display {
        text-align:     left;
        font-family:    arial;
        font-size:      small;
        height:         10px;
}
table.display thead tr .header {
        background-color:    ${HDRSORT_COLOR};
        background-image:    url(${IMG_URL}/bg.gif) no-repeat center right;
        height:              30px;
        vertical-align:      middle;
}
table.display thead tr .sorting_desc {
        background-color:    ${HDRSORT_COLOR};
        background:          url(${IMG_URL}/desc.gif) no-repeat center right;
}
table.display thead tr .sorting {
        background-color:    ${HDRSORT_COLOR};
        background:          url(${IMG_URL}/bg.gif) no-repeat center right;
}
table.display thead tr .sorting_asc {
        background-color:    ${HDRSORT_COLOR};
        background:    	     url(${IMG_URL}/asc.gif) no-repeat center right;
}
</style>
</head>

<body>
<table>
  <tr>
    <td COLSPAN=100 bgcolor=#CCCCCC>

      <table cellpadding="0" cellspacing="0" border=0>
        <tr align="left" valign="bottom">
         <td>

           <table style="border-collapse: collapse;" border="1" bordercolor="#ffffff" cellpadding="0" cellspacing="0">
             <tr>
               <td bgcolor="${REPORT_COLOR}" height="40" width="40" align="left" valign="bottom"><b><font color="#ffffff" size="3">
                 <a title="detailed overview of calibration completeness">CAL</a></font></b>
               </td>
             </tr>
           </table>

         </td>
         <td valign="bottom"><b><font color=${REPORT_COLOR} size=+2>${TITLE}</font></b></td>
        </tr>
      </table>
    </td>
  </tr>

  <tr>
    <td colspan=99>
      <font size=2><a href="javascript:self.close();">close window</a> | <a href=./${RESULTS_NAME}_setup_all.html>search full table</a>
    </td>
  </tr>

  <tr bgcolor=#eeeeee><td colspan=99><font size=2 color=#666666><i>
EOT
	cat $TMP_DIR/result_setup_text >> ${RESULTS}_setup.html
	cat >> ${RESULTS}_setup.html <<EOT
</i></font></td></tr>
</table>

$BOTTOM
<table class="display"> <!--SORTED-->
<tr bgcolor=#CCCCCC>	<!--SORTED-->
  <th>SETUP</th>	<!--SORTED-->
</tr>			<!--SORTED-->
EOT

	cat $TMP_DIR/setups | awk '{print $1}' | sed "s|^.*|<tr bgcolor=#eeeeee><td><font size=1><a href=\"#&\">&</a></font></td></tr>  <!--SORTED-->|" >> ${RESULTS}_setup.html

	cat >> ${RESULTS}_setup.html <<EOT
</table>	<!--SORTED-->

${TOP} <font size=2><b> FIRSTSETUP:<p>	<!--SORTED-->

<table class="display dataTable" BORDER=0 WIDTH="100%">
$THEAD

<tbody>
EOT

# setup: $8 (if more columns added, make sure to correct this index for setup string!) #COLUMNS
	FIRST_SETUP=`cat $RAW_RESULTS | awk '{print $8}' | sort -u | head -1`
	cat $RAW_RESULTS | sort -k8 |\
 awk '{ if ( $8 == prevsetup ) { print }
        if ( $8 != prevsetup ) { prevsetup=$8; print "<aname="$8"></a>"$8":NEWLINE"; print }
      }' prevsetup="$FIRST_SETUP" > ${RESULTS}2

	if [ $CAL4CAL = YES ]
	then
       		cat ${RESULTS}2 |\
 sed "s/\*\*\*/\*/g" |\
 sed "s/\*\*/\*<\/tr>/g" |\
 sed "s/ $//g" |\
 sed "s/ /<\/td><td><font size=2>/g" |\
 sed "s/^.*/<tr bgcolor=#CCCCCC><td><font size=2><b>&/" |\
 tr "*" "\012" |\
 sed "s/^.*OKK/<td bgcolor=${OK_COLOR} nowrap><font size=1>&/g"    | sed "s/:OKK//g"  | sed "s/[A-Z]:/& /"|\
 sed "s/^.*NOK/<td bgcolor=${NOK_COLOR} nowrap><font size=1>&/g"   | sed "s/:NOK//g"  | sed "s/[A-Z]:/& /"|\
 sed "s/^.*MISS/<td bgcolor=${MISS_COLOR} nowrap><font size=1>&/g" | sed "s/:MISS//g" | sed "s/[A-Z]:/& /"|\
 sed "s/^.*/&<\/td>/" |\
 sed "s/<\/tr><\/td>/<\/tr>/" |\
 sed "s/^.*aname/<a name/" |\
 sed "s|^.*NEWLINE.*|</tbody></table>$TOP<p> <font size=2><b>&<table class=\"display dataTable\" BORDER=0 WIDTH=100%>${THEAD}<tbody><!--NEWLINE-->|" |\
 sed "s/VM_COLOR/<font color=${VM_COLOR}><b>VM<\/font>/" | sed "s/SM_COLOR/<font color=${SM_COLOR}><b>SM<\/font>/" |\
 sed "s/HREF/<a href=/" | sed "s/IREF/ target=_blank>/" | sed "s/JREF/<\/a>/" |\
 sed "s/aname=/a name=/" |\
 sed "s/OBCOLORYOBGRADE/OBCOLOR<font color=#666666>n\/a<\/font>/" |\
 sed "s/><font size=2>#......OBCOLOR/ bgcolor=&/" |\
 sed "s/><font size=2>#/#/" |\
 sed "s/OBCOLOR/><font size=2>/" |\
 sed "s|NONE|<font color=#666666>n/a</font>|" |\
 sed "s/OBGRADE//g" |\
 sed "s/><font size=2>PROGID/ nowrap><font size=2 color=#666666>/g" |\
 sed "s/font size=2/font size=1/g" \
>> ${RESULTS}_setup.html
	else
       		cat ${RESULTS}2 |\
 sed "s/\*\*\*/\*/g" |\
 sed "s/\*\*/\*<\/tr>/g" |\
 sed "s/ $//g" |\
 sed "s/ /<\/td><td><font size=2>/g" |\
 sed "s/^.*/<tr bgcolor=#CCCCCC><td><font size=2><b>&/" |\
 tr "*" "\012" |\
 sed "s/^.*OKK/<td bgcolor=${OK_COLOR} nowrap><font size=1>&/g"    | sed "s/:OKK//g"  | sed "s/[A-Z]:/& /"|\
 sed "s/^.*NOK/<td bgcolor=${NOK_COLOR} nowrap><font size=1>&/g"   | sed "s/:NOK//g"  | sed "s/[A-Z]:/& /"|\
 sed "s/^.*MISS/<td bgcolor=${MISS_COLOR} nowrap><font size=1>&/g" | sed "s/:MISS//g" | sed "s/[A-Z]:/& /"|\
 sed "s/^.*/&<\/td>/" |\
 sed "s/<\/tr><\/td>/<\/tr>/" |\
 sed "s/^.*aname/<a name/" |\
 sed "s|^.*NEWLINE.*|</tbody></table>$TOP <font size=2><b>&<p><table class=\"display dataTable\" BORDER=0 WIDTH=100%>${THEAD}<tbody><!--NEWLINE-->|" |\
 sed "s/VM_COLOR/<font color=${VM_COLOR}><b>VM<\/font>/" | sed "s/SM_COLOR/<font color=${SM_COLOR}><b>SM<\/font>/" |\
 sed "s/HREF/<a href=/" | sed "s/IREF/ target=_blank>/" | sed "s/JREF/<\/a>/" |\
 sed "s/aname=/a name=/" |\
 sed "s/OBCOLORYOBGRADE/OBCOLOR<font color=#666666>n\/a<\/font>/" |\
 sed "s/><font size=2>#......OBCOLOR/ bgcolor=&/" |\
 sed "s/><font size=2>#/#/" |\
 sed "s/OBCOLOR/><font size=2>/" |\
 sed "s/OBGRADE//g" |\
 sed "s/><font size=2>PROGID/ nowrap><font size=2>/g" |\
 sed "s/font size=2/font size=1/g" \
>> ${RESULTS}_setup.html
	fi
# the last line has been introduced with v4.0 and is for more compact output

	grep -v "<!--NEWLINE-->" ${RESULTS}_setup.html |\
 grep -v "<!--SORTED-->" |\
 sed "/search full table/s|setup_all.html|setup.html|" |\
 sed "s/search full table/back to setup table/" |\
 sed "s/, sorted by setups//g" |\
 sed "s/The tables can be sorted/The table can be sorted/" |\
 sed "s/<itf>/<fitif>/" |\
 sed "s/NEWLINE//g" \
> ${RESULTS}_setup_all.html
	cat >> ${RESULTS}_setup_all.html <<EOT
</tbody></table>
$TOP
<a name="bottom"></a>
</body></html>
EOT

	sed -i -e "s/NEWLINE//g" ${RESULTS}_setup.html

# fill empty lines to improve display of anchors
	cat >> ${RESULTS}_setup.html <<EOT
</tbody></table>
$TOP
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<a name=bottom></a>
</body></html>
EOT

# replace placeholder by $FIRST_SETUP
	if [ -s ${RESULTS}_setup.html ]
	then
		sed -i -e "/FIRSTSETUP/s|FIRSTSETUP|$FIRST_SETUP|" ${RESULTS}_setup.html
		cp ${RESULTS}_setup.html ${CALREPORT_DIR}
	fi

	if [ -s ${RESULTS}_setup_all.html ]
	then
		cp ${RESULTS}_setup_all.html ${CALREPORT_DIR}
	fi
fi

# ======================================================================
# 4.4 replace PRO.CATG by RAW.TYPE, using $TMP_DIR/list_rawpro
# ======================================================================

if [ -s $CONSOL ]
then
	REPLACE="$DFO_MON_DIR/calChecker.replace"
	cp $CONSOL $REPLACE

	for PRO_CATG in `cat $TMP_DIR/list_rawpro | awk '{print $2}'`
	do
		RAW_TYPE=`grep "[[:space:]]${PRO_CATG}$" $TMP_DIR/list_rawpro | awk '{print $1}'`
		sed -i -e "s/\*${PRO_CATG}:/\*${RAW_TYPE}:/g" $REPLACE
	done

	rm -f $TMP_DIR/setups
	cat $CONSOL | awk '{print $3,$2}' | sort -u > $TMP_DIR/setups
fi

# ======================================================================
# 5. create main table 
# 5.1 sort by setups
# ======================================================================

if [ -s $CONSOL ]
then
	echo "- create main result page ..."
else
	case $CAL4CAL in
	 "NO"  ) echo "- no science files found." ;;
	 "YES" ) echo "- no calibration files found." ;;
	esac
fi

# loop over dates, to prepare for "action required"
# $OUT lists setups with outdated or missing files
# $OUT_LOST lists setups with lost files
OUT="$DFO_MON_DIR/calChecker.out"
OUT_LOST="$DFO_MON_DIR/calChecker.out_lost"
rm -f $OUT $OUT_LOST
touch $OUT $OUT_LOST

# LOST: list of rawtypes
UT_NOW=`date -u +%H`

rm -f $TMP_DIR/cc_list_LOST
grep "^RAWFILE_NIGHT" $DFO_CONFIG_DIR/config.calChecker | awk '{print $2}' > $TMP_DIR/cc_list_LOST

if [ -s $CONSOL ]
then
	for RT in `cat $TMP_DIR/setups | awk '{print $2}' | sort -u`
        do
		for S in `cat $TMP_DIR/setups | grep $RT | awk '{print $1}'`
		do
			STRING=""
			STRING_LOST=""
			for D in `cat $TMP_DIR/cc_list_dates4`
			do
				ENTRY=""
				ENTRY_LOST=""
				MISS1=`grep "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S[[:space:]]" | egrep ":NOK*|:MISS*" | awk '{print $4}' | head -1`
# Find difference between MISS and LOST; only if current night is over
				CHECK_LOST=""
				if [ "Q$MISS1" != "Q" ]
				then
# this is the condition for LOST: $D != $TODAY OR, if $TODAY, later than $UT_LIMIT
					if [ $D != $TODAY ] || [ $UT_NOW -ge $UT_LIMIT ] 
					then
						rm -f $TMP_DIR/cc_check_LOST
						grep "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S[[:space:]]" | egrep ":NOK*|:MISS*" | awk '{print $4}' | sed "s/\*\*\*/ /g" | sed "s/\*\*//" | sed "s/\*//" | tr " " "\012" | egrep "MISS|NOK" | sed "s/:.*//" | sort -u > $TMP_DIR/cc_check_LOST
						if [ -s $TMP_DIR/cc_list_LOST ]
						then
							CHECK_LOST=""
							for RT_LOST in `cat $TMP_DIR/cc_check_LOST`
							do
								CHECK_ENTRY=`grep "^${RT_LOST}$" $TMP_DIR/cc_list_LOST`
								if [ Q$CHECK_ENTRY != Q ]
								then
									CHECK_LOST=${RT_LOST}
									break
								fi
							done	
						fi

						if [ Q$CHECK_LOST != Q ]
						then
# NOK have age of NOK calibs, comes as +/-....
							sed -i -e "/^$D ${RT} $S /s|${CHECK_LOST}:MISS|${CHECK_LOST}:LOST|"       $REPLACE
							sed -i -e "/^$D ${RT} $S /s|${CHECK_LOST}:....:NOK|${CHECK_LOST}:LOST|"   $REPLACE
							sed -i -e "/^$D ${RT} $S /s|${CHECK_LOST}:-....:NOK|${CHECK_LOST}:LOST|"  $REPLACE
							sed -i -e "/^$D ${RT} $S /s|${CHECK_LOST}:-.....:NOK|${CHECK_LOST}:LOST|" $REPLACE
						fi
					fi
				fi
				LOST=`grep  "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S[[:space:]]" | grep ":LOST*" | awk '{print $4}' | tail -1`
				MISS=`grep  "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S[[:space:]]" | grep ":MISS*" | awk '{print $4}'`
				NOK=`grep   "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S[[:space:]]" | grep ":NOK*"  | awk '{print $4}'`
# LOST: kept separate from NOK/MISS
				if [ "Q$LOST" != "Q" ] 
				then
					ENTRY_LOST=`grep  "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S[[:space:]]" | grep ":LOST*" | awk '{print $4}' |\
 sed "s/\*/ /g" |\
 tr  " " "\012" |\
 grep ":LOST" |\
 sort -u |\
 sed "s/:LOST//" |\
 tr "\012" " "`
					STRING_LOST=`echo "${STRING_LOST}|$ENTRY_LOST"`
 				fi

# MISS, NOK: kept separate from LOST
				if [ "Q$MISS" != "Q" ]
				then
# Aggregation:
# MISS or NOK handled the same here (MISS/NOK or MISS/MISS: MISS; NOK or NOK/NOK: NOK)
					ENTRY=`echo $MISS |\
 sed "s/\*/ /g" |\
 tr  " " "\012" |\
 sed "/^$/d" |\
 egrep ":MISS|:NOK" |\
 sed "s/:MISS//"  |\
 sed "s/:NOK//"  |\
 tr "\012" " "`
				elif [ "Q$NOK" != "Q" ]
				then
					ENTRY=`echo $NOK |\
 sed "s/\*/ /g" |\
 tr  " " "\012" |\
 sed "/^$/d" |\
 grep ":NOK" |\
 sed "s/:NOK//"  |\
 tr "\012" " "`
				fi
	
				STRING=`echo "${STRING}|$ENTRY"`
# end of date loop
			done

			STRING1=`echo $STRING_LOST | sed "s/|//g"`
			if [ "Q$STRING1" != Q ]
			then
				echo "$RT $S:$STRING1" >> $OUT_LOST
			fi

			STRING1=`echo $STRING | sed "s/|//g"`
			if [ "Q$STRING1" != Q ]
			then
				echo "$RT $S:$STRING1" >> $OUT
			fi
		done
	done
fi

# ======================================================================
# 5.2 Technical (maintenance) calibrations 
# ======================================================================

if [ $FORCED_MODE = YES ]
then
	writeLog "8. Check long-term calibrations ..."
	scpLog
fi

techCals

# ======================================================================
# 5.3 loop over dates: HTML
# 5.3.1 header part
# ======================================================================

TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
LAST_DATE=`echo $TIMESTAMP | cut -c1-10`
LAST_HOUR=`echo $TIMESTAMP | cut -c12,13`
BOUNDARY=`echo 12 $DFO_OFFSET | awk '{print $1+$2}'`

if [ $LAST_HOUR -ge $BOUNDARY ]
then
	CURRENT_LAST_DATE=$LAST_DATE
else
	CURRENT_LAST_DATE=`qcdate $LAST_DATE -1`
fi

cat > $HTML <<EOT
<html>
<!-- created by calChecker v${TOOL_VERSION} -->
<head>
<META HTTP-EQUIV="Refresh" CONTENT="$AUTO_REFRESH">
EOT

includeStyles

case $CAL4CAL in
 "NO"  ) TITLE_MON="${DFO_INSTRUMENT} calChecker: calibration completeness monitor for science data" ;;
 "YES" ) TITLE_MON="${DFO_INSTRUMENT} calChecker: calibration completeness monitor for calibration data" ;;
esac

cat >> $HTML <<EOT

<title>${TITLE_MON}</title>
</head>

<body class=page>
<script language="JavaScript">
<!--
function openWin(URL) {
  aWindow=window.open(URL,"a","toolbar=yes,width=1200,height=700,scrollbars=yes,menubar=yes");
  }
function openWin2(URL) {
  aWindow=window.open(URL,"a","toolbar=yes,width=900,height=700,scrollbars=yes,menubar=no");
  }
//-->
</script>

EOT

cat $TMP_DIR/offlink >> $HTML

# include header
cat >> $HTML <<EOT

<!--begin include $HEADER -->		<!--QCWEB--> <!--LOCAL-->
<!--#include virtual="/observing/dfo/quality/ALL/common/$HEADER"-->	<!--QCWEB--> <!--LOCAL-->
<!--end include vnavbar -->		<!--QCWEB--> <!--LOCAL-->

EOT

# main table
cat >> $HTML <<EOT
<!--begin main table -->
<div align=right><i><font color=#999999 size=1>
[all links are internal]
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;
<i><a id="refresh_link">[page auto-refreshes after $AUTO_REFRESH sec]</a> &nbsp;
   [<a href="javascript:OffLink()" onClick="window.stop()" title="click to temporarily stop browser refresh (will resume on reload)"><font color=#999999>stop</font></a>
  | <a href="javascript:location.reload()" title="click to resume browser refresh"><font color=#999999>on</font></a>]
EOT

cat >> $HTML <<EOT
&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;[press Ctrl+Shift+R to enforce refresh of "ago" time information]</font></i></div>
<table CELLSPACING=1 CELLPADDING=1 BORDER=0 WIDTH=100%>
  <tr VALIGN=TOP>
    <td width=1% ROWSPAN=100 BGCOLOR=${TITLE_COLOR}><!--VNAVBAR-->
EOT

# ===================================================================================
# 5.3.2 include the vnavbar
# ===================================================================================

# local navbar
if [ ! -s $ORIG_MON_DIR/vnavbar_calChecker.html ]
then
        echo "*** WARNING: no $ORIG_MON_DIR/vnavbar_calChecker.html found. Create it with dfoMonitor, option \"navigation bar: refresh\". "
else
	cat $ORIG_MON_DIR/vnavbar_calChecker.html >> $HTML
fi

# exported version: edit and include general navbar
case $CAL4CAL in
 "NO"  ) TARG_DIR="CAL" ;;
 "YES" ) TARG_DIR="CAL/CAL4CAL" ;;
esac

if [ $SCP = YES ]
then

# we edit and cp the general navbar directly on ${DFO_WEB_SERVER}, keeping the virtual includes for the cal_scores, so they auto-update
# strictly speaking this is needed only for a new instrument since otherwise nothing changes
	ssh -o BatchMode=yes ${DFO_WEB_SERVER} "sed \"/calChecker_${DFO_INSTRUMENT}.html/s|nav-subitem|this-nav-subitem|g\" ${ALL_DIR}/common/$NAVBAR > ${WEB_DIR}/${TARG_DIR}/navbar_CAL_${DFO_INSTRUMENT}.html; chmod u+x ${WEB_DIR}/${TARG_DIR}/navbar_CAL_${DFO_INSTRUMENT}.html"

	cat >> $HTML <<EOT
	<!--begin include vnavbar -->			<!--QCWEB--> 
	<!--#include virtual="./navbar_CAL_${DFO_INSTRUMENT}.html"-->	<!--QCWEB--> 
	<!--end include vnavbar -->			<!--QCWEB--> 
EOT
fi

# ===================================================================================
# 5.3.3 include the tutorial part
# ===================================================================================

# health watcher (from ngasMonitor)
UPDATE=`date -u +%Y-%m-%d"T"%H:%M:%S`
jCheckHealth $UPDATE 2

case $CAL4CAL in
 "NO"  ) TITLE_MON="${DFO_INSTRUMENT} calChecker: calibration completeness monitor" ;;
 "YES" ) TITLE_MON="${DFO_INSTRUMENT} calChecker: calibration completeness monitor (CAL4CAL)" ;;
esac

case $CAL4CAL in
 "NO"  ) QCWEB_TARG_DIR="CALCHECK" ;;
 "YES" ) QCWEB_TARG_DIR="CALCHECK/CAL4CAL" ;;
esac

cat >> $HTML <<EOT
    </td>
    <td width=10 ROWSPAN=100 bgcolor=#FFFFFF><img src=${IMG_URL}/itemOff.gif width="10" border="0"></td>

    <td COLSPAN=3 bgcolor=#CCCCCC>

      <table width=100% cellpadding=0 cellspacing=1 border=0>
        <tr>
          <td bgcolor=#CCCCCC align=left valign=bottom nowrap> 

            <table cellpadding="0" cellspacing="0" border=0>
              <tr align="left" valign="bottom">
                <td>
                  <table style="border-collapse: collapse;" border="1" bordercolor="#ffffff" cellpadding="0" cellspacing="0">
                    <tr><td bgcolor="${REPORT_COLOR}" height="40" width="40" align="left" valign="bottom"><b><font color="#ffffff" size="3"><a title="overview of calibration completeness for last $N_VCAL_LIST days">CAL</a></font></b></td></tr>
                  </table>
                </td> 
                <td valign="bottom" bgcolor=#CCCCCC><b><font color=${REPORT_COLOR} size=+2>${TITLE_MON} </font></b></td>
              </tr>
            </table>

          </td>
        </tr>

        <tr bgcolor=#CCCCCC>
          <td align=left>

            <table cellpadding="1" cellspacing="1" border="0" width=100%>
              <tr bgcolor=$SM_COLOR>
                <td nowrap width=1% align=right><font size=2 color=#FFFFFF>Last update:</td>
                <td nowrap width=1%  align=left><font size=2 color=#FFFFFF>$TIMESTAMP (UT)</font></td>
                <td nowrap width=1%> 
                  <table border=0 cellpadding=1 cellspacing=1>
		    <tr>
EOT
cat $TMP_DIR/cal_checkhealth >> $HTML

cat >> $HTML <<EOT
		  <td><font color=#FFFFFF size=2><b>[<a href="javascript:openWin('${QC_URL}/CC_help_update.html');" title="info about outdated"><font color=#FFFFFF>?</a>]</b></font></td>
		    </tr>
		  </table>
                </td>

                <td nowrap align=left><font size=2 color=#FFFFFF>&nbsp; Paranal date*: <b>${CURRENT_LAST_DATE} &nbsp; &nbsp; &nbsp; &nbsp; <font color=$TITLE_COLOR font size=1><a title=`hostname`>`hostname`</a></font> [<a href="javascript:openWin('${QC_URL}/CC_help_update.html');" title="info about update"><font color=#FFFFFF>?</font></a>]</b></font></td>
                <td nowrap><font size=2 color=#FFFFFF>&nbsp; server: <b>
<!--#set var="SRV" value="\$SERVER_NAME" -->
<!--#echo var="SRV" -->
<!--#if expr="\"\$SRV\"=\"www.eso.org\"" -->
HQ
<!--#else -->
PL
<!--#endif -->
		</b></font></td> 
                <td nowrap><font size=2 color=#FFFFFF>&nbsp; machine: <b>`hostname`</b></font></td> <!--FILTER--><!--QCWEB-->
                <td nowrap><font size=2 color=#FFFFFF>&nbsp; account: <b>`whoami`</b></font></td> <!--FILTER--><!--QCWEB-->
                <td nowrap width=10>
            <table width=100% align=right>
              <tr bgcolor="$PRODUCT_COLOR">
                <td nowrap align=right> 
                   <a href="javascript:openWin('${QC_URL}/CC_help.html');" title="HELP about calChecker"><font size="2" color=#000000> HELP </font></a>
                </td>
                <td nowrap align=right>
                  <a href="javascript:openWin('${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/${QCWEB_TARG_DIR}/${RAWTYPES}');" title="find association rules and raw data types [internal link]"><font size="2" color=#000000> ASSOC-RULES </font></a>
                </td>
                <td nowrap align=right>
                  <a href="javascript:openWin('${CALREPORT_DIR}/calResults_${DFO_INSTRUMENT}_setup.html');" title="result details per setup [internal link]"><font size="2" color=#000000> DETAILS </font></a>
                </td>
              </tr>
            </table>
  
                </td>
              </tr>

              <tr bgcolor=$SM_COLOR>
                <td align=right nowrap><font size=2 color=#FFFFFF>Last header: ${DFO_FILE_NAME}.</font></td>
                <td align=left  nowrap><font size=2 color=#FFFFFF>available:</font></td> <!-- LAST_HDR -->
                <td align=left bgcolor=$GRY>
EOT

cat >> $HTML <<EOT

<!-- transferWatcher and ngasWatcher -->			<!--QCWEB--> <!--LOCAL-->
                <table cellspacing=1 border=0 width=100%>	<!--QCWEB--> <!--LOCAL-->
                  <tr>						<!--QCWEB--> <!--LOCAL-->
<!--#include virtual="${REL_URL}/transferWatcher.html" -->	<!--QCWEB--> <!--LOCAL-->
<td><font size=1>&nbsp; &nbsp; </td>				<!--QCWEB--> <!--LOCAL-->
<!--#include virtual="${REL_URL}/ngasWatcher.html" -->		<!--QCWEB--> <!--LOCAL-->

     <td align=right>&nbsp; &nbsp;<font color=#666666 size=2><b>[<a href="javascript:openWin('${QC_URL}/CC_help_update.html');" title="info about watcher flags">?</a>]</b></font></td>	<!--QCWEB--> <!--LOCAL-->
                  </tr> 					<!--QCWEB--> <!--LOCAL-->
                </table>					<!--QCWEB--> <!--LOCAL-->

                </td>
EOT

cat >> $HTML <<EOT
		<td bgcolor=#CCCCCC align=left COLSPAN=4>
                  <font size=2>&nbsp; *</font><font size=1><i>Date on this monitor changes at ${BOUNDARY}:00 UT.
                  Refresh frequency: 1/2hr day and night</i></font>
                </td>
              </tr>

            </table>

          </td>
        </tr>
    </table>

    </td></tr>
EOT

# POSTIT section
cat >> $HTML <<EOT

<!-- POSTIT section -->
  <tr>
    <td COLSPAN=3 ALIGN=LEFT VALIGN=TOP bgcolor=#eee> 

    <table width=100% border=0 align=left cellpadding=1 cellspacing=1>
      <tr >
        <td height="5" width=5% nowrap bgcolor=#eee valign=top align=right>
          <font color=$TITLE_COLOR size=2><b><a title="General news">General news:</a></b></font>
        </td>
        <td bgcolor=$LYLW valign=top><font color=$TITLE_COLOR size=2><b>

<!--begin include CC_POSTIT -->
EOT

# wget CC_POSTIT (general news): the master is on ${PUB_QC_WEB_SERVER_URL}; force it to one-line for the local version,
# for later editing before scp
rm -f CC_POSTIT*
ssh -o BatchMode=yes ${DFO_WEB_SERVER} "if [ ! -s ${ALL_DIR}/CC_POSTIT ]; then touch ${ALL_DIR}/CC_POSTIT; fi" 1>/dev/null
eval "wget -q ${PUB_QC_WEB_SERVER_URL}/ALL/CC_POSTIT -T 5 -t 1"
if [ ! -s CC_POSTIT ]
then
	echo " " > CC_POSTIT
fi

cat CC_POSTIT | tr "\012" " " | sed "s/^.*/& <!--CC_POSTIT-->/" >> $HTML
rm CC_POSTIT

cat >> $HTML <<EOT
<!--end include CC_POSTIT -->

          </b></font></td>
<!-- result table for long-term calibs-->
EOT

if [ -s $TMP_DIR/cc_tech.html1 ]
then
	cat $TMP_DIR/cc_tech.html1 >> $HTML
fi

cat >> $HTML <<EOT

        </td>
      </tr>

      <tr>
       <td width=2% nowrap bgcolor=#eee valign=top align=right>
         <font color=$TITLE_COLOR size=2><b><a title="News for $DFO_INSTRUMENT">$DFO_INSTRUMENT news:</a></b></font>
       </td>
       <td bgcolor=$LYLW width=58% valign=top><font color=$TITLE_COLOR size=2>
EOT

# instrument POSTIT file: always edited to have one line only, to make replacement by edit_CAL_POSTIT.esh a lot more easier
cat >> $HTML <<EOT

<!--begin include $DFO_INSTRUMENT CAL_POSTIT -->
EOT

# CAL_POSTIT (INS news): the master is on ${PUB_QC_WEB_SERVER_URL}; wget, force it to one-line for the local version,
# for later editing before scp

rm -f CAL_POSTIT*
ssh -o BatchMode=yes ${DFO_WEB_SERVER} "if [ ! -s ${WEB_DIR}/CAL/CAL_POSTIT ]; then touch ${WEB_DIR}/CAL/CAL_POSTIT; fi" 1>/dev/null
eval "wget -q ${PUB_QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/reports/CAL/CAL_POSTIT -T 5 -t 1"
if [ ! -s CAL_POSTIT ]
then
	echo " " > CAL_POSTIT
fi

cat CAL_POSTIT | tr "\012" " " | sed "s/^.*/& <!--CAL_POSTIT-->/" >> $HTML
rm CAL_POSTIT

echo "
<!--end include $DFO_INSTRUMENT CAL_POSTIT -->
" >> $HTML

# continue
cat >> $HTML <<EOT
  </td></tr></table>

  </td></tr>

<!-- refresh, help etc. --> 				<!--FILTER--> <!--QCWEB-->
  <tr bgcolor=$TITLE_COLOR>    				<!--FILTER--> <!--QCWEB-->
    <td ALIGN=LEFT VALIGN=BOTTOM nowrap><font color=#FFFFFF>   <!--FILTER--> <!--QCWEB-->
      <img src=$IMG_URL/back.gif width=20 height=20 border=0>&nbsp; <!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="refresh this page" href=$DFO_GUI_DIR/refresh_calChecker.esh> <!--FILTER--> <!--QCWEB-->
      <img src=$DFOS_URL/images/refresh.png border=0></a>   <!--FILTER--> <!--QCWEB-->
      <img src=${IMG_URL}/greenball.gif width=10 height=10 border=0><a class=dfos_item title="view today's calChecker log" href=$DFO_GUI_DIR/view_calChecker_log.esh>[calChecker.log]</a>  <!--FILTER--> <!--QCWEB--> 
      <img src=${IMG_URL}/greenball.gif width=10 height=10 border=0><a class=dfos_item><font size=2>[edit:]</a>		<!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="edit CAL_POSTIT (changes can be updated immediately)" href=$DFO_GUI_DIR/$EDIT_POSTIT>NEWS</a> &middot	<!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="edit CAL_ANALYZED file" href=$DFO_GUI_DIR/$EDIT_ANALYSIS>CAL_ANALYSIS</a> &middot 	<!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="edit BADQUAL file" href=$DFO_GUI_DIR/$EDIT_BADQUAL>BADQUAL</a> &nbsp;			<!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="mail to PSO" href=$DFO_GUI_DIR/launch_calmail.esh><img src=${IMG_URL}/greenball.gif width=10 height=10 border=0>[send mail to PSO]</a> 	<!--FILTER--> <!--QCWEB-->
    </td>   <!--FILTER--> <!--QCWEB-->

    <td align=left valign=bottom nowrap><font color=#FFFFFF> <!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="calChecker OCA configuration files" href=$DFO_CONFIG_DIR/>OCA config</a> | <!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="ANALYSIS file" href=$DFO_MON_DIR/CAL_ANALYZED target=_blank>ANALYSIS file</a> | <!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="open ANALYSIS template file" href=$DFOS_URL/config/analysis.html target=_blank>ANALYSIS template</a> | <!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="open BADQUAL file" href=$DFO_MON_DIR/CAL_BADQUAL target=_blank>BADQUAL file</a> <!--FILTER--> <!--QCWEB-->
    </td>   <!--FILTER--> <!--QCWEB-->

    <td align=right valign=bottom nowrap><font color=#FFFFFF> <!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="help for this page"          href=$DFO_DOC_DIR/calChecker.h>tool help</a> |  <!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="edit config file"            href=$DFO_GUI_DIR/edit_calChecker.esh>[config]</a> | <!--FILTER--> <!--QCWEB-->
      <a class=dfos_item title="documentation for this tool" href=$DFOS_URL>DFOS</a>  <!--FILTER--> <!--QCWEB-->
    </td>       <!--FILTER--> <!--QCWEB-->
  </tr>	 	<!--FILTER--> <!--QCWEB-->
EOT

case $CAL4CAL in
 "NO"  ) HIST_LINK="CALCHECK"; ANALYSIS_LINK="CAL" ;;
 "YES" ) HIST_LINK="CALCHECK/CAL4CAL"; ANALYSIS_LINK="CAL/CAL4CAL" ;;
esac

# note: the HC_REPORT link below is later edited, with sed on "Health Check Monitor", so don't modify
cat >> $HTML <<EOT
<!-- links -->
  <tr bgcolor=#FFFFFF>
    <td ALIGN=LEFT VALIGN=BOTTOM COLSPAN=99 nowrap bgcolor=#eee>
       <table border=0 cellspacing=1 cellpadding=0 bgcolor=#fff>
         <tr>
          <td valign="bottom" nowrap bgcolor=#eee>
     <br>
     <a href="${REL_URL}/HEALTH/${HC_REPORT}" title="go to Health Check Monitor for $DFO_INSTRUMENT: overview of instrument health" class="nounder"><input type="button" value="HC" class="hclabel"></a> 
     &nbsp; <a href="javascript:openWin2('${PUB_QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/reports/CAL/php/calForcedRefresh.php?instrSelect=${DFO_INSTRUMENT}');"  title="forced refresh of calChecker" class="nounder"><input type="button" value="refresh" class="esocal2"></a> 
     &nbsp; <a href="javascript:openWin2('${PUB_QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/reports/${ANALYSIS_LINK}/php/calAnalysis.php');" title="analyze NOK, MISS and LOST issues" class="nounder"><input type="button" value="analyze ISSUES" class="esocal2"></a> 
     &nbsp; <a href="javascript:openWin2('${PUB_QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/reports/CAL/php/calBadQual.php');"  title="mark BAD QUALITY calibrations" class="nounder"><input type="button" value="mark BAD QUALITY" class="esocal2"></a> &nbsp; </td>
          <td valign="bottom" nowrap bgcolor=#eee><br><font size=1>
     <img src=${IMG_URL}/right.gif width="10" height="11"><a href="javascript:openWin('${QC_URL}/CC_help.html');" title="help about calChecker" style="background-color:#66ccff; color:#333333">HELP</a> |
     <a href="javascript:openWin('${QC_URL}/calQuestions.html');" title="questions and answers about calChecker" style="background-color:#66ccff; color:#333333">Q&A</a> |
     <a href="javascript:openWin('${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/${QCWEB_TARG_DIR}/${RAWTYPES}');" title="association rules and raw data types [internal link]" style="background-color:#66ccff; color:#333333">ASSOC-RULES</a> &nbsp;
      <img src=${IMG_URL}/right.gif width="10" height="11"><a href=${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/${HIST_LINK} title="find calChecker result pages for older dates [internal link]" target=_blank><font color=$SM_COLOR>history...</font></a>
EOT

if [ Q$QC_ADDRESS != Q ]
then
	cat >> $HTML <<EOT
     | <a href="mailto:${QC_ADDRESS}?subject=${DFO_INSTRUMENT} calChecker" title="mail about this calChecker result page"><font color=$SM_COLOR>contact</font></a>
EOT
else
	echo "***WARNING: \$QC_ADDRESS should be configured in your .dfosrc. Pls add!"
fi

cat >> $HTML <<EOT
     &nbsp; <img src=${IMG_URL}/right.gif width="10" height="11"> monitors:
     <a href=${TRANSFER_URL} title="status of ${DFO_INSTRUMENT} fits file transfer [internal link, powered by SDD]" target=_blank><font color=$SM_COLOR>DataTransferMonitor</font></a> 
      | <a href=${BANDWIDTH_URL} title="status of transfer link bandwidth [internal link, powered by DHA]" target=_blank> <font color=$SM_COLOR>BandWidth</font></a> 
              </td>
            </tr>
          </table>

    </td>
  </tr>	
<!-- end of header table -->

<!-- cal4cal navigation -->
<tr bgcolor=#FFFFFF>
  <td ALIGN=LEFT VALIGN=BOTTOM COLSPAN=99 nowrap>
    <table width="100%" border="0" cellpadding="0" cellspacing="0">
EOT

echo "" > $ORIG_TMP_DIR/CALCHECK/tell_warn.html
if [ $CHECK_TELLTRACK = YES ]
then
	if [ -s $ORIG_TMP_DIR/CALCHECK/tt_issue_$TODAY ]
	then
		MARK_ISSUE="<font size=\"2\" color=red><b><a title=\"tellTracker issue with last night ($TODAY), not recoverable\">!</b></a></font>&nbsp;"
		echo "$MARK_ISSUE" > $ORIG_TMP_DIR/CALCHECK/tell_warn.html
	else
		MARK_ISSUE=""
	fi

        TELLTRACK_TAG="<td width=3% align=center bgcolor=#FFD5BF nowrap>${MARK_ISSUE}<b><font size=1>&nbsp;<a href=\"${QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/CALCHECK/TELLTRACK/tellTracker_${CURRENT_LAST_DATE}.html\" title=\"tellTracker: telluric standards for science [internal link]\"><font color=#333333 size=1>tellTracker</font></a>&nbsp;</b></td>"
else
        TELLTRACK_TAG="<td width=3% bgcolor=#FFFFFF><font size="1">&nbsp;</font></td>"
fi

# provide always the tell_warn.html file for the navbar, even though it is empty in all cases unless tellTracker is enabled and has an issue
scp -o BatchMode=yes -p $ORIG_TMP_DIR/CALCHECK/tell_warn.html ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}/ 1>/dev/null

AVAIL_TEXT="<i>&nbsp; &nbsp; <font size=1 color=#666666>Product availability depends on the data transfer to Garching and the archive access there (check the \"transfer\" and \"ngas\" flags above).  </font></i>"

if [ $CHECK_C4C = YES ]
then
	if [ $CAL4CAL = NO ]
	then
		cat >> $HTML <<EOT
      <tr bgcolor="#FFFFFF" height="3">
        <td align=center bgcolor="#FF9966"></td>
        <td></td>
        <td align=center></td>
        <td></td>
        <td></td>
        <td></td>
        <td></td>
      </tr>
      <tr>
        <td width="3%" align=center bgcolor="#FF9966" nowrap><b><font size="1">&nbsp;MARK_SCIENCE<a href="calChecker.html" title="calibration completeness for science"><font color="#333333">science</font></a>&nbsp;</font></b></td> <!--SCIENCE_NAV-->
        <td width="1"><font size=1 color=#FFFFFF>.</font></td>
        <td width="3%" align=center bgcolor="#FFD5BF" nowrap><b><font size="1">&nbsp;MARK_CAL4CAL<a href="CAL4CAL/calChecker.html" title="calibration completeness for calibrations"><font color="#333333" size="1">cal4cal</font></a>&nbsp;</b></td> <!--CAL4CAL_NAV-->
        <td width="1"><font size=1 color=#FFFFFF>.</font></td>
	$TELLTRACK_TAG
        <td width="3%" align=left bgcolor="#FFFFFF" nowrap><font size="1" color=#666666>&nbsp;[<a href="javascript:openWin('${QC_URL}/CC_help_long.html#cal4cal');" title="info about CAL4CAL">?</a>]</font></td>
        <td width="88%" align=right>${AVAIL_TEXT}</td>
      </tr>
      <tr bgcolor="#FF9966" height="3">
        <td colspan="7"></td>
      </tr>
EOT
# CAL4CAL = YES
	else
		cat >> $HTML <<EOT
      <tr bgcolor="#FFFFFF" height="3">
        <td align=center></td>
        <td></td>
        <td align=center bgcolor="#FF9966"></td>
        <td></td>
        <td></td>
        <td></td>
        <td></td>
      </tr>
      <tr>
        <td width="3%" align=center bgcolor="#FFD5B5" nowrap><b><font size="1">&nbsp;MARK_SCIENCE<a href="../calChecker.html" title="calibration completeness for science"><font color="#333333">science</font></a>&nbsp;</font></b></td> <!--SCIENCE_NAV-->
        <td width="1"><font size=1 color=#FFFFFF>.</font></td>
        <td width="3%" align=center bgcolor="#FF9966" nowrap><b><font size="1">&nbsp;MARK_CAL4CAL<a href="calChecker.html" title="calibration completeness for calibrations (only if configured)"><font color="#333333" size="1">cal4cal</font></a>&nbsp;</b></td> <!--CAL4CAL_NAV-->
        <td width="1"><font size=1 color=#FFFFFF>.</font></td>
	$TELLTRACK_TAG
        <td width="3%" align=left bgcolor="#FFFFFF" nowrap><font size="1" color=#666666>&nbsp;[<a href="javascript:openWin('${QC_URL}/CC_help_long.html#cal4cal');" title="info about CAL4CAL">?</a>]</font></td> 
        <td width="88%" align=right>${AVAIL_TEXT}</td>
      </tr>
      <tr bgcolor="#FF9966" height="3">
        <td colspan="7"></td>
      </tr>
EOT
	fi
# check_c4C = NO
else
	cat >> $HTML <<EOT
      <tr bgcolor="#FFFFFF" height="3">
        <td align=center bgcolor="#FF9966"></td>
        <td align=center></td>
        <td></td>
        <td></td>
        <td></td>
      </tr>
      <tr>
        <td width="3%" align=center bgcolor="#FF9966" nowrap><b><font size="1">&nbsp;MARK_SCIENCE<a href="calChecker.html" title="calibration completeness for science"><font color="#333333">science</font></a>&nbsp;</font></b></td> <!--SCIENCE_NAV-->
        <td width="3%" align=center bgcolor="#eeeeee" nowrap><font size="1">&nbsp;<font color="#333333"><a title="${DFO_INSTRUMENT} has no checks for calibration completeness for calibrations">no cal4cal</a></font>&nbsp;</font></td>
	$TELLTRACK_TAG
        <td width="3%" align=left bgcolor="#FFFFFF" nowrap><font size="1" color=#666666>&nbsp;[<a href="javascript:openWin('${QC_URL}/CC_help_long.html#cal4cal');" title="info about CAL4CAL">?</a>]</font></td> <!--CAL4CAL_NAV-->
        <td width="88%" align=right>${AVAIL_TEXT}</td>
      </tr>
      <tr bgcolor="#FF9966" height="3">
        <td colspan="5"></td>
      </tr>
EOT
fi

cat >> $HTML <<EOT
    </table>
   </td>
  </tr>	

<!-- begin of result table -->
  <tr>
    <td bgcolor=#eeeeee COLSPAN=3> 
EOT

case $CAL4CAL in
 "YES" ) RESULT_BGCOLOR=#DDDDDD ;;
 "NO"  ) RESULT_BGCOLOR=$GRY ;;
esac
OLD_RESULT_BGCOLOR=#EEEEEE

TR="<tr bgcolor=$RESULT_BGCOLOR valign=top>"
TH="<th valign=top><font size=2>"

# ===================================================================================
# 5.4 result table
# 5.4.1 top part
# ===================================================================================

beocal

case $DAYCAL in
 0 ) EOCAL="[daytime calibrations not yet started]" ;
     BGCOLOR=$GRY ;
     DAYCAL_STATUS="<i>&nbsp; pending</i>" ;;
 1 ) EOCAL="[daytime calibrations ongoing, started ${FIRST_BOCAL}UT]"  ;
     #DAYCAL_STATUS="<marquee direction=left scrollamount=2>$EOCAL</marquee>" ;
     DAYCAL_STATUS="<i>&nbsp; $EOCAL</i>" ;
     BGCOLOR=$LGRN ;;
 2 ) EOCAL="[daytime calibrations finished ($FIRST_BOCAL - $LAST_EOCAL UT)]" ;
     DAYCAL_STATUS="<i>finished ${LAST_EOCAL}UT</i>" ;
     BGCOLOR=$GRN ;;
esac

N_DATES=`cat $TMP_DIR/cc_list_open_issues 2>/dev/null $TMP_DIR/cc_list_dates2 | wc -l | awk '{print $1}'`
N_DATES1=`echo $N_DATES | awk '{print $1+2}'`

cat >> $HTML <<EOT

<table BORDER=0 cellpadding="0" cellspacing="0"><tr><td bgcolor=#FFFFFF> <!--begin of white border table-->
<table BORDER=0 cellpadding="1" cellspacing="1">
  <tr bgcolor=white>
    <td COLSPAN=$N_DATES1 align=right><font size=1>&nbsp;</font></td>
    <td BGCOLOR=${TITLE_COLOR} nowrap width=100><a title="status of daytime calibration queue (calOBBuilder)"><font size=1 color=white>daytime calibs:</font></a></td>
    <td width=2 bgcolor=${TITLE_COLOR}> </td>
EOT

cat >> $HTML <<EOT
    <td COLSPAN=3 bgcolor=${BGCOLOR} nowrap><font size=1 color=${TITLE_COLOR}><a title="$EOCAL">$DAYCAL_STATUS</a></font></td>
  </tr>
EOT

cat >> $HTML <<EOT
  $TR
    <td valign=top align=right COLSPAN=2>
      <font size=2><b><a title="detailed information about dates">DATE*:&nbsp;&nbsp;</a> <font color=#666666>[<a href="javascript:openWin('${QC_URL}/CC_help_results.html');" title="information about dates">?</a>]</font></b></font><br>
      <font color=#666666 size=1><i>[color if science data acquired]</i></font>
    </td>
    <td width=2 bgcolor=${TITLE_COLOR}> </td>
EOT

# products table: included in calChecker, plus created separately to be exported to HC monitor
# the exported one has no dates with old issues
# needed only for CAL4CAL=NO, but created in either case to keep the code cleaner
PRODUCTS_TABLE="$DFO_MON_DIR/products.html"

cat > $PRODUCTS_TABLE <<EOT
<table BORDER=0 cellpadding="1" cellspacing="1" width=100%>
  <tr>
  <td valign=top align=right bgcolor=#CCCCCC><font size=2 color=#666666><b><a title="detailed information about dates">DATE*:</a>&nbsp;[<font color=#333333><a href="javascript:openWin('${HCHELP_URL}#navigation');" title="info about navigation">?</a></font>]</b></font></td>
EOT

# ===================================================================================
# 5.4.2 DATE loop
# ===================================================================================

rm -f $TMP_DIR/cc_list_scinum
for D in `cat $TMP_DIR/cc_list_open_issues 2>/dev/null $TMP_DIR/cc_list_dates2`
do
# check if old issue--> different font sizes and colors
	CHECK_OLD=`grep $D $TMP_DIR/cc_list_open_issues 2>/dev/null`
	if [ Q$CHECK_OLD != Q ]
	then
		CHECK_OLD=YES
	else
		CHECK_OLD=NO
	fi

# cut date string if more than 15 entries
	if [ $N_VCAL_LIST -ge 15 ]
	then
		D2=`echo $D | awk '{print substr($1,1,5),substr($1,6,10)}' | sed "s/ /<br>/"`
	else
		D2=$D
	fi

	SCI_YN=N
	SM_VM=""
	SCI_NUM=0

	REPORT="<font color=#666666 size=1><a title=\"no data taken\"><i>no data</i></a></font>"
	REPORT2="$REPORT"
	NR=""

	if [ -s $DFO_HDR_DIR/$D ]
	then
		CHECK_EXIST=`ls $DFO_HDR_DIR/$D | grep hdr | head -1`
		if [ "Q$CHECK_EXIST" != "Q" ]
		then
			cd $DFO_HDR_DIR/$D

# grep much faster than dfits: better filter for SCIENCE first
			CHECK_SCI=`grep SCIENCE *hdr | head -1 | awk '{print $1}'`	
			if [ Q$CHECK_SCI != Q ]
			then
				getSCI_NUM 
			fi

# ===================================================================================
# 5.4.3 Report links
# ===================================================================================
# SM_VM flag
			if [ $SCI_YN = Y ]
			then
				SM_VM=`getFlag`
			fi

# raw data report
			if [ -s $DFO_LST_DIR/REPORT/list_${DFO_INSTRUMENT}_${D}_data.html ]
			then
      				REPORT="<a href=$REPORT_DIR/list_${DFO_INSTRUMENT}_${D}_data.html target=_blank title=\"raw data report [internal link]\"><font color=${REPORT_COLOR} size=1>report</font></a>" 
      				REPORT2="<a href=${REPORT_DIR2}/list_${DFO_INSTRUMENT}_${D}_data.html target=_blank title=\"raw data report [internal link]\"><font color=${REPORT_COLOR} size=1>report</font></a>" 
			else
				REPORT="<font color=#666666 size=1><a title=\"no data taken\"><i>no data</i></a></font>"
				REPORT2=$REPORT
			fi
		fi
	fi

# for later reference to detect header problem: no CAL result but $SCI_NUM > 0 --> suspicious, no entry in cc_list_dates_OKK, scan later
	if [ $CHECK_OLD = NO ]
	then
		echo "$D $SCI_NUM" >> $TMP_DIR/cc_list_scinum
	fi

# nightlog link
	NR="| <a href=\"${NLT_URL}?date=${D}&instrument=$PANL_INSTRUMENT\" title=\"instrument night report from NLT [internal link, password required]\" target=\"_blank\"><font size=1 color=${REPORT_COLOR}>NLT</font></a>"

	if [ $SCI_YN = N ]
	then
		FTCOLOR="#666666"
		BGCOLOR="$RESULT_BGCOLOR"
		SCI_NUM=""
	else
		FTCOLOR="#000000"
		BGCOLOR=$SCI_COLOR
		SCI_NUM="<i><font size=1 color=#666666><a title=\"number of science files\">${SCI_NUM}</a></font></i>"
	fi

	case $CHECK_OLD in
	 YES ) FTSIZE=1 ; ITALIC="<i>" ; OLD_ISSUE="old issue <br>"; BGCOLOR=$OLD_SCI_COLOR ;;
	 NO  ) FTSIZE=2 ; ITALIC="" ; OLD_ISSUE="" ;;
	esac

	cat >> $HTML <<EOT
    <td align=center valign=top BGCOLOR=$BGCOLOR nowrap>$ITALIC<font size=$FTSIZE color=$FTCOLOR>${OLD_ISSUE}<b>$D2<br>
      <a title="SM: service mode; VM: visitor mode">$SM_VM</a></b>${SCI_NUM}<br>
      $REPORT 
$NR
   </font> </td>
EOT
	if [ $CHECK_OLD = NO ]
	then
		cat >> $PRODUCTS_TABLE <<EOT
    <td align=center valign=top BGCOLOR=$BGCOLOR nowrap><font size=1 color=$FTCOLOR>$D2<br>
      $REPORT2
$NR
    </font></td>
EOT
	fi
done

cat >> $PRODUCTS_TABLE <<EOT
</tr>

EOT

# ===================================================================================
# 5.4.4 Last 3 cells (LOST, actions, setups)
# ===================================================================================

case $CAL4CAL in
  "NO"  ) LOST="<font size=2><a title=\"Lost calibration type, validity one night only, cannot be taken in another night\"><b>LOST?</b></a></font><br> <font color=#666666 size=1><i>[may require OB<br>grade review]</i></font>" ;;
  "YES" ) LOST="<font size=2 color=#666666>LOST?</font><br><font size=1><i>[not applicable <br>for 'cal4cal']</i></font>" ;;
esac

cat >> $HTML <<EOT
    <td width=2 bgcolor=${TITLE_COLOR}> </td>
    <td valign=top nowrap>$LOST</td>
    <td valign=top nowrap><font size=2><a title="Take these calibration data types to resolve the calChecker issue"><b>Calibration action?</b></a>
      &nbsp; &nbsp; <font color=#666666><b>[<a href="javascript:openWin('${QC_URL}/CC_help_results.html');" title="information about results">?</a>]</b></font><br>
      <font color=#666666 size=1><i>[take these data types ...</i></font></font></td>
    <td valign=top align=left nowrap><font size=2><a title="Setup of required calibrations"><b>Setup:</b></a><br>
      <font color=#666666 size=1><i>... for these setups]</i></font></font>
    </td>
  </tr>
EOT

# ===================================================================================
# 5.4.5 rows with raw disp and product links
# ===================================================================================
HTML1="$DFO_MON_DIR/calChecker_${DFO_INSTRUMENT}1.html"

cat > $HTML1 <<EOT
  $TR
    <td valign=top align=right COLSPAN=2 nowrap>
     <table border="0" cellspacing="1" cellpadding="1">
        <tr>
          <td bgcolor="#FFFFFF">
            <table border="0" cellpadding="0" cellspacing="0">
              <tr>
                <td width="25" height="19" align="left" valign="bottom" bgcolor="#006A9D"><b><font color="#FFFFFF" size="2"><a title="overview of all processed CAL data per date, with scores and quality information; click on 'products' links for details">P...</a></font></b></td>
              </tr>
            </table>
          </td>

          <td valign="bottom" nowrap>
            <font size=2><b><a title="overview of all processed CAL data per date, with scores and quality information; click on 'products' links for details">CAL product quality:</a></b></font>
            &nbsp; &nbsp; <font size=2 color=#666666><b>[<a href="javascript:openWin('${QC_URL}/CC_help_results.html');" title="information about product quality">?</a>]</b></font>
          </td>
        </tr>
      </table>

</td>
<td width=2 bgcolor=${TITLE_COLOR}> </td>
EOT

# ===================================================================================
# 5.4.6 next line in PRODUCTS_TABLE
# 	used twice (products, raw)
# ===================================================================================
PRODUCTS_TABLE1=$DFO_MON_DIR/products1.html

cat > $PRODUCTS_TABLE1 <<EOT
<tr>
   <td valign=top align=right bgcolor=#CCCCCC nowrap>

     <table border="0" cellspacing="1" cellpadding="1">
        <tr>
          <td bgcolor="#FFFFFF">
            <table border="0" cellpadding="0" cellspacing="0">
              <tr>
                <td width="25" height="19" align="left" valign="bottom" bgcolor="#006A9D"><b><font color="#FFFFFF" size="2"><a title="overview of all processed CAL data per day, with scores and quality information; click on 'products' links for details">P...</a></font></b></td>
              </tr>
            </table>
          </td>

          <td valign="bottom" nowrap>
            <font size=2 color=#666666><b><a title="overview of all processed CAL data per day, with scores and quality information; click on 'products' links for details">CAL product quality:</a></b></font>
          </td>
        </tr>
      </table>

    </td>
EOT

# ===================================================================================
# 5.4.7 Products links for each date with data and ABs
# ===================================================================================

for D in `cat $TMP_DIR/cc_list_open_issues 2>/dev/null $TMP_DIR/cc_list_dates2`
do
	CHECK_OLD=`grep $D $TMP_DIR/cc_list_open_issues 2>/dev/null`
	if [ Q$CHECK_OLD != Q ]
	then
		CHECK_OLD=YES
	else
		CHECK_OLD=NO
	fi

# default: no CAL products, grey
	AB_LINK="<a title=\"no data or no CAL products available\"><font size=1 color=#666666><i>no CAL products</i></font></a>"
	AB_LINK1=$AB_LINK
	BGCOLOR=$RESULT_BGCOLOR
	SM_VM=""

# headers found: SCIENCE? If yes, SM and/or VM?
	SCI_NUM=`grep $D $TMP_DIR/cc_list_scinum | awk '{print $2}'`

# product link only if products exist
	CHECK_EXIST=""
	if [ -s $DFO_HDR_DIR/$D ]
        then
        	CHECK_EXIST=`ls $DFO_HDR_DIR/$D | grep hdr | head -1`
        fi

# old issues not listed in cc_list_scinum --> check for those
	if [ $CHECK_OLD = YES ]
	then
		getSCI_NUM	
	fi

	SCI_YN=N
	if [ $SCI_NUM -gt 0 ]
	then
		SCI_YN=Y
		SM_VM=`getFlag`
	fi

# check if ABs exist
	CHECK_NOAB=""
	if [ -s $DFO_LOG_DIR/$D/status_${D}.html ]
	then
		CHECK_NOAB=`grep "No ABs found" $DFO_LOG_DIR/$D/status_${D}.html`
	elif [ -s $ORIG_MON_DIR/status_${D}.html ]
	then
		CHECK_NOAB=`grep "No ABs found" $ORIG_MON_DIR/status_${D}.html`			
	fi

# data and ABs exist: check further if certified products exist
	CHECK_SCI=`grep SCIENCE *hdr | head -1 | awk '{print $1}' 2>/dev/null`
	if [ "Q$CHECK_EXIST" != "Q" ] && [ "Q$CHECK_NOAB" = Q ]
	then
		FT_SIZE=2
		if [ "Q$SM_VM" = "Q" ]
		then
			BGCOLOR=$RESULT_BGCOLOR
		else
			BGCOLOR="${SCI_COLOR}"
			if [ $CHECK_OLD = YES ]
			then
				BGCOLOR="${OLD_SCI_COLOR}"
				FT_SIZE=1
			fi
		fi

# mark if already certified by QCG
		CERTIF_YN=`grep " ${D} " $ORIG_MON_DIR/DFO_STATUS | grep "^cal_Certif"`
		if [ "Q$CERTIF_YN" != Q ]
		then
			CERTIF_TAG="<img src=${IMG_URL}/${OK_GIF} title=\"CALIB products certified by QCG\" width=10 height=11 border=0> "
			CERTIF_TAG2="<b>products</b>"
			CERTIF_TAG3="products certified by QC"
		else
			if [ -s $ORIG_MON_DIR/status_${D}.html ]
			then
				CHECK_UNSUPPORTED=`grep "<blink>PSO:" $ORIG_MON_DIR/status_${D}.html | head -1 |  awk '{print $1}'`
			elif [ -s $DFO_LOG_DIR/$D/status_${D}.html ]
			then
				CHECK_UNSUPPORTED=`grep "<blink>PSO:" $DFO_LOG_DIR/$D/status_${D}.html | head -1 |  awk '{print $1}'`
			fi	
			if [ "Q$CHECK_UNSUPPORTED" != Q ]
			then
				CERTIF_TAG="<i><b><a title=\"CALIB ABs in unsupported modes - pls. check the raw frames!\"> ! </a></b></i>"
			else
				CERTIF_TAG=""
			fi
			CERTIF_TAG2="<i>products</i>"
			CERTIF_TAG3="products not yet certified"
		fi
# for calChecker page
		AB_LINK="${CERTIF_TAG}<a href=${AB_DIR}/$D/status_${D}.html title=\"all processed CAL data with scores and quality information; ${CERTIF_TAG3} [internal link]\" target=_blank><font size=$FT_SIZE color=${SM_COLOR}>${CERTIF_TAG2}</font></a>"
# for included product links on HC monitor
		AB_LINK1="<a href=${AB_DIR}/$D/status_${D}.html title=\"all processed CAL data with scores and quality information [internal link]\" target=_top><font size=1 color=${SM_COLOR}>${CERTIF_TAG2}</font></a>"
	fi

# fill AB_LINK
	cat >> $HTML1 <<EOT
    <td align=center valign=middle BGCOLOR=$BGCOLOR nowrap>${AB_LINK} </td>
EOT
	if [ $CHECK_OLD = NO ]
	then
		cat >> $PRODUCTS_TABLE1 <<EOT
    <td align=center valign=middle BGCOLOR=$BGCOLOR nowrap>${AB_LINK1}</td>
EOT
	fi
done

cat >> $PRODUCTS_TABLE1 <<EOT
  </tr>
EOT

# for $HTML1 only
cat >> $HTML1 <<EOT
    <td width=2 bgcolor=${TITLE_COLOR}> </td>
    <td><font size=1>&nbsp;</td> 
    <td><font size=1>&nbsp;</td> 
    <td><font size=1>&nbsp;</td> 
  </tr> 

EOT

# ===================================================================================
# 5.4.8 Duplicate line and adapt for 'raw' links
# ===================================================================================
# products (specific link structure, will break if e.g. files are not on qcweb any longer)
cat $PRODUCTS_TABLE1 |\
 sed "s/P\.\.\./r\.\.\./" |\
 sed "s|overview of all processed CAL data per da.*links for details|overview of CAL raw file displays per day; click on 'raw' links for details|g" |\
 sed "s/CAL product quality:/Raw CAL displays:/" |\
 sed "s/all processed CAL data with scores.*link\]/raw file displays for all CAL data/" |\
 sed "s|href=http.*logs|href=${REL_URL}/RAWDISP|" |\
 sed "s/status_/rawfiles_START_/" |\
 sed "s/no CAL products/no CAL raw files/"g |\
 sed "s/no data or no raw files/no data or no CAL raw files/" |\
 sed "s|color=.*<b>products</b|color=${REPORT_COLOR}><i><font size=1>raw</font></i|" |\
 sed "s|color=.*<i>products</i|color=${REPORT_COLOR}><i><font size=1>raw</font></i|" >> $PRODUCTS_TABLE

# RAWDISP_SUPPORT=NO: disable links
if [ $RAWDISP_SUPPORT = NO ]
then
	sed -i -e "s|no raw files|n/a|" $PRODUCTS_TABLE
	sed -i -e "/rawfiles_START/s|<a href=.*>raw<|<i><font size=1 color=#666><a title=\"not available\">n/a</a><|" $PRODUCTS_TABLE
fi

# raw disp links
cat $PRODUCTS_TABLE1 >> $PRODUCTS_TABLE
rm $PRODUCTS_TABLE1

cat $HTML1 |\
 sed "s/P\.\.\./r\.\.\./" |\
 sed "s|overview of all processed data per date.*links for details|overview of CAL raw file displays per day; click on 'raw' links for details; last 7 days only|g" |\
 sed "s/CAL product quality:/Raw CAL displays:/" |\
 sed "s/all processed data with scores.*link]/raw file displays for all CAL data/" |\
 sed "s|href=http.*logs|href=${REL_URL}/RAWDISP|" |\
 sed "s/<img src.*border=0>//" |\
 sed "s/status_/rawfiles_START_/" |\
 sed "s/information about product quality/information about raw file displays/" |\
 sed "s/no CAL products/no CAL raw files/"g |\
 sed "s/no data or no raw files/no data or no CAL raw files/" |\
 sed "s|color=.*<b>products</b|color=${REPORT_COLOR}><i>raw</i|" |\
 sed "s|color=.*<i>products</i|color=${REPORT_COLOR}><i>raw</i|" >> $HTML

# RAWDISP_SUPPORT=NO: disable links
if [ $RAWDISP_SUPPORT = NO ]
then
	sed -i -e "/rawfiles_START/s|<a href=.*>raw<|<i><font size=1 color=#666><a title=\"not available\">n/a</a><|" $HTML
fi

# raw disp links
cat $HTML1 >> $HTML

cat >> $HTML <<EOT
<tr>
<td COLSPAN=100><img src="${IMG_URL}/sqg.png" height=1 width=100% border=0></td>
</tr>

EOT
rm $HTML1 

# ===================================================================================
# 5.4.9 CAL_ANALYZED
# ===================================================================================
# cleanup CAL_ANALYZED for outdated entries (outdated: not from this or past year)

if [ $FORCED_MODE = YES ]
then
	writeLog "9. Check issues and analysis ..."
	scpLog
fi

CURRENT_YEAR=`date -u +%Y`
PAST_YEAR=`echo $CURRENT_YEAR | awk '{print $1-1}'`

if [ -s $DFO_MON_DIR/CAL_ANALYZED ]
then
	rm -f $DFO_MON_DIR/CAL_ANALYZED.$$
	grep -v "^#" $DFO_MON_DIR/CAL_ANALYZED | awk '{print "CAL"$1"CAL",$0}' | sed "s/-/ /" | awk '{if ($1==current || $1==past) {print}}' current=CAL$CURRENT_YEAR past=CAL$PAST_YEAR > $DFO_MON_DIR/CAL_ANALYZED.$$
	if [ -s $DFO_MON_DIR/CAL_ANALYZED.$$ ]
	then
		sed "s/^.................//" $DFO_MON_DIR/CAL_ANALYZED.$$ > $DFO_MON_DIR/CAL_ANALYZED
	fi
	rm -f $DFO_MON_DIR/CAL_ANALYZED.$$
fi

# analysis file: check if not yet analyzed cases exist on $DFO_WEB_SERVER
# calEditor.dat: contains MISS/NOK cases from last run, edited by calEditor.php
# these cases are merged with CAL_ANALYZED
# [in case of calEditor.dat not existing on web server, this will cause an error in their log, but this can be accepted, according to mnaumann]
cd $TMP_DIR
echo "- check for ${PUB_QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/reports/${TARG_DIR}/php/calEditor.dat ..."
rm -f calEditor.dat
eval "wget -q --user=QC1_USER --password=$CALEDIT_PWD ${PUB_QC_WEB_SERVER_URL}/${DFO_INSTRUMENT}/reports/${TARG_DIR}/php/calEditor.dat -T 5 -t 1"

if [ ! -s calEditor.dat ]
then
	echo "  ... none found."
else
	echo "  ... found and downloaded."
fi

# merge with local CAL_ANALYZED
if [ -s calEditor.dat ]
then
	cp $DFO_MON_DIR/CAL_ANALYZED $DFO_MON_DIR/CAL_ANALYZED.save
	rm -f $TMP_DIR/calEditor.analyzed $TMP_DIR/calEditor.preliminary

# calEditor.analyzed
	grep -v "^#" calEditor.dat | grep "ANALYZED:" > $TMP_DIR/calEditor.analyzed
	if [ -s $TMP_DIR/calEditor.analyzed ]
	then
		for LINE in `cat $TMP_DIR/calEditor.analyzed | awk '{print $1}'`
		do
			SETUP=`echo "$LINE" | sed "s|</td><td>| |g" | awk '{print $1,$2,$3}' | sort -u` 
			STRING=`sed "s|</td><td>| |g" $TMP_DIR/calEditor.analyzed | grep "^$SETUP" | sed "s/<!--CAL1.*CAL2-->//"  | sed "s/<font size=1> //"`
			if [ "Q$STRING" != Q ]
			then
				sed -i -e "s|^${SETUP}.*|${STRING}|" $DFO_MON_DIR/CAL_ANALYZED
			fi
		done
	fi

# calEditor.preliminary
	grep -v "^#" calEditor.dat | grep "PRELIMINARY" > $TMP_DIR/calEditor.preliminary
	if [ -s $TMP_DIR/calEditor.preliminary ]
	then
		for LINE in `cat $TMP_DIR/calEditor.preliminary | awk '{print $1}'`
		do
			SETUP=`echo "$LINE" | sed "s|</td><td>| |g" | awk '{print $1,$2,$3}' | sort -u` 
			STRING=`sed "s|</td><td>| |g" $TMP_DIR/calEditor.preliminary | grep "^$SETUP" | sed "s/<!--CAL1.*CAL2-->//"  | sed "s/<font size=1> //"`
			if [ "Q$STRING" != Q ]
			then
				sed -i -e "s|^${SETUP}.*|${STRING}|" $DFO_MON_DIR/CAL_ANALYZED
			fi
		done
	fi
fi

# ===================================================================================
# 5.4.9 scan for VCALs
# ===================================================================================

echo "- scan VCALs ..."
if [ ! -s $TMP_DIR/setups ]
then
	touch $TMP_DIR/setups
fi

rm -f $TMP_DIR/vcal_dates
if [ "Q$CHECK_VCAL" != "Q" ]
then
	dfits `ls $DFO_CAL_DIR/VCAL | grep .hdr` | fitsort -d DATE | awk '{print $2}' | sort -u > $TMP_DIR/vcal_dates
fi

# ===================================================================================
# 5.4.10 setup part
# ===================================================================================

echo "- create setup results line by line ..."
FIRST_S=`cat $TMP_DIR/setups | head -1 | awk '{print $1}'`
N_SETUPS=`wc -l $TMP_DIR/setups | awk '{print $1}'`
rm -f $TMP_DIR/ANALYSIS.html $TMP_DIR/list_blink

case $CAL4CAL in
"NO"  ) NO_SCIENCE="- no science files found, nothing to report." 
  DATA_TYPES="Science data types are defined by instrument mode. See also 'ASSOC-RULES'" ;;
"YES" ) NO_SCIENCE="- no calibration files to check, nothing to report." 
  DATA_TYPES="Data types of triggering calibration data. See also 'ASSOC-RULES'" ;;
esac

if [ ! -s $CONSOL ]
then
	cat >> $HTML <<EOT
$TR
<th align=left COLSPAN=100><font size=2 color=#666666>${NO_SCIENCE}</font></th></tr>
EOT
else
	cat >> $HTML <<EOT

  <tr bgcolor=${RESULT_BGCOLOR}>
    <th align=left nowrap><font size=2><a title="${DATA_TYPES}">Data types:</a></font></th>
    <th align=left nowrap><font size=2><a title="Setup is a unique string composed of header keys like read mode, filter, grating etc. See also 'Information' below">Setup:</a></font></th>
    <td width=2 bgcolor=${TITLE_COLOR}> </td>
    <th COLSPAN=${N_DATES}>&nbsp;</th>
    <td width=2 bgcolor=${TITLE_COLOR}> </td>
    <td>&nbsp;</td>
    <td>&nbsp;</td>
    <td>&nbsp;</td>
</tr>

EOT

	rm -f $TMP_DIR/cc_list_override
	rm -f $TMP_DIR/calEditor_added.dat

# do user-defined sorting on raw_types
ANALYSIS_INDEX=1
FIRST_RT=`cat $TMP_DIR/cc_sorted_rawtypes | head -1`
for RT in `cat $TMP_DIR/cc_sorted_rawtypes`
do

# CMAP_LINK (from getStatusAB): for CAL4CAL=NO
	CMAP_LINK=""
	rm -f $TMP_DIR/cs_cmap

	grep "^MODE" $CONFIG_CALCHECK/OCA/config.createCalibMap | grep "[[:space:]]${RT}[[:space:]]" | awk '{print $3}' > $TMP_DIR/cs_cmap
	if [ -s $TMP_DIR/cs_cmap ]
	then
		CHECK_CMAP=`cat $TMP_DIR/cs_cmap | wc -l`
		if [ $CHECK_CMAP = 1 ]
		then
			CMODE=`cat $TMP_DIR/cs_cmap`
			CTAG=""
		else
			CMODE=ALL
			CTAG="(used in more than one mode)"
		fi

		if [ $CMODE = DET ]
		then
			CMODE=ALL
			CTAG="(used in more than one mode)"
		fi
		CMAP_LINK="${CMAP_URL}/cascade_${DFO_INSTRUMENT}_${CMODE}.html"
	else
		CMODE=ALL
		CTAG="(not configured for any mode)"
		CMAP_LINK="${CMAP_URL}/cascade_${DFO_INSTRUMENT}_${CMODE}.html"
	fi

	if [ Q$CMAP_LINK != Q ] && [ $CAL4CAL = NO ]
	then
		RT_LINK="<a href=\"javascript:openWin('$CMAP_LINK');\" title=\"find $RT in the CALCHECKER calibration cascade ${DFO_INSTRUMENT}_${CMODE} $CTAG\"><font color=$TITLE_COLOR>$RT</font></a>"
        	else
                	RT_LINK=$RT
        	fi

		NUMBER_S=`cat $TMP_DIR/setups | grep "[[:space:]]${RT}$" | wc -l`
		if [ $NUMBER_S = 0 ]
		then
			continue
		else
			FIRST_S=`cat $TMP_DIR/setups | grep "[[:space:]]${RT}$" | awk '{print $1}' | sort -n | head -1`
		fi

		for S in `cat $TMP_DIR/setups | grep "[[:space:]]${RT}$" | awk '{print $1}' | sort -n`
		do
# OVERALL_TAG marks possible cases :miss or :nok and controls whether the 'action required' section is shown
			OVERALL_TAG=""

			if [ $APPLY_MODIF = TRUE ]
			then
				S_MODIF=`eval "echo $S | $TMP_DIR/cc_modif_string"`
			else
				S_MODIF=$S
			fi

			TD="<th align=right><font size=2>"
			if [ $S = "$FIRST_S" ]
			then
				cat >> $HTML <<EOT
  $TR
    <th align=right valign=top ROWSPAN=${NUMBER_S}><font size=2>${RT_LINK}</font></th>
    $TD ${S_MODIF}</font></th>
    <td width=2 bgcolor=${TITLE_COLOR} ROWSPAN=${NUMBER_S}> </td>
EOT
			else
				cat >> $HTML <<EOT
  $TR
    $TD ${S_MODIF}</font></th>
EOT
			fi

			rm -f $TMP_DIR/last_hdr 

			for D in `cat $TMP_DIR/cc_list_open_issues 2>/dev/null $TMP_DIR/cc_list_dates2`
			do
				CHECK_OLD=`grep $D $TMP_DIR/cc_list_open_issues 2>/dev/null`
				if [ Q$CHECK_OLD != Q ]
				then
					CHECK_OLD=YES
				else
					CHECK_OLD=NO
				fi

				CHECK_EXIST=""
				if [ -s $DFO_HDR_DIR/$D ]
				then
					CHECK_EXIST=`ls $DFO_HDR_DIR/$D | grep hdr`
				fi

# no data (yet)
				if [ ! -s $DFO_HDR_DIR/$D ] || [ "Q$CHECK_EXIST" = "Q" ] 
				then
					if [ $S = $FIRST_S ] && [ $RT = $FIRST_RT ]
					then
						NO_HDR_TAG="&nbsp;"
# case no headers: check if attempted to download
       			        		CHECK_SCANNED=`grep " $D " $ORIG_MON_DIR/DFO_STATUS | grep "hdr_Notexist" | head -1`
       			         		if [ "Q$CHECK_SCANNED" != "Q" ] && [ $D = $TODAY ]
       		      		   		then
							NO_HDR_TAG="<i>no data (yet)</i>"
						fi
					else
						NO_HDR_TAG="&nbsp;"
					fi

 	      	             		cat >> $HTML <<EOT
<td align=center valign=middle width=1%><font size=1 color=#666666> $NO_HDR_TAG </font></td>
EOT

# normal case: data exist
				else
					ls $DFO_HDR_DIR/$D | grep .hdr | tail -1 >> $TMP_DIR/last_hdr
					TITLE=""
	
					S1=`echo $S | sed "s/\&\&/ /"`
					ENTRY=`grep "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S1[[:space:]]"`

					MISS=`grep  "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S1[[:space:]]" | grep ":MISS*" | head -1`
					NOK=`grep   "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S1[[:space:]]" | grep ":NOK*"  | head -1`
					LOST=`grep  "^$D" $REPLACE | grep "[[:space:]]${RT}[[:space:]]" | grep "[[:space:]]$S1[[:space:]]" | grep ":LOST*" | head -1`

					REF="href=\"javascript:openWin('${CALREPORT_DIR}/${RESULTS_NAME}_$D.html');\""
					ANALYSIS=""
					ANALYSIS_TAG=""
					ACTION="<!--FILTER-->"

# the sequence of events is important: empty, MISS, NOK, LOST (--> a NOK is only flagged if no MISS occured, a LOST only if no NOK/MISS)
					if [ "Q$ENTRY" = "Q" ]
					then
						ENTRY="&nbsp;"
						COLOR="$RESULT_BGCOLOR"
						if [ $CHECK_OLD = YES ]
						then
							COLOR="$OLD_RESULT_BGCOLOR"
						fi
						TITLE=""
						REF=""
					elif [ "Q$MISS" != "Q" ]
					then
						ENTRY="miss"
# check for entry and analysis in CAL_ANALYZED
						CHECK_ENTRY=`grep "^$D " $DFO_MON_DIR/CAL_ANALYZED | grep " $RT " | grep " $S_MODIF "`
						if [ "Q$CHECK_ENTRY" = Q ]
						then
							echo "$D $RT $S_MODIF :MISS " >> $DFO_MON_DIR/CAL_ANALYZED
						fi

						TITLE="title=\"missing calibrations, should be acquired as soon as possible\""
						COLOR="$RED"

						getAnalysis
			
# complete but some outdated
					elif [ "Q$NOK" != "Q" ]
					then
						ENTRY="nok"
						TITLE="title=\"all calibrations existing but some are timed-out\""
						COLOR="$YLW"
# check for entry and analysis in CAL_ANALYZED
						CHECK_ENTRY=`grep "^$D " $DFO_MON_DIR/CAL_ANALYZED | grep " $RT " | grep " $S_MODIF "`
						if [ "Q$CHECK_ENTRY" = Q ]
						then
							echo "$D $RT $S_MODIF :NOK " >> $DFO_MON_DIR/CAL_ANALYZED
						fi

						getAnalysis

# only if no MISS/NOK: LOST displayed (since no action required)
					elif [ "Q$LOST" != "Q" ]
					then
						ENTRY="LOST"
						COLOR="$LOST_COLOR"
						TITLE="title=\"lost calibration(s), cannot be taken anymore\""
# check for entry and analysis in CAL_ANALYZED: no entry --> enter :LOST in CAL_ANALYZED
						CHECK_ENTRY=`grep "^$D " $DFO_MON_DIR/CAL_ANALYZED | grep " $RT " | grep " $S_MODIF "`
						CHECK_ENTRY2=`echo "$CHECK_ENTRY" | grep -v "ANALYZED:" | grep ":MISS"`
						if [ "Q$CHECK_ENTRY" = Q ]
						then
							echo "$D $RT $S_MODIF :LOST " >> $DFO_MON_DIR/CAL_ANALYZED
# a :MISS could have become a :LOST in the meantime (after UT_LIMIT), hence check and update CAL_ANALYZED
						elif [ "Q$CHECK_ENTRY" != Q ] && [ "Q$CHECK_ENTRY2" != Q ]
						then
							sed -i -e "s|^$D $RT $S_MODIF :MISS|$D $RT $S_MODIF :LOST|" $DFO_MON_DIR/CAL_ANALYZED
						fi
						getAnalysis

# complete and in time
					else
						CHECK_ENTRY=`grep "^$D " $DFO_MON_DIR/CAL_ANALYZED | grep " $RT " | grep " $S_MODIF "`
						if [ "Q$CHECK_ENTRY" != "Q" ]
						then
							grep -v "$CHECK_ENTRY" $DFO_MON_DIR/CAL_ANALYZED > $DFO_MON_DIR/CAL_ANALYZED1
							mv $DFO_MON_DIR/CAL_ANALYZED1 $DFO_MON_DIR/CAL_ANALYZED
						fi
						ENTRY="ok"
						TITLE="title=\"all calibrations existing and within configured time range\""
						COLOR="$GRN"
					fi

# override automatic entries by analysis entries; OK and NOK results get lighter colours to mark possible deviation from automatic result; 
# LOST and MISS remain RED since this is an alert
# exceptions for overriding:
#  ENTRY=ok cannot be overridden by ANALYSIS_TAG NOK/MISS/LOST (for this we have BADQUAL mechanism)
#  ENTRY=nok cannot be overridden by ANALYSIS_TAG MISS/LOST (again: BADQUAL)
# in short: ANALYSIS_TAG grade cannot be lower than automatic ENTRY grade; 
# in such case, a CAL_ANALYZED entry is de-activated

# a) de-activate entry in CAL_ANALYZED if it is lower than automatic grade (ENTRY)
					if  ([ "Q${ENTRY}" = "Qok"  ] && [ "Q${ANALYSIS_TAG}" != Q     ]) \
					 || ([ "Q${ENTRY}" = "Qnok" ] && [ "Q${ANALYSIS_TAG}"  = QMISS ])
					then
# be careful, there are setups including '/' strings
						sed -i -e "s|^$D $RT $S_MODIF |#&|" $DFO_MON_DIR/CAL_ANALYZED
						getAnalysis
						ANALYSIS_STRING=`grep ">${RT}<.*>${S_MODIF}<.*>${D}" $TMP_DIR/ANALYSIS.html`
						if [ "Q$ANALYSIS_STRING" != Q ]
						then
							grep -v ">${RT}<.*>${S_MODIF}<.*>${D}" $TMP_DIR/ANALYSIS.html > $TMP_DIR/analysis.html.$$
							if [ -s $TMP_DIR/analysis.html.$$ ]
							then
								mv $TMP_DIR/analysis.html.$$ $TMP_DIR/ANALYSIS.html
							else
								rm $TMP_DIR/ANALYSIS.html
							fi
						fi
					fi

# b) now check for override cases
					if [ "Q${ENTRY}" = "Q&nbsp;" ] || [ "Q${ANALYSIS_TAG}" = Q ]
					then						
						:
					elif [ "Q${ENTRY}" = "Qmiss" ] ||\
                                            ([ "Q${ENTRY}" = "Qnok"  ] && [ "Q${ANALYSIS_TAG}" != QMISS ]) ||\
					    ([ "Q${ENTRY}" = "QLOST" ] && [ "Q${ANALYSIS_TAG}" != QMISS ] && [ "Q${ANALYSIS_TAG}" != QNOK ])
					then
						echo "$D $RT $S1 $ENTRY ${ANALYSIS_TAG}" >> $TMP_DIR/cc_list_override

						case ${ANALYSIS_TAG} in
						 "OK"   ) ENTRY=ok   ; COLOR="$LGRN" ; TITLE="title=\"analyzed: no issue with calibrations\"" ;;
         					 "NOK"  ) ENTRY=nok  ; COLOR="$LYLW" ; TITLE="title=\"analyzed: all calibrations existing but some are timed-out\"" ;;
         					 "MISS" ) ENTRY=miss ; COLOR="$RED" ; TITLE="title=\"analyzed: missing calibrations, should be acquired as soon as possible\"" ;;
        					esac
					fi

# LOST and not analyzed: make the text readable
					if [ "Q${ENTRY}" = "QLOST" ]
					then
						ANALYSIS=`echo $ANALYSIS | sed "s/${TITLE_COLOR}/white/"`
					fi	

# OVERALL_TAG: empty if all OK, otherwise NOK; a NOK could have become OK by overriding $ANALYSIS_TAG
					if [ "Q$ENTRY" = Qnok ] || [ "Q$ENTRY" = Qmiss ] || [ "Q$ENTRY" = QLOST ]
					then
						OVERALL_TAG="NOK"
					fi

					TD="<td align=center bgcolor=$COLOR><font size=2>"
					cat >> $HTML <<EOT
    ${TD}<a $REF $TITLE>${ENTRY}</a><br>
${ACTION}
${ANALYSIS}
</font></td>
EOT
				fi
# end of DATE loop
			done

			if [ $S = "$FIRST_S" ]
			then
				echo "    <td width=2 bgcolor=${TITLE_COLOR} ROWSPAN=${NUMBER_S}> </td>" >> $HTML
			fi

# ======================================================================
# 5.5 evaluate required actions per RT and SETUP (table row; 
#     if entry found in $OUT, and OVERALL_TAG=NOK)
# ======================================================================

			CHECK_ACT=`grep "^${RT} ${S}:" $OUT | sed "s!^${RT} ${S}:!!"`
			CHECK_ACT=`squeeze "$CHECK_ACT"`

			CHECK_LOST=`grep "^${RT} ${S}:" $OUT_LOST | sed "s!^${RT} ${S}:!!"`
			CHECK_LOST=`squeeze "$CHECK_LOST"`

			if [ "Q$CHECK_ACT" = Q ]
			then
				CHECK_ACT=NO
			fi

			if [ "Q$CHECK_LOST" = Q ]
			then
				CHECK_LOST=NO
			fi

			if ([ "Q$CHECK_ACT" != "QNO" ] || [ "Q$CHECK_LOST" != QNO ]) && [ Q$OVERALL_TAG = QNOK ]
			then

# store info about missing calibs for later reference in calEditor.dat
# avoid '|' since this breaks the php script calEditor.php
# reconstruct $DATE from $REPLACE
# LOST: entry in calEditor_added.dat only if no MISS|NOK, otherwise no entry (would be confusing)
				for D1 in `grep " $RT ${S_MODIF} " $REPLACE | egrep "MISS|NOK|LOST" | awk '{print $1}' | sort -u`
				do
					CHECK_EXIST=`grep "^$D1 ${RT} ${S_MODIF}" $REPLACE | egrep "MISS|NOK" | head -1`
					if [ "Q$CHECK_EXIST" != Q ]
					then
						echo "$D1 ${RT} ${S_MODIF} $CHECK_ACT"  | sed "s/|//g" >> $TMP_DIR/calEditor_added.dat
					else
						echo "$D1 ${RT} ${S_MODIF} $CHECK_LOST" | sed "s/|//g" >> $TMP_DIR/calEditor_added.dat
					fi
				done

				case $CHECK_LOST in
				 "NO"  ) LOST_ENTRY="<td>&nbsp;</td>" ;;
				 *     ) LOST_ENTRY="<td bgcolor=$YLW align=center><font size=1><a href=\"javascript:openWin('${CALREPORT_DIR}/${RESULTS_NAME}_setup.html#${S}');\" title=\"all results for setup ${S_MODIF}\">$CHECK_LOST</a> </font></td>" ;;
			 	esac

				case $CHECK_ACT in
				 "NO"  ) ACTION_ENTRY1="<td>&nbsp;</td>"
					 ACTION_ENTRY2="<td>&nbsp;</td>" ;;
				 *     ) ACTION_ENTRY1="<td bgcolor=$YLW align=center><font size=1><a href=\"javascript:openWin('${CALREPORT_DIR}/${RESULTS_NAME}_setup.html#${S}');\" title=\"all results for setup ${S_MODIF}\">$CHECK_ACT</a> </font></td>"
					 ACTION_ENTRY2="<th bgcolor=$YLW align=center><font size=1><a href=\"javascript:openWin('${CALREPORT_DIR}/${RESULTS_NAME}_setup.html#${S}');\">${S_MODIF}</a> </font></th>" ;;
				esac

				cat >> $HTML <<EOT
    $LOST_ENTRY
    $ACTION_ENTRY1
    $ACTION_ENTRY2
    </tr>
EOT
			else
				cat >> $HTML <<EOT
    <td>&nbsp;</td>
    <td align=center BGCOLOR=$GRN><font size=2><a href="javascript:openWin('${CALREPORT_DIR}/${RESULTS_NAME}_setup.html#${S}');" title="all results for setup ${S_MODIF}">all ok</a></font></td>
    <td>&nbsp;</td>
    </tr>
EOT
			fi
		done

		cat >> $HTML <<EOT

    <tr>
      <td COLSPAN=100><img src="${IMG_URL}/sqg.png" height=1 width=100% border=0></td>
    </tr>
EOT
	done
fi

# replace string LAST_HDR by actual value
if [ -s $TMP_DIR/last_hdr ]
then
	LAST_HDR=`cat $TMP_DIR/last_hdr | sort -u | tail -1`
	LAST_HDR=`basename $LAST_HDR | sed "s/${DFO_FILE_NAME}\.//"`
	sed -i -e "/<!-- LAST_HDR/s/available:/$LAST_HDR/" $HTML
fi

# add ANALYSIS.html if existing
if [ -s $TMP_DIR/ANALYSIS.html ]
then
	cat >> $HTML <<EOT
<tr><td COLSPAN=100>
 <table border=0 cellspacing=1 cellpadding=1 width=100%>
    <tr bgcolor=#CCCCCC align=left><th COLSPAN=6><font size=2>ANALYSIS NOTES:</font></th></tr>
    <tr bgcolor=#CCCCCC>
      <td><font size=1>Index</font></td>
      <td nowrap><font size=1>data type</font></td>
      <td><font size=1>setup</font></td>
      <td><font size=1>date</font></td>
      <td><font size=1>flag</font></td>
      <td width=80%><font size=1>analysis</font></td>
    </tr>
EOT

	cat $TMP_DIR/ANALYSIS.html >> $HTML
	echo "    </table>"   >> $HTML
	echo "</td></tr>" >> $HTML
fi

cat >> $HTML <<EOT

    </table>
</td></tr></table> <!-- end of white border table-->
    </td> 
  </tr>

  <tr>
    <td COLSPAN=3>&nbsp;</td>
  </tr>
EOT

# specific information 
cat >> $HTML <<EOT
    <tr VALIGN=TOP ALIGN=LEFT BGCOLOR=#CCCCCC>
      <td width=50%>
EOT

if [ "Q$INFO_NAME" != "Q" ]
then
	cat >> $HTML <<EOT
      <table width=100% border="0" cellpadding="1" cellspacing="1">
        <tr>
          <th bgcolor=#999999> <font size=2 color=#FFFFFF>INFORMATION SPECIFIC TO ${DFO_INSTRUMENT}
            &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; [<a href="javascript:openWin('${QC_URL}/CC_help_config.html');" title="info about configuration">?</a>]</font>
</th>
        </tr>
        <tr>
          <td><font size=2>
EOT
	cat $DFO_CONFIG_DIR/$INFO_NAME | grep -v "^#" >> $HTML
	cat >> $HTML <<EOT
          </font></td>
        </tr>
      </table>
EOT
else
	echo "&nbsp;" >> $HTML
fi

cat >> $HTML <<EOT
    </td>

    <td width=50% bgcolor=#CCCCCC COLSPAN=2>

      <table width=100% border="0" cellpadding="1" cellspacing="1">
        <tr bgcolor=#999999>
          <th COLSPAN=2><font size=2 color=#FFFFFF>CONFIGURATION
            &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; [<a href="javascript:openWin('${QC_URL}/CC_help_config.html');" title="info about configuration">?</a>]</font>
          </th>
        </tr>
        <tr bgcolor=#999999>
          <td align=right><font size=2>Number of days scanned:       </td>
          <td BGCOLOR=#999999 valign=top><font size=2><b>$N_VCAL_LIST</b> </td>
        </tr>
      
        <tr bgcolor=#999999>
          <td align=right><font size=2>Range of days for the issue memory <font size=1><i>(configured)</font>: </td>
          <td BGCOLOR=#999999 valign=top><font size=2><b>$N_VCAL_MEM </b></td>
        </tr>

        <tr bgcolor=#999999>
          <td align=right><font size=2>Range of days for the calibration memory <font size=1><i> (issue memory plus longest validity)</font>: </td>
          <td BGCOLOR=#999999 valign=top><font size=2><b>$N_VCAL_LIST_EX</b></td>
        </tr>

        <tr bgcolor=#999999>
          <td align=right nowrap><font size=2>Days in the calibration&issue memory: </td>
          <td BGCOLOR=#999999>
            <select>
EOT

# pseudo-form for vcalib
CHECK_VCAL=`ls $DFO_CAL_DIR/VCAL | grep hdr`
if [ "Q$CHECK_VCAL" = "Q" ]
then
	echo "<option value=\"none\">none</option>" >> $HTML
else
	cd $DFO_CAL_DIR/VCAL
	dfits `ls | grep .hdr` | fitsort -d DATE | awk '{print $2}' | sort -u | sed "s/^.*/                <option value=\"&\">&<\/option>/" >> $HTML
fi

if [ $FORCED_MODE = YES ]
then
	writeLog "10. Add daytime calibration signals ..."
	scpLog
fi

EOCAL_LABEL="Daytime calibration signals for last 7 days* (UT):"
if [ $DFO_FILE_NAME = UVES ]
then
	EOCAL_LABEL="${EOCAL_LABEL}<br><font size=1>(UVES queue first, then FLAMES queue)</font>"
fi

cat >> $HTML <<EOT
            </select>
          </td>
        </tr>

	<tr bgcolor=#999999> 	<!--FILTER-->
	  <td align=right><font size=2>${EOCAL_LABEL}</td>	<!--FILTER-->
          <td BGCOLOR=#999999>	<!--FILTER-->
            <select>			<!--FILTER-->
	     <option value="none">DATE___BOCAL_EOCAL_INS</option>	<!--FILTER-->
             <option value="none">=====================</option>		<!--FILTER-->
EOT

# here we don't add cc_list_open_issues, not relevant anymore
cat $TMP_DIR/cc_list_dates2 > $TMP_DIR/cc_list_dates1
echo $TODAY1 >> $TMP_DIR/cc_list_dates1

for D in `cat $TMP_DIR/cc_list_dates1`
do
# EOCAL: different for UVES and others (because of FLAMES)
	if [ -s $TMP_DIR/beocal_results ] && [ $DFO_INSTRUMENT != UVES ]
	then
		FIRST_BOCAL=`grep $D $TMP_DIR/beocal_results | grep BOCAL | head -1 | awk '{print $2}'`
		FIRST_BOCAL=`echo $FIRST_BOCAL NONE | awk '{print $1}'`
		LAST_EOCAL=`grep  $D $TMP_DIR/beocal_results | grep EOCAL | tail -1 | awk '{print $2}'`
		LAST_EOCAL=`echo $LAST_EOCAL NONE | awk '{print $1}'`
		if [ $D = $TODAY1 ]&& [ $LAST_EOCAL = NONE ]
		then
			LAST_EOCAL="..."
		fi
		echo "<option value=\"none\">${D} $FIRST_BOCAL $LAST_EOCAL $DFO_FILE_NAME </option> <!--FILTER-->" >> $HTML
	elif [ -s $TMP_DIR/beocal_results ] && [ $DFO_INSTRUMENT = UVES ]
	then
		for I in `echo UVES GIRAF`
		do
			I1=`echo $I  | sed "s/GIRAF/FLAMES/"`
			FIRST_BOCAL=`grep $D $TMP_DIR/beocal_results | grep BOCAL | grep $I | head -1 | awk '{print $2}'`
			FIRST_BOCAL=`echo $FIRST_BOCAL NONE | awk '{print $1}'`
			LAST_EOCAL=`grep  $D $TMP_DIR/beocal_results | grep EOCAL | grep $I | tail -1 | awk '{print $2}'`
			LAST_EOCAL=`echo $LAST_EOCAL NONE | awk '{print $1}'`
			if [ $D = $TODAY1 ]&& [ $LAST_EOCAL = NONE ]
			then
				LAST_EOCAL="..."
			fi
			echo "<option value=\"none\">${D} $FIRST_BOCAL $LAST_EOCAL $I1 </option> <!--FILTER-->" >> $HTML
		done
	fi
done

cat >> $HTML <<EOT
            </select>	<!--FILTER-->
          </td>		<!--FILTER-->
        </tr>		<!--FILTER-->

        <tr bgcolor=#999999>
          <td align=right><font size=2>*Date on this monitor changes at:</font></td>
          <td BGCOLOR=#999999 valign=top><font size=2><b>${BOUNDARY}:00 UT</b></font></td>
        </tr>

      </table>

    </td>
  </tr>
EOT

# last line
case $INCREM in
 "YES" ) MARK_INCREM="bgcolor=\"#CCCCCC\"><a title=\"executed incrementally\"><b><font color=\"${REPORT_COLOR}\" size=1>increm</font></b></a>" ;;
 "NO"  ) MARK_INCREM="bgcolor=\"${REPORT_COLOR}\"><a title=\"full execution\"><b><font color=\"#CCCCCC\" size=1>full</font></b></a>" ;;
esac

cat >> $HTML <<EOT
  <tr BGCOLOR=#CCCCCC>
    <td ALIGN=RIGHT COLSPAN=3>
      <table width="100%" border="0" cellpadding="0" cellspacing ="0">
        <tr>
          <td ALIGN=RIGHT>
            <font size=2 COLOR=#FFFFFF> <b>powered by QC</b> [calChecker v${TOOL_VERSION}]&nbsp;&nbsp;</font>
          </td>
          <td width="1%" ${MARK_INCREM}</td>
        </tr>
      </table>
    </td>
  </tr>

</table>

</body>
</html>
EOT

# ======================================================================
# 5.6 create calEditor interface and upload calEditor.dat
# 5.6.1 create calEditor interface
# ======================================================================

echo "- create calEditor and calBadQual interfaces ..."
createCalEditor 
createCalBadQual

# ======================================================================
# 5.6.2 security: index.html and .esoacc (to protect the nightlogs)
# .esoacc: is taken care by OTS, they protect each directory containing
# 'NIGHTLOG' on qcweb; this .esoacc file here is created only for reference
# (the operational one should have the same content)
# ======================================================================

cat > index.html <<EOT
Access not permitted
EOT

cat > .esoacc <<EOT
AuthName "calChecker analysis editor:"
AuthType Basic

AuthUserFile /home/web/eso/docs/observing/dfo/quality/.webauth/passwordfile
require valid-user

EOT
chmod o+r .esoacc

# ======================================================================
# 5.6.3 create and upload up-to-date calEditor.dat
# ======================================================================
if [ $FORCED_MODE = YES ]
then
	writeLog "11. Upload up-to-date calEditor.dat ..."
	scpLog
fi

rm -f $TMP_DIR/calEditor.dat $TMP_DIR/calEditor.dat1
if [ -s $DFO_MON_DIR/CAL_ANALYZED ]
then
	grep -v "^#" $DFO_MON_DIR/CAL_ANALYZED |\
 grep -v "ANALYZED:" |\
 grep -v "^$" \
> $TMP_DIR/calEditor.dat
fi

# provide top-down sorting as on HTML output page
if [ -s $TMP_DIR/calEditor.dat ]
then
	rm -f $TMP_DIR/calEditor.dat1
	for RT in `cat $TMP_DIR/cc_sorted_rawtypes`
	do
		CHECK_RT=`grep " $RT " $TMP_DIR/calEditor.dat`
		if [ "Q$CHECK_RT" != Q ]
		then
			grep " $RT " $TMP_DIR/calEditor.dat | sort -k3 >> $TMP_DIR/calEditor.dat1
		fi
	done

	if [ -s $TMP_DIR/calEditor.dat1 ]
	then
		mv $TMP_DIR/calEditor.dat1 $TMP_DIR/calEditor.dat
	fi

# add column with missing files
	rm -f $TMP_DIR/calEditor.dat1
	for STRING in `cat $TMP_DIR/calEditor.dat | awk '{print $1"&"$2"&"$3}'`
	do
		STRING1=`echo $STRING | sed "s/\&/ /"g`
		FLAG=`grep "${STRING1}[[:space:]]" $TMP_DIR/calEditor.dat | awk '{print $4}'`
# careful: there might be setups string with '/' included, disliked by sed
		MISSING=""
		if [ -s $TMP_DIR/calEditor_added.dat ]
		then
			MISSING=`grep "${STRING1}[[:space:]]" $TMP_DIR/calEditor_added.dat | sed "s|^${STRING1}||"`
		fi

# new with v3.7.2: suppress 'not_found_since_out_of_DATE_range' since useless
		if [ "Q$MISSING" = Q ]
		then
			#MISSING="not_found_since_out_of_DATE_range"
			:
		else
# CAL1...CAL2 is for marking the $MISSING calibs (to be removed for CAL_ANALYZED when downloaded again)
			echo "$STRING1 $FLAG <!--CAL1-->$MISSING<!--CAL2-->" >> $TMP_DIR/calEditor.dat1
		fi
	done
fi

# sort by DATE (first column)
rm -f $TMP_DIR/calEditor.dat
if [ -s $TMP_DIR/calEditor.dat1 ]
then
	cat $TMP_DIR/calEditor.dat1 |\
 sort -k1,1 |\
 sed "s| |</td><td>|" |\
 sed "s| |</td><td>|" |\
 sed "s| |</td><td>|" |\
 sed "s| |</td><td><font size=1>|" \
> $TMP_DIR/calEditor.dat
fi

ssh -o BatchMode=yes $DFO_WEB_SERVER "bin/qcDircheck ${WEB_DIR}/${TARG_DIR}/php" 
scp -o BatchMode=yes -p $TMP_DIR/calEditor.php index.html $TMP_DIR/calBadQual.php $TMP_DIR/addBadQual.php .esoacc ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}/php 1>/dev/null
rm index.html .esoacc

ssh -o BatchMode=yes $DFO_WEB_SERVER "rm -f ${WEB_DIR}/${TARG_DIR}/php/calEditor.dat"
if [ -s $TMP_DIR/calEditor.dat ]
then
	chmod a+w $TMP_DIR/calEditor.dat
	scp -o BatchMode=yes -p $TMP_DIR/calEditor.dat ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}/php/
fi

# ======================================================================
# 5.7 define cc_list_dates_OKK
# ======================================================================

# find OKK dates, store in $PREVIOUS_DIR/cc_list_dates_OKK; will be used the next time to skip OKK dates
rm -f $PREVIOUS_DIR/cc_list_dates_OKK
if [ -s $PREVIOUS_DIR/cc_list_memory_OKK ]
then
	cat $PREVIOUS_DIR/cc_list_memory_OKK | sort -u > $PREVIOUS_DIR/cc_list_dates_OKK
fi

# no data --> no issues: store plain copy of cc_list_dates4 as cc_list_dates_OKK
if [ ! -s $RESULTS ]
then
        cat $TMP_DIR/cc_list_dates4 >> $PREVIOUS_DIR/cc_list_dates_OKK
fi

# data exist --> investigate closer; expand cc_list_dates2 by cc_list_open_issues since the latter gets reconstructed next time anyway
if [ -s $RESULTS ]
then
	for D in `cat $TMP_DIR/cc_list_open_issues $TMP_DIR/cc_list_dates2 2>/dev/null`
	do

# all entries OKK: $D to go to cc_list_dates_OKK if plausibility check passed (LOST doesn't count since they cannot be cured)
		CHECK_NOK=`grep "^$D" $RESULTS | egrep "NOK|MISS" | head -1`
		CHECK_OKK=`grep "^$D" $RESULTS | grep "OKK" | head -1`
		CHECK_ENTRY=`grep "^$D" $RESULTS | egrep -v "NOK|OKK|MISS" | head -1`
		SCI_NUM=`grep   "^$D" $TMP_DIR/cc_list_scinum | awk '{print $2}'`
		if [ $CAL4CAL = YES ]
		then
			SCI_NUM=0
		fi

		if [ "Q$CHECK_NOK" = Q ]
		then
# no NOK/MISS, OKK entries exist: OKK
			if [ "Q$CHECK_OKK" != Q ] 
			then
				echo $D >> $PREVIOUS_DIR/cc_list_dates_OKK
				continue
# no NOK/MISS, no OKK, SCI_NUM=0: no science data, OKK (only for $CAL4CAL = NO)
			elif [ "Q$CHECK_OKK" = Q ] && [ Q$SCI_NUM = Q0 ]
			then
				echo $D >> $PREVIOUS_DIR/cc_list_dates_OKK
				continue
# no NOK/MISS, no OKK, SCI_NUM>0: two possibilities
# a) exceptionally no calibs required (VISIR SCI_SPEC_ECH case): entry exists but no flag 
			elif [ "Q$CHECK_ENTRY" != Q ] && [ Q$SCI_NUM != Q0 ]
			then
				echo $D >> $PREVIOUS_DIR/cc_list_dates_OKK
				continue	
# b) something went wrong with the headers; schedule for next-time re-checking
			elif [ "Q$CHECK_OKK" = Q ] && [ Q$SCI_NUM != Q0 ] && [ $CAL4CAL = NO ]
			then
				if [ -s $DFO_MON_DIR/DFO_MESSAGE ]
				then
					CHECK_DFO_STATUS=`grep "issue with $D having no calChecker results but SCI_NUM" $DFO_MON_DIR/DFO_MESSAGE`
				fi

				if [ "Q$CHECK_DFO_STATUS" = Q ]
				then
					echo "***WARNING: Date $D has no calChecker results but SCI_NUM=$SCI_NUM. Something went wrong.
This date will be analyzed again the next time.
Alternatively you may want to call the tool in REPAIR mode (-d $D) on the command line." > $TMP_DIR/message
					echo "calChecker for `whoami`@`hostname`: issue with $D having no calChecker results but SCI_NUM=$SCI_NUM" >> $DFO_MON_DIR/DFO_MESSAGE
					mail -s "calChecker warning" $OP_ADDRESS <$TMP_DIR/message
				fi
				continue
			fi
		fi

# at least one entry NOK/MISS and no $TMP_DIR/cc_list_override: $D to be checked again, no entry in cc_list_dates_OKK
		if [ "Q$CHECK_NOK" != Q ] && [ ! -s $TMP_DIR/cc_list_override ]
		then
			continue
		fi

# at least one entry NOK/MISS: check out if entry in $TMP_DIR/cc_list_override overrides NOK/MISS
		if [ "Q$CHECK_NOK" != Q ] && [ -s $TMP_DIR/cc_list_override ]
		then
			CHECK_EXIST1=`grep "^$D " $TMP_DIR/cc_list_override | head -1`
			CHECK_EXIST2=`grep "^$D " $TMP_DIR/cc_list_override | egrep " NOK| MISS" | head -1`

# check cc_list_override: if at least one entry with NOK/MISS --> no entry in cc_list_dates_OKK
# if no entry for $D: no entry in cc_list_dates_OKK
			if [ "Q$CHECK_EXIST2" != Q ] || [ "Q$CHECK_EXIST1" = Q ]
			then
				continue
			fi

# if entry with NOK/MISS exists for $D: evaluate if all entries with NOK/MISS in $RESULTS for $D are listed as OK in cc_list_override; if so: entry in cc_list_dates_OKK
			rm -f $TMP_DIR/cc_list_check_override
			grep "^$D" $RESULTS | egrep "NOK|MISS" > $TMP_DIR/cc_list_check_override
			CHECK_OK=`grep "^$D " $TMP_DIR/cc_list_override | grep " OK$" | head -1`
			if [ "Q$CHECK_OK" != Q ]
			then
				for RT in `grep "^$D " $TMP_DIR/cc_list_override | grep " OK$" | awk '{print $2}' | sort -u`
				do
					rm -f $TMP_DIR/cc_list_override_setups
					grep "^$D " $TMP_DIR/cc_list_override | grep " OK$" | grep " $RT "| awk '{print $3}' | sort -u > $TMP_DIR/cc_list_override_setups
# remove each combination of $D, $RT, $S; if nothing remains, that date is clean in terms of OKK and is added to cc_list_dates_OKK
# egrep doesn't like '+', and some setups contain '+'; we replace it by '.'
					for S in `cat $TMP_DIR/cc_list_override_setups | sed "s/+/\./g"`
					do
						rm -f $TMP_DIR/cc_list_check_override1
# one issue less ...
						egrep -v "${D}.*${RT} $S" $TMP_DIR/cc_list_check_override > $TMP_DIR/cc_list_check_override1
						if [ ! -s $TMP_DIR/cc_list_check_override1 ]
						then
							rm $TMP_DIR/cc_list_check_override
							break
						else
							mv $TMP_DIR/cc_list_check_override1 $TMP_DIR/cc_list_check_override
						fi
					done
				done
			fi

			if [ ! -s $TMP_DIR/cc_list_check_override ]
			then
				echo $D >> $PREVIOUS_DIR/cc_list_dates_OKK
			fi	
		fi
	done
fi

# make cc_list_dates_OKK unique
if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ]
then
	cat $PREVIOUS_DIR/cc_list_dates_OKK | sort -u > $PREVIOUS_DIR/cc_list_dates_OKK1
	mv $PREVIOUS_DIR/cc_list_dates_OKK1 $PREVIOUS_DIR/cc_list_dates_OKK
fi

if [ -s $RESULTS ]
then
	mv $RESULTS ${RESULTS}_raw $PREVIOUS_DIR/
fi

mv $OUT $OUT_LOST $REPLACE $PREVIOUS_DIR/

case $CAL4CAL in
 "NO"  ) echo "- \$DFO_MON_DIR/CALCHECK/PREVIOUS/cc_list_dates_OKK for next incremental run prepared." ;;
 "YES" ) echo "- \$DFO_MON_DIR/CALCHECK/CAL4CAL/PREVIOUS/cc_list_dates_OKK for next incremental run prepared." ;;
esac

if [ -s $TMP_DIR/cc_list_comments ]
then
	mv $TMP_DIR/cc_list_comments $PREVIOUS_DIR/
fi

# ======================================================================
# 5.8 find issues (FLAG_ISSUE, edit MARK_SCIENCE and MARK_CAL4CAL)
# ======================================================================

if [ $FORCED_MODE = YES ]
then
	writeLog "12. Determine cal_scores ..."
	scpLog
fi

echo "- determine cal_score (in $ORIG_TMP_DIR/CALCHECK/calScores) ..."

MARK_OKK="<img src=${IMG_URL1}/cal_score_green.png width=10 height=10 title=\"all calibrations existing and within validity time range\"> "
MARK_NOK="<img src=${IMG_URL1}/cal_score_red.png width=12 height=14 title=\"issue with outdated or missing calibrations\"> "
MARK_UNC="<img src=${IMG_URL1}/open.gif width=10 height=12 title=\"issue with outdated or missing calibrations but daytime calibs not yet finished\"> "

# if cc_list_dates_OKK is complete, all is ok
FLAG_ISSUE="none"

if [ -s $PREVIOUS_DIR/cc_list_dates_OKK ]
then
	rm -f $TMP_DIR/cc_diff
	diff $PREVIOUS_DIR/cc_list_dates_OKK $TMP_DIR/cc_list_dates4 | egrep "<|>" | awk '{print $2}' > $TMP_DIR/cc_diff

# no diff --> no issue; diff? --> investigate further 
	if [ -s $TMP_DIR/cc_diff ]
	then
		FLAG_ISSUE1=`cat $TMP_DIR/cc_diff | grep -v $TODAY | head -1`
		FLAG_ISSUE2=`cat $TMP_DIR/cc_diff | grep $TODAY`
	fi

# no cc_list_dates_OKK but cc_list_dates4: issue every day
elif [ -s $TMP_DIR/cc_list_dates4 ]
then
	FLAG_ISSUE1=`cat $TMP_DIR/cc_list_dates4 | grep -v $TODAY | head -1`
	FLAG_ISSUE2=`cat $TMP_DIR/cc_list_dates4 | grep $TODAY`
fi

# if cc_list_open_issues exists, there is an issue by definition
if [ -s $TMP_DIR/cc_list_open_issues ]
then
	FLAG_ISSUE1=`cat $TMP_DIR/cc_list_open_issues | grep -v $TODAY | head -1`
fi

# diff with date < $TODAY --> issue
if [ Q$FLAG_ISSUE1 != Q ]
then
	FLAG_ISSUE=$FLAG_ISSUE1
# diff for $TODAY && day calibs finished --> issue
elif [ Q$FLAG_ISSUE2 != Q ] && [ Q$DAYCAL = Q2 ] 
then
	FLAG_ISSUE=$FLAG_ISSUE2	
# diff for $TODAY only but daycalibs not yet finished --> probably an issue
elif [ Q$FLAG_ISSUE2 != Q ] && [ Q$DAYCAL != Q2 ]
then
	FLAG_ISSUE="probably"
fi

# evaluate checks: if $FLAG_ISSUE not empty, mark issue
if [ Q$FLAG_ISSUE != Q ] && [ Q$FLAG_ISSUE != Qnone ] && [ Q$FLAG_ISSUE != Qprobably ]
then
	CAL_SCORE=NOK
	MARK_SCORE=$MARK_NOK
elif [ Q$FLAG_ISSUE = Qprobably ]
then
# unclear
	CAL_SCORE=UNC
	MARK_SCORE=$MARK_UNC
else
	CAL_SCORE=OKK
	MARK_SCORE=$MARK_OKK
fi
		
if [ -s $ORIG_TMP_DIR/CALCHECK/calScores ]
then
# initialization
	CHECK_SCIENCE=`grep SCIENCE $ORIG_TMP_DIR/CALCHECK/calScores`
	if [ "Q$CHECK_SCIENCE" = Q ] && [ $CAL4CAL = NO ]
	then
		echo "SCIENCE $CAL_SCORE" >> $ORIG_TMP_DIR/CALCHECK/calScores
	fi

	CHECK_CAL4CAL=`grep CAL4CAL $ORIG_TMP_DIR/CALCHECK/calScores`
	if [ "Q$CHECK_CAL4CAL" = Q ] && [ $CAL4CAL = YES ]
	then
		echo "CAL4CAL $CAL_SCORE" >> $ORIG_TMP_DIR/CALCHECK/calScores
	fi

	case $CAL4CAL in
	 "YES" ) sed -i -e "/CAL4CAL/s|^.*|CAL4CAL $CAL_SCORE|" $ORIG_TMP_DIR/CALCHECK/calScores ;;
	 "NO"  ) sed -i -e "/SCIENCE/s|^.*|SCIENCE $CAL_SCORE|" $ORIG_TMP_DIR/CALCHECK/calScores ;;
	esac
else
	case $CAL4CAL in
	 "YES" ) echo "CAL4CAL $CAL_SCORE" >> $ORIG_TMP_DIR/CALCHECK/calScores ;;
	 "NO"  ) echo "SCIENCE $CAL_SCORE" >> $ORIG_TMP_DIR/CALCHECK/calScores ;;
	esac
fi

# insert the cal_score
# for CAL4CAL: we insert the SCIENCE cal_score here, although it's still the old one,
# and update it later 
if [ $CAL4CAL = YES ]
then
	sed -i -e "s|MARK_CAL4CAL|${MARK_SCORE}|" $HTML
	sed -i -e "s|MARK_CAL4CAL|${MARK_SCORE}|" $TMP_DIR/calAnalysis.php

# read the other cal_score
	CAL_SCORE=`grep "SCIENCE" $TMP_DIR/../calScores | awk '{print $2}' | tail -1`
	if [ Q$CAL_SCORE = Q ]
	then
		CAL_SCORE=OKK
	fi

	case $CAL_SCORE in
	"NOK" )	MARK_SCORE=$MARK_NOK ;;
	"UNC" )	MARK_SCORE=$MARK_UNC ;;
	"OKK" )	MARK_SCORE=$MARK_OKK ;;
	esac	
	sed -i -e "s|MARK_SCIENCE|${MARK_SCORE}|" $HTML
	sed -i -e "s|MARK_SCIENCE|${MARK_SCORE}|" $TMP_DIR/calAnalysis.php
fi

if [ $CAL4CAL = NO ]
then
	sed -i -e "s|MARK_SCIENCE|${MARK_SCORE}|" $HTML
	sed -i -e "s|MARK_SCIENCE|${MARK_SCORE}|" $TMP_DIR/calAnalysis.php

# read the other cal_score
	CAL_SCORE=`grep "CAL4CAL" $TMP_DIR/calScores | awk '{print $2}' | tail -1`
	if [ Q$CAL_SCORE = Q ]
	then
		CAL_SCORE=OKK
	fi

	case $CAL_SCORE in
	"NOK" )	MARK_SCORE=$MARK_NOK ;;
	"UNC" )	MARK_SCORE=$MARK_UNC ;;
	"OKK" )	MARK_SCORE=$MARK_OKK ;;
	esac	
	sed -i -e "s|MARK_CAL4CAL|${MARK_SCORE}|" $HTML
	sed -i -e "s|MARK_CAL4CAL|${MARK_SCORE}|" $TMP_DIR/calAnalysis.php
fi

# ======================================================================
# 6. transfer to external web site (via $DFO_WEB_SERVER)
#    and MIRROR web site (main page only)
# ======================================================================

if [ $SCP = YES ]
then
	if [ $FORCED_MODE = YES ]
	then
		writeLog "13. Upload results ..."
		scpLog
	fi

	sed -e "s|${REPORT_DIR}|${REPORT_DIR2}|g" $HTML |\
	 sed -e "s|${CALREPORT_DIR}|${CALREPORT_DIR2}|g" |\
	 sed -e "/<!--CC_POSTIT-->/s|^.*|        <!--#include virtual=\"/observing/dfo/quality/ALL/CC_POSTIT\"-->|" |\
	 sed -e "/<!--CAL_POSTIT-->/s|^.*|        <!--#include virtual=\"/observing/dfo/quality/${DFO_INSTRUMENT}/reports/CAL/CAL_POSTIT\"-->|" |\
	 sed -e "/<!--CAL4CAL_NAV-->/s|calChecker.html|calChecker_${DFO_INSTRUMENT}.html|"g |\
	 sed -e "/<!--SCIENCE_NAV-->/s|calChecker.html|calChecker_${DFO_INSTRUMENT}.html|"g |\
	 grep -v "<\!\-\-FILTER\-\->" |\
	 sed "s/<!--LOCAL-->//"g |\
	 sed "s/<!--QCWEB-->//"g \
	 > $TMP_DIR/calChecker_${DFO_INSTRUMENT}.html

# scp main page, CAL_TECH.html, rawtypes
# CAL pages go to reports/CAL while CAL4CAL pages go to reports/CAL/CAL4CAL
	echo "- scp to $DFO_WEB_SERVER (public web site $PUB_QC_WEB_SERVER_URL) ..."

# make executable for the virtual include of the vnavbar; scp -p --> preserve mode
	chmod u+x $TMP_DIR/calChecker_${DFO_INSTRUMENT}.html
	scp -o BatchMode=yes -p $TMP_DIR/calChecker_${DFO_INSTRUMENT}.html $DFO_MON_DIR/$RAWTYPES ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR} 1>/dev/null

# to $MIRROR_WEB_SERVER: only the real dynamic pages; all others will be replicated anyway
# edit www.eso.org to www.pl.eso.org except for php links and gnlt link which remain at www.eso.org
	echo "- scp to $MIRROR_WEB_SERVER (mirror, background)"
	sed "s|www.eso.org|www.pl.eso.org|g" $TMP_DIR/calChecker_${DFO_INSTRUMENT}.html |\
 sed "/SRV/s|www.pl.eso.org|www.eso.org|g" |\
 sed "/calForcedRefresh.php/s|www.pl.eso.org|www.eso.org|g" |\
 sed "/calAnalysis.php/s|www.pl.eso.org|www.eso.org|g" |\
 sed "/calBadQual.php/s|www.pl.eso.org|www.eso.org|g" |\
 sed "/gnlt/s|www.pl.eso.org|www.eso.org|g" \
 > $TMP_DIR/calChecker_${DFO_INSTRUMENT}.html_mirror
	chmod u+x $TMP_DIR/calChecker_${DFO_INSTRUMENT}.html_mirror
	scp -q -o BatchMode=yes -p $TMP_DIR/calChecker_${DFO_INSTRUMENT}.html_mirror ${MIRROR_WEB_SERVER}:${MIRROR_WEB_DIR}/${TARG_DIR}/calChecker_${DFO_INSTRUMENT}.html 2>/dev/null &

# update and transfer the science flag for the CAL4CAL page
	if [ $CHECK_C4C = YES ] && [ $CAL4CAL = NO ]
	then
		CAL_SCORE=`grep "SCIENCE" $TMP_DIR/calScores | awk '{print $2}' | tail -1`
		case $CAL_SCORE in
		"NOK" )	MARK_SCORE=$MARK_NOK ;;
		"UNC" )	MARK_SCORE=$MARK_UNC ;;
		"OKK" )	MARK_SCORE=$MARK_OKK ;;
		esac	

# download calChecker_${DFO_INSTRUMENT}.html for CAL4CAL
		ORIG_PATH=$PWD
		cd $TMP_DIR/CAL4CAL
		scp -o BatchMode=yes -p ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}/CAL4CAL/calChecker_${DFO_INSTRUMENT}.html ./

# replace old MARK_SCORE by  new one
		sed -i -e "/SCIENCE_NAV/s|^.*|        <td width=3% align=center bgcolor=#FFD5B5 nowrap><b><font size=1>\&nbsp;${MARK_SCORE} <a href=\"../calChecker_${DFO_INSTRUMENT}.html\" title=\"calibration completeness for science\"><font color=#333333>science</font></a>\&nbsp;</font></b></td> <!--SCIENCE_NAV-->|" calChecker_${DFO_INSTRUMENT}.html

# upload modified page again 
		scp -o BatchMode=yes -p calChecker_${DFO_INSTRUMENT}.html ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}/CAL4CAL/
		sed "s|www.eso.org|www.pl.eso.org|g" $TMP_DIR/calChecker_${DFO_INSTRUMENT}.html |\
 sed "/calForcedRefresh.php/s|www.pl.eso.org|www.eso.org|g" |\
 sed "/calAnalysis.php/s|www.pl.eso.org|www.eso.org|g" |\
 sed "/calBadQual.php/s|www.pl.eso.org|www.eso.org|g" |\
 sed "/gnlt/s|www.pl.eso.org|www.eso.org|g" \
 > calChecker_${DFO_INSTRUMENT}.html calChecker_${DFO_INSTRUMENT}.html_mirror
		scp -q -o BatchMode=yes -p calChecker_${DFO_INSTRUMENT}.html_mirror ${MIRROR_WEB_SERVER}:${MIRROR_WEB_DIR}/${TARG_DIR}/CAL4CAL/calChecker_${DFO_INSTRUMENT}.html 2>/dev/null &

# upload calAnalysis.php with embedded scores
		scp -o BatchMode=yes $TMP_DIR/calAnalysis.php         ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}/php 1>/dev/null
		scp -o BatchMode=yes $TMP_DIR/CAL4CAL/calAnalysis.php ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}/CAL4CAL/php 1>/dev/null
		cd $ORIG_PATH
# BWo 2015-03-05: make sure to scp calAnalysis.php
# for instruments without configured cal4cal
	elif [ $CHECK_C4C = NO ] && [ $CAL4CAL = NO ]
	then
		scp -o BatchMode=yes $TMP_DIR/calAnalysis.php         ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}/php 1>/dev/null
	fi

# determine total cal_score and upload: only for SCIENCE, not for CAL4CAL
	if [ $CAL4CAL = NO ]
	then
		CAL_SCORE=`cat $TMP_DIR/calScores | awk '{print $2}' | grep NOK | head -1`
		if [ Q$CAL_SCORE != Q ]
		then
			TOT_CAL_SCORE=NOK
		else
			CAL_SCORE=`cat $TMP_DIR/calScores | awk '{print $2}' | grep UNC | head -1`
			if [ Q$CAL_SCORE != Q ]
			then
				TOT_CAL_SCORE=UNC
			else
				TOT_CAL_SCORE=OKK
			fi
		fi

		case $TOT_CAL_SCORE in
		"NOK" )	MARK_SCORE=$MARK_NOK ;;
		"UNC" )	MARK_SCORE=$MARK_UNC ;;
		"OKK" )	MARK_SCORE=$MARK_OKK ;;
		esac

		echo $MARK_SCORE > $TMP_DIR/cal_score.html
		scp    -o BatchMode=yes    $TMP_DIR/cal_score.html ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}
		scp -q -o BatchMode=yes -p $TMP_DIR/cal_score.html ${MIRROR_WEB_SERVER}:${MIRROR_WEB_DIR}/${TARG_DIR} 2>/dev/null &

# upload last_update info
		jCheckHealth $UPDATE 1
		scp    -o BatchMode=yes    $TMP_DIR/cal_checkhealth ${DFO_WEB_SERVER}:${UPDATE_DIR}/cal_checkhealth_${DFO_INSTRUMENT}
		scp -q -o BatchMode=yes -p $TMP_DIR/cal_checkhealth ${MIRROR_WEB_SERVER}:${MIRROR_UPDATE_DIR}/cal_checkhealth_${DFO_INSTRUMENT} 2>/dev/null &

# upload cal_blink
		if [ -s $TMP_DIR/list_blink ] || [ -s $TMP_DIR/CAL4CAL/list_blink ]
		then
			echo "<font size=1 color=#FF0000><i><blink>[not yet analyzed]</blink></i></font>" > $TMP_DIR/cal_blink
		else
			echo "<font size=1>&nbsp;</font>" > $TMP_DIR/cal_blink
		fi
		scp -o BatchMode=yes $TMP_DIR/cal_blink ${DFO_WEB_SERVER}:${UPDATE_DIR}/cal_blink_${DFO_INSTRUMENT}
	fi		
		
# CAL_TECH.html: to ${DFO_WEB_SERVER} but not to ${MIRROR_WEB_SERVER} (replicated!)
	if [ -s $DFO_MON_DIR/CAL_TECH.html ]
	then
		scp -o BatchMode=yes $DFO_MON_DIR/CAL_TECH.html ${DFO_WEB_SERVER}:${WEB_DIR}/${TARG_DIR}
	fi

# products.html goes to HC site for trendPlotter (virtual include)
	if [ $CAL4CAL = NO ]
	then
        	ssh -o BatchMode=yes ${DFO_WEB_SERVER} "bin/qcDircheck ${WEB_DIR}/HEALTH" 1>/dev/null
		scp -o BatchMode=yes $PRODUCTS_TABLE ${DFO_WEB_SERVER}:${WEB_DIR}/HEALTH
		scp -o BatchMode=yes $PRODUCTS_TABLE ${MIRROR_WEB_SERVER}:${MIRROR_WEB_DIR}/HEALTH 2>/dev/null &
	fi

else
	echo "- no scp to $DFO_WEB_SERVER (${PUB_QC_WEB_SERVER_URL}), results are kept local."
fi

# create dummy page for tomorrow
createTomorrow

# scp to $DFO_WEB_SERVER
if [ $SCP = YES ]
then
	echo "- scp to $DFO_WEB_SERVER (QC web site $QC_WEB_SERVER_URL ...) "
	echo ""
# make sure target directory exists
	ssh -o BatchMode=yes $DFO_WEB_SERVER "${QCBIN_DIR}/webDircheck ${INSTR_DIR}"
     	ssh -o BatchMode=yes $DFO_WEB_SERVER "${QCBIN_DIR}/webDircheck ${INSTR_DIR}/${QCWEB_TARG_DIR}"

# scp $RAWTYPES
	scp -o BatchMode=yes $DFO_MON_DIR/${RAWTYPES} ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}

# scp edited setup result page
	if [ -s ${RESULTS}_setup.html ]
	then
		sed -e "s|${REPORT_DIR}|${REPORT_DIR2}|g" $DFO_MON_DIR/calResults_${DFO_INSTRUMENT}_setup.html |\
	 sed "s/<a href=.*target=_blank>//" |\
	 sed "s/<!--FILTER-->//"g |\
	 sed "s/<!--LOCAL-->//"g \
	 > $TMP_DIR/calResults_${DFO_INSTRUMENT}_setup.html
		if [ -s $DFO_MON_DIR/calResults_${DFO_INSTRUMENT}_setup_all.html ]
		then
			sed -e "s|${REPORT_DIR}|${REPORT_DIR2}|g" $DFO_MON_DIR/calResults_${DFO_INSTRUMENT}_setup_all.html |\
	 sed "s/<a href=.*target=_blank>//" |\
	 sed "s/<!--FILTER-->//"g |\
	 sed "s/<!--LOCAL-->//"g \
	 > $TMP_DIR/calResults_${DFO_INSTRUMENT}_setup_all.html
		fi
		scp -o BatchMode=yes $TMP_DIR/calResults_${DFO_INSTRUMENT}_setup*.html ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}
	fi

# edit result pages per DATE
	rm -f $TMP_DIR/calResults_${DFO_INSTRUMENT}_*.html
	for D in `cat $TMP_DIR/cc_list_dates4`
	do
		if [ -s ${CALREPORT_DIR}/calResults_${DFO_INSTRUMENT}_$D.html ]
		then
			sed -e "s|${REPORT_DIR}|${REPORT_DIR2}|g" ${CALREPORT_DIR}/calResults_${DFO_INSTRUMENT}_$D.html |\
 sed "s/<a href=.*target=_blank>//" |\
 grep -v "<!\-\-FILTER\-\->" > $TMP_DIR/calResults_${DFO_INSTRUMENT}_$D.html
		fi
	done

# scp edited result pages 
	scp -r -o BatchMode=yes $TMP_DIR/calResults_${DFO_INSTRUMENT}_*.html ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}
	scp -r -o BatchMode=yes $TMP_DIR/calChecker_tomorrow.html            ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}/calResults_${DFO_INSTRUMENT}_${TOMORROW}.html
else
	echo "- no scp to $DFO_WEB_SERVER ($QC_WEB_SERVER_URL)."
fi

# ======================================================================
# 7. End
# 7.1 cleanup and save results for incremental call
# ======================================================================

# edit LOCAL version (to make links to external server work)
grep -v "<\!\-\-LOCAL\-\->" $HTML |\
 sed "s|src=\"/observing/dfo/quality|src=\"http://www.eso.org/observing/dfo/quality|"g |\
 sed "s|src=/observing/dfo/quality|src=http://www.eso.org/observing/dfo/quality|"g |\
 sed "s|openWin('/observing/dfo/quality|openWin('http://www.eso.org/observing/dfo/quality|"g |\
 sed "/Health Check Monitor for/s|href=\"|&http://www.eso.org|" |\
 sed "/<!--VNAVBAR-->/s|$TITLE_COLOR|#CCCCCC|" |\
 sed "/RAWDISP/s|href=|&http://www.eso.org|" \
 > $TMP_DIR/html1

mv $TMP_DIR/html1 $HTML
cp $HTML $DFO_MON_DIR/calChecker.html

if [ -s $TMP_DIR/cc_list_override ]
then
	mv $TMP_DIR/cc_list_override $PREVIOUS_DIR/
fi

# manage alog files (for local HTML pages): save all alog files from $DFO_AB_DIR to PREVIOUS
CHECK_EXIST=`ls $DFO_AB_DIR | grep .alog | head -1`
CHECK_EXIST2=`ls $PREVIOUS_DIR | grep .alog | head -1`
if [ Q$CHECK_EXIST2 != Q ]
then
        rm $PREVIOUS_DIR/*alog
fi

if [ Q$CHECK_EXIST != Q ]
then
	cp $DFO_AB_DIR/*alog $PREVIOUS_DIR
fi

# remove outdated alog and ab files from $DFO_AB_DIR
# outdated: neither from this nor from previous month
THIS_MONTH=`date -u +%Y-%m`
PREV_MONTH=`qcdate ${THIS_MONTH}-01 -1 | cut -c 1-7`
CHECK_EXIST=`ls $DFO_AB_DIR | grep .alog | egrep -v "${THIS_MONTH}|${PREV_MONTH}" | head -1`

if [ Q$CHECK_EXIST != Q ]
then
        rm `ls $DFO_AB_DIR/*alog | egrep -v "${THIS_MONTH}|${PREV_MONTH}"`
        rm `ls $DFO_AB_DIR/*ab   | egrep -v "${THIS_MONTH}|${PREV_MONTH}"`
fi

# clean-up
rm -f $DFO_MON_DIR/DFO_STATUS  $DFO_MON_DIR/AB_list* $DFO_MON_DIR/*.html1
rm -f ${RESULTS}1 ${RESULTS}2 $TMP_DIR/cc_counterlog

# ======================================================================
# 7.2 'finish' message
# ======================================================================

TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
echo "[$TIMESTAMP] calChecker finished." >> $LOGFILE

case "$CAL4CAL" in
 "NO"  ) echo "calChecker finished: `date -u`" ;;
 "YES" ) echo "calChecker CAL4CAL finished: `date -u`" ; exit ;;
esac

echo "
Find 
- result in \$DFO_MON_DIR/CALCHECK/calChecker_${DFO_INSTRUMENT}.html ,
- log file for createReport in $ORIG_TMP_DIR/CALCHECK/out_CreateReport ,"

if [ $SCP = YES ]
then
	echo "- log file in \$DFO_MON_DIR/CALCHECK/calChecker.log ,"
	echo "- exported files on ${QC_WEB_SERVER_URL}/$DFO_INSTRUMENT/CALCHECK and "
	echo "                    ${PUB_QC_WEB_SERVER_URL}/$DFO_INSTRUMENT/reports/CAL."
else
	echo "- log file in \$DFO_MON_DIR/CALCHECK/calChecker.log."
fi

if [ $FORCED_MODE = YES ]
then
	writeLog "14. Finished"
	echo "-------------------------------------------------------------------" >> $FORCED_LOG
	scpLog
fi

# ======================================================================
# 7.3 compute the calChecker execution time [MNe]
# ======================================================================

# extract exectime
START_TIME=`grep "calChecker started" $LOGFILE | awk '{print $1}' | sed "s/:$//" | sed "s/\[//" | sed "s/\]//"`
END_TIME=`grep  "calChecker finished" $LOGFILE | awk '{print $1}' | sed "s/:$//" | sed "s/\[//" | sed "s/\]//"`
if [ "Q$START_TIME" = "Q" ] || [ "Q$END_TIME" = "Q" ]
then
	TEXEC=0
else
        START_MIN=`echo $START_TIME | sed "s/:/ /g" | sed "s/T/ /" | awk '{print $2*60.+$3+$4/60.}'`
        DATE1=`echo $START_TIME | sed "s/T/ /" | awk '{print $1}'`

        END_MIN=`echo $END_TIME | sed "s/:/ /g" | sed "s/T/ /" | awk '{print $2*60.+$3+$4/60.}'`
        DATE2=`echo $END_TIME | sed "s/T/ /" | awk '{print $1}'`

	if [ $DATE1 = $DATE2 ]
        then
                TEXEC=`echo $END_MIN $START_MIN | awk '{print $1-$2}'`
        else
                TEXEC=`echo $END_MIN $START_MIN | awk '{print $1+1440-$2}'`
        fi
fi
TEXEC=`echo $TEXEC | awk '{ printf"%3.2f\n",$1}'`

# define a number of further parameters and ingest all into the QC1 exec_time table 
CIVIL_DATE=`echo $DATE1 | sed "s/-//g"`
LOCAL_TIME=`echo $START_TIME | sed "s/-/ /g" | sed "s/T/ /g" | awk '{print $4}'`

echo "qc1Ingest -table exec_time -civil_date $CIVIL_DATE -local_time $LOCAL_TIME -mjd_obs $TODAY_MJD2 -exec_time $TEXEC -version $TOOL_VERSION -instrument $DFO_INSTRUMENT -hostname $DFO_MACHINE -tool $TOOL_NAME -process_load $PROCESS_LOAD " >> $DFO_TREND_DIR/dfo_exec_time

qc1Ingest -table exec_time -civil_date $CIVIL_DATE -local_time $LOCAL_TIME -mjd_obs $TODAY_MJD2 -exec_time $TEXEC -version $TOOL_VERSION -instrument $DFO_INSTRUMENT -hostname $DFO_MACHINE -tool $TOOL_NAME -process_load $PROCESS_LOAD | grep -v "Connecting to" | grep -v "duplicated rows" | sed "s/Successfully.*/- execution time: $TEXEC min (ingested into exec_time)/" | tee -a $LOGFILE

# create a TEXEC score and export it 
TEXEC1=`echo $TEXEC | awk '{ if ($1 >= 10) {print 100} else {print 1}}'`
if [ $TEXEC1 -gt 10 ]
then
	echo "<font size=1 color=red>$TEXEC</font>" > $TMP_DIR/cal_texec
else
	echo "<font size=1>$TEXEC</font>" > $TMP_DIR/cal_texec
fi

scp -o BatchMode=yes $TMP_DIR/cal_texec ${DFO_WEB_SERVER}:${UPDATE_DIR}/cal_texec_${DFO_INSTRUMENT} 1>/dev/null

# ======================================================================
# 7.4 Check for outdated histo pages (FULL mode only)
# ======================================================================

if [ $INCREM = NO ]
then
	YY=`date -u +%Y`
# older than 2 yrs: dynamic
	YY2=`echo $YY | awk '{print $1-2}'`
	ssh $DFO_WEB_SERVER "rm -f ${INSTR_DIR}/CALCHECK/calResults_${DFO_INSTRUMENT}_${YY2}*"
fi

exit 0
