#!/bin/sh
# PURPOSE:	tellTracker - checks each SCIENCE file that requires a TELLURIC calibration
# AUTHOR:	Reinhard Hanuschik / ESO
# VERSIONS:	1.0  -- 2011-03-14 released
# 		1.0.1 - provide undated output on qcweb; measure load; improve top text (2011-03-16)
#		1.0.2 - bug fix for "non recoverable" (2011-03-22)
#		1.1  -- added: detailed result page; column toggle added; DATE removed (2011-03-25)
#		1.1.1 - navigation tags improved, default page (2011-04-01)
#		1.2  -- getObInfo used for OB grades and comments; calChecker logo added (2012-10-17)
#		1.2.1 - suppress message from getObInfo (2012-10-18)
#		1.2.2 - mark issue in tab; TWILIGHT calculation turned off (2013-10-10)
#		1.2.3 - bug fix for issue marker (2013-10-17)
#		1.3 --  telluric STD types can be associated to different science raw_types (2013-11-08) [WHu]
#		1.4 --  dataclient replaced by HdrDownloader (2014-05-05)
#		1.4.1- 	optional config file for HdrDownloader recognized (2014-05-27)
#		1.4.2-  standard dfs call of HdrDownloader; cfg file terminated (2015-01-13)
#		1.4.3-	fixed bug with not propagated red flag in MISS case (line 1400) (2015-08-31) [BWo]
#		1.5 --  extra HDR directory for tellTracker: DFO_HDR_DIR/TELLTRACK; enable complete header scans while avoiding incomplete ABs (2016-11-16)
#		1.5.1-	prepared for P100 (2017-03-21)
#		1.5.2-	no sourcing of .qcrc in :224 (2018-06-16)
#		1.5.3-  updated getFlag to support both old and new (P105+) runID format (2020-02-07)
#
# CONFIG:	config.tellTracker; tellTracker.cfg (for header keys); tellTracker_radec.cfg (RA, DEC, LST)
#		info.tellTracker in $DFO_CONFIG_DIR: information text like for calChecker
# TOOLS:	getObInfo for the comments
# OUTPUT:	exported HTML page per $DATE under http://qcweb.hq.eso.org/<ins>/CALCHECK/TELLTRACK
# COMMENTS:	this is an optional dfos tool; its output is linked to calChecker; it is required only for
#		tracking of telluric standard stars 
# 		The HTML table is enhanced with tablesorter (column sorting) and dataTable (filtering)
# 		tablesorter: java code comes from http://tablesorter.com/docs. Find more in 'getStatusAB'.
#		dataTable: java code comes from http://www.datatables.net. Find both installations under
#		$JSCRIPT_URL.
# ======================================================================
TOOL_VERSION="1.5.3"
TOOL_NAME="tellTracker"

# =========================================================================
# 0. Initialize
# 0.1 some hard-coded settings
# =========================================================================

source $HOME/.dfosrc 	#useful if called from cronjob

# hardcoded tool configuration
# for HdrDownloader wrapper (see calChecker for more information)
SLEEP=5        	#check time: every SLEEP sec we check again up to $SLEEP_MAX
SLEEP_MAX=30   	#max waiting time: if HdrDownloader takes longer than that, we kill it
	       	#this is shorter than in calChecker

AUTO_REFRESH=300 #time in secs after which HTML page refreshes

# colors, cloned from calChecker
GRN="#33FF33"   # green
LGRN="#99FF99"  # light-green

YLW="#FFFF00"   # yellow
LYLW="#FFFF99"  # light-yellow

RED="#FF6666"   # red
LRED="#FF9999"  # light-red

HDRSORT_COLOR="#8dbdd8" #lightblue
TELL_COLOR="#CC9999"	#lightorange
REPORT_COLOR="#FF6600"  #orange

SM_COLOR="#006A9D"  #eso-blue:  color for SM nights
VM_COLOR="#666666"  #dark_grey: color for VM nights

# web and URLs
INSTR_DIR="/home/qc/public_html/${DFO_INSTRUMENT}" # on $DFO_WEB_SERVER
QCWEB_TARG_DIR="CALCHECK/TELLTRACK"		   # subdir for tellTracker results

IMG_URL="http://www.eso.org/qc/ALL/img" #URL for images
QC_URL="http://www.eso.org/qc/ALL"      #URL for CALCHECK help site
JSCRIPT_URL="http://www.eso.org/observing/dfo/quality/ALL/jscript"       #URL for java scripts
#SKYCALC_URL="http://www.eso.org/sci/observing/bin/skycalcw?cmd=a&site=v" #URL for SKYCALC: unsupported, unstable
CALCHECK_URL="http://www.eso.org/qc/${DFO_INSTRUMENT}/reports/CAL"
QCWEB_URL="http://qcweb.hq.eso.org/${DFO_INSTRUMENT}"

STYLES="${QC_URL}/common/qc_eso.style"

# =========================================================================
# 0.2 usage; read options and parameters; kill older instances
# =========================================================================

cat > $TMP_DIR/tt_usage <<EOT
tellTracker: plugin tool for calChecker, checking status of telluric and science data
Calls:
tellTracker 		 update current date, with header download (this is the standard call to be used in a cronjob)
tellTracker -N		 update current date, no header download
tellTracker -d <date> -O specify date, with header download
tellTracker -d <date>	 specify date, no header download
tellTracker -V		 verbose output
tellTracker -v		 version and exit
tellTracker -h		 help and exit
EOT

VERBOSE=NO
NEW_HDR=YES		#for $TODAY
OLD_HDR_UPDATE=NO	#for older dates: header update usually not required

while getopts hvNOVd: OPTION
do
	case "$OPTION" in
         v ) echo $TOOL_VERSION
             exit 0 ;;
         h ) cat $TMP_DIR/tt_usage 
             exit 0 ;;
	 d ) DATE=$OPTARG ;;
	 N ) NEW_HDR=NO ;;
	 O ) OLD_HDR_UPDATE=YES ;;
	 V ) VERBOSE=YES ;;
	 ? ) echo $TOOL_VERSION; exit 0;;
	esac
done

# check for currently running versions and kill all but latest
# (we call this tool with high frequency, it should take less than a minute)
rm -f $TMP_DIR/tt_all_pids*
ps -wfC tellTracker | grep -v CMD | sort -k5,5 > $TMP_DIR/tt_all_pids

LAST_PID=`cat $TMP_DIR/tt_all_pids | tail -1 | awk '{print $2}'`
grep -v " $LAST_PID " $TMP_DIR/tt_all_pids > $TMP_DIR/tt_all_pids1

# turned off: occasionally forked instances occur, and tool commits suicide; need better solution!
rm -f $TMP_DIR/tt_all_pids1
#

if [ -s $TMP_DIR/tt_all_pids1 ]
then
	for P in `cat $TMP_DIR/tt_all_pids1 | awk '{print $2}'`
	do
		kill $P
	done
fi

# to measure execution time and load (to correlate long exec_times with load)
START_TIME=`date -u +%Y-%m-%d"T"%H:%M:%S`
LOAD=`uptime | awk '{print $10}' | sed "s/,//"`

# RANGE: validity range as configured
RANGE=`grep "^RANGE" 	$DFO_CONFIG_DIR/config.tellTracker | awk '{print $2}'`
DRANGE=`echo $RANGE | awk '{print $1/60/24}'`
DRANGE1=`echo $DRANGE | awk '{printf"%5.3f\n",$1}'`
DRANGE2=`echo $RANGE | awk '{printf"%5.1f\n",$1/60}'`

D_AIRM=`grep "^D_AIRM" 	$DFO_CONFIG_DIR/config.tellTracker | awk '{print $2}'`

TODAY=`date -u +%Y-%m-%d`
TODAY_DFO=$TODAY
TODAY_HH=`date -u +%H`
TODAY_MM=`date -u +%M`
TODAY_MJD=`qcdate $TODAY`
TURNOVER=`echo $DFO_OFFSET | awk '{print $1+12}'`

if [ $TODAY_HH -lt $TURNOVER ]
then
	TODAY_DFO=`qcdate $TODAY -1`
fi

if [ Q$DATE = Q ]
then
	DATE=$TODAY_DFO
fi

TOMORROW=`qcdate $TODAY_DFO +1`

# correct MJD for exec_time
TODAY_MJD2=`echo $TODAY_MJD $TODAY_HH $TODAY_MM | awk '{printf"%10.6f\n", $1+($2+$3/60)/24}'`

CHECK_C4C=`grep "^CHECK_C4C" $DFO_CONFIG_DIR/CALCHECK/config.calChecker | awk '{print $2}'`
if [ Q$CHECK_C4C != QYES ]
then
	CHECK_C4C=NO
fi

# =====================================================================================================
# 0.3 procedures
# 0.3.1 getObGrades: obsolete, replaced by getObInfo
# =====================================================================================================

# =========================================================================
# 0.3.2 getFlag: Querying sched_rep for SM_VM_FLAG (from createAB)
#	old format: 	    sched_rep
#	new format (P105+): v_sched_rep (view)
# =========================================================================
getFlag(){
cat > $TMP_DIR/tt_flag_query <<EOT
SELECT
        period ,
        progid ,
        obs_mode
FROM
        opc70..sched_rep
WHERE
        progid in (
EOT

rm -f $TMP_DIR/tt_list_progids_science
cat $TMP_DIR/tt_list_all | awk '{print $8}' | sort -u | egrep -v "^60\.|^060\."  > $TMP_DIR/tt_list_progids_science

cat $TMP_DIR/tt_list_progids_science |\
 sed "s/^.*/\"&\" ,/" |\
 sed "$ s/,//" >> $TMP_DIR/tt_flag_query

cat >> $TMP_DIR/tt_flag_query <<EOT
        )
ORDER BY
        period desc
go
EOT

# new format
cat $TMP_DIR/tt_flag_query | sed "s/sched_rep/v_&/" > $TMP_DIR/tt_flag_query_new

# both queries together should always deliver a result
rm -f $TMP_DIR/tt_out_flagquery
isql -Uqc -S${ARCH_SERVER} -P`cat ${ARCH_PWD}` -w 999 -i $TMP_DIR/tt_flag_query |\
 sed "1,2 d" |\
 grep -v affected |\
 sed "/^$/d" |\
 awk '{print $2,$3}' |\
 sort -u > $TMP_DIR/tt_out_flagquery

isql -Uqc -S${ARCH_SERVER} -P`cat ${ARCH_PWD}` -w 999 -i $TMP_DIR/tt_flag_query_new |\
 sed "1,2 d" |\
 grep -v affected |\
 sed "/^$/d" |\
 awk '{print $2,$3}' |\
 sort -u >> $TMP_DIR/tt_out_flagquery

cat $TMP_DIR/tt_out_flagquery | sort -u > $TMP_DIR/tt_out_flagquery1
mv $TMP_DIR/tt_out_flagquery1 $TMP_DIR/tt_out_flagquery
}

# =====================================================================================================
# 0.3.3 createWrapper: create a wrapper around HdrDownloader
# =====================================================================================================
createWrapper(){
cat > $TMP_DIR/wrappedHdrDownloader <<EOT
#!/bin/sh
export DFO_HDR_DIR=$DFO_HDR_DIR/TELLTRACK
if [ ! -d \$DFO_HDR_DIR ]
then
	mkdir \$DFO_HDR_DIR
fi

if [ ! -d \$DFO_HDR_DIR/$DATE ]
then
	rm -rf \$DFO_HDR_DIR/*
	mkdir \$DFO_HDR_DIR/$DATE
fi
	
HdrDownloader -d $DATE 1>/dev/null
PID=\$!
ALREADY_SLEPT=0
sleep 3

while [ \$ALREADY_SLEPT -lt $SLEEP_MAX ]
do
        CHECK_PID=\`ps \$PID | grep -v PID  | awk '{print \$6}'\`
        if [ Q\${CHECK_PID} != Q ]
        then
                ALREADY_SLEPT=\`echo \$ALREADY_SLEPT $SLEEP | awk '{print \$1+\$2}'\`
                TIMESTAMP=\`date -u +%Y-%m-%d"T"%H:%M:%S\`
		if [ $VERBOSE = YES ]
		then
                	echo "[\$TIMESTAMP] echo ALREADY_SLEPT: \$ALREADY_SLEPT" 
		fi
                sleep $SLEEP
        else
                break
        fi
done

CHECK_PID=\`ps \$PID | grep -v PID | awk '{print \$6}'\`
if [ Q\${CHECK_PID} != Q ]
then
        kill -9 \$PID 2>&1 1>/dev/null
        echo "TIMEOUT: HdrDownloader killed after $SLEEP_MAX secs."
fi
EOT
chmod u+x $TMP_DIR/wrappedHdrDownloader
}

# =========================================================================
# 0.3.4 createTomorrow: dummy page to avoid broken 'next' link for current date
# =========================================================================
createTomorrow(){
cat > $TMP_DIR/tellTracker_tomorrow.html <<EOT
<html>
This date is currently in the future. <p>
[<a href=tellTracker_${TODAY_DFO}.html>back</a>]
</html>
EOT
}

# =====================================================================================================
# 0.3.5 getTwilight: read skycalc for given date, calculate mjd_obs for end/begin of twilight 
#       turned off 2013 since URL unstable
# =====================================================================================================
getTwilight(){
DATE1=`echo $DATE | sed "s/-/+/g"`

rm -f $TMP_DIR/eph.txt
wget -q "${SKYCALC_URL}&sdate=${DATE1}" T 5 -t 1 -O $TMP_DIR/eph.txt 

MJD_MID=`grep "^Local midnight" $TMP_DIR/eph.txt | sed "s/^.*UT, or JD//" | awk '{printf"%5.5f\n", $1-2400000-0.5}'`

# LMST at midnight: our reference value
LMST_mid=`grep "Local Mean Sidereal Time at midnight" $TMP_DIR/eph.txt | sed "s/^.*midnight =//" | awk '{print ($1+$2/60+$3/60/60)/24}'`
LMST_mor=`grep "LMST at morning twilight" $TMP_DIR/eph.txt | sed "s/^.*LMST at morning twilight://" | awk '{print ($1+$2/60+$3/60/60)/24}'`
LMST_eve=`grep "LMST at evening twilight" $TMP_DIR/eph.txt | sed "s/^.*LMST at evening twilight://" | awk '{print ($1+$2/60+$3/60/60)/24}'`
delta1=`echo $LMST_mid $LMST_eve | awk '{print $1-$2}'`
delta2=`echo $LMST_mid $LMST_mor | awk '{print $2-$1}'`

MJD_EVE=`echo $MJD_MID $delta1 | awk '{printf"%5.5f\n",$1-$2}'`
MJD_MOR=`echo $MJD_MID $delta2 | awk '{printf"%5.5f\n",$1+$2}'`
MJD_CEN=`echo $MJD_EVE $MJD_MOR | awk '{printf"%5.5f\n",($1+$2)/2}'`
}

# =========================================================================
# 0.4 javascript to control browser refresh (from getStatusAB)
# =========================================================================
cat > $TMP_DIR/offlink <<EOT
<!-- java script for switching off/on the refresh link-->
<script type="text/javascript">
function OffLink () {
  if (document.getElementById("refresh_link").firstChild.nodeValue = "[page auto-refreshes after $AUTO_REFRESH sec]") {
      document.getElementById("refresh_link").firstChild.nodeValue = "[auto-refresh turned off, page will outdate quickly; click \'on\' to resume] ";
      document.getElementById("refresh_link").className = "off";
} else {
      document.getElementById("refresh_link").firstChild.nodeValue = "on";
      }
}
</script>

<!-- style for switching off the refresh -->
<style type="text/css">
.off  { color:red; }
</style>

EOT

# =========================================================================
# 0.5 createOutput: top part of the HTML output page, used both for "no data" 
#     and significant output
# =========================================================================
createOutput(){
TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`
if [ -d $DFO_HDR_DIR/$DATE ]
then
	LAST_HDR=`ls $DFO_HDR_DIR/$DATE | grep .hdr | tail -1 | sed "s/${DFO_FILE_NAME}.//"`
else
	LAST_HDR=""
fi
DATE_minus=`qcdate $DATE -1`
DATE_plus=`qcdate  $DATE +1`
PREV_HTMLTAG="<a href=tellTracker_${DATE_minus}.html><img src=${IMG_URL}/left.gif width=16 height=17 border=0 title=\"previous night\"></a>"
NEXT_HTMLTAG="<a href=tellTracker_${DATE_plus}.html><img src=${IMG_URL}/right.gif width=16 height=17 border=0 title=\"next night\"></a>"

cat > $TMP_DIR/output_top <<EOT
<html>
<!-- created by ${TOOL_NAME} v${TOOL_VERSION} -->
<head>
<META HTTP-EQUIV="Refresh" CONTENT="$AUTO_REFRESH">

<title>${DFO_INSTRUMENT} calChecker: telluric star tracker for science files, for date $DATE</title>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery-latest.js"></script>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery.tablesorter.js"></script>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery.dataTables.js"></script>

<script type="text/javascript">
\$(document).ready(function()
{
        \$("#issuesTableA").tablesorter({
        headers:
          {
	9: {sorter: "text"},
        10: {sorter: false },
         }
       , sortList: [[8,0]]
        });
});
</script>

<script type="text/javascript">
\$(document).ready(function()
{
        \$("#issuesTableB").tablesorter({
	headers:
	{
	9: {sorter: "text"},
        10: {sorter: false },
	}
        , sortList: [[8,0]]
     	});
});
</script>

<!--
<script type="text/javascript">
\$(document).ready(function()
{
        \$("#completeTable").tablesorter({
        headers:
          {
          11: {sorter: false },
          8: {sorter: "text"},
          10: {sorter: "text"}
         }
       , sortList: [[8,0]]
        });
});
</script>
-->

<script type="text/javascript" charset="utf-8">
\$(document).ready(function()
{
	\$('#completeTable').dataTable({
	"aaSorting": [[8,'asc']],
	"aoColumnDefs": [
          { "bSortable": false, "aTargets": [ 11 ] }
                      ],
	"sDom": '<iftif>',
	"oLanguage": {
         "sSearch": "Search (browser refresh: <a href=\"javascript:OffLink()\" onClick=\"window.stop()\">stop</a> | <a href=\"javascript:location.reload()\">on</a>)",
         "sInfo": "Showing _START_ to _END_ of _TOTAL_ files",
         "sInfoEmpty": "Showing 0 to 0 of 0 files",
         "sInfoFiltered": "(filtered from _MAX_ total files)"
        },
	"bPaginate": false
         });
});
function fnShowHide( iCol )
{
	/* Get the DataTables object again - this is not a recreation, just a get of the object */
	var oTable = \$('#completeTable').dataTable();
	
	var bVis = oTable.fnSettings().aoColumns[iCol].bVisible;
	oTable.fnSetColumnVis( iCol, bVis ? false : true );
}
</script>

<script language="JavaScript">
<!--
function openWin(URL) {
  aWindow=window.open(URL,"a","toolbar=yes,width=800,height=700,scrollbars=yes,menubar=yes");
  }
//-->
</script>
EOT

cat $TMP_DIR/offlink >> $TMP_DIR/output_top

cat >> $TMP_DIR/output_top <<EOT
<link rel="StyleSheet" href="$STYLES" type="text/css">
<!-- styles for tablesorter (issue tables) -->
<style type="text/css">
table.tablesorter {
        text-align:     left;
        font-family:    arial;
        font-size:      small;
        height:         10px;
}
table.details {
        text-align:     left;
        font-family:    arial;
        font-size:      small;
}
table.tablesorter thead tr .header {
        background-color:    #cccccc;
        background-image:    url(${IMG_URL}/bg.gif);
        background-repeat:   no-repeat;
        background-position: center right;
        height:              30px;
        vertical-align:      middle;
}
table.tablesorter thead tr .headerSortUp {
        background-color:    ${HDRSORT_COLOR};
        background-image:    url(${IMG_URL}/desc.gif);
}
table.tablesorter thead tr .headerSortDown {
        background-color:    ${HDRSORT_COLOR};
        background-image:    url(${IMG_URL}/asc.gif);
}
</style>

<!-- styles for datatable -->
<style type="text/css">
table.display {
        text-align:     left;
        font-family:    arial;
        font-size:      small;
        height:         10px;
}
table.display thead tr .header {
        background-color:    #CCCCCC;
        background-image:    url(${IMG_URL}bg.gif);
        background-repeat:   no-repeat;
        background-position: center right;
        height:              30px;
        vertical-align:      middle;
}
table.display thead tr .sorting_desc {
	background-color:    ${HDRSORT_COLOR};
	background: url(${IMG_URL}/desc.gif) no-repeat center right;
}
table.display thead tr .sorting {
	background-color:    ${HDRSORT_COLOR};
	background: url(${IMG_URL}/bg.gif) no-repeat center right;
}
table.display thead tr .sorting_asc {
	background-color:    ${HDRSORT_COLOR};
	background: url(${IMG_URL}/asc.gif) no-repeat center right;
}
</style>
</head>

<body>
<!-- header table -->
<a name="issues"></a>
<a name="top"></a>
<table cellpadding="1" cellspacing="1" border="0" width=1000>
  <tr>
    <td bgcolor=#FFFFCC COLSPAN=6>

     <table cellpadding="0" cellspacing="0" border=0>
       <tr align="left" valign="bottom">
         <td>
      <table style="border-collapse: collapse;" border="1" bordercolor="#ffffff" cellpadding="0" cellspacing="0">
        <tr>
          <td bgcolor="${REPORT_COLOR}" height="40" width="40" align="left" valign="bottom"><b><font color="#ffffff" size="3">
            <a title="detailed overview of calibration completeness">CAL</a></font></b>
          </td>
        </tr>
      </table>
      </td>
    <td valign="bottom" nowrap><b><font color=${REPORT_COLOR} size=+2>${DFO_INSTRUMENT} calChecker: telluric star tracker for science files, for date $DATE </font></b></td>
    </tr> 
    </table>
    </td>
  </tr>
  <tr bgcolor=$SM_COLOR>
    <td nowrap align=right width=17%><font size=2 color=#FFFFFF>Last update:</td>
    <td nowrap align=left  width=17%><font size=2 color=#FFFFFF>$TIMESTAMP (UT)</font></td>
    <td nowrap align=right width=17%><font size=2>&nbsp;</td>
    <td nowrap align=right width=17%><font size=2>&nbsp;</td>
    <!--<td nowrap align=left  width=17%><font size=2 color=#FFFFFF><a title="time window for taking TELL star">${TWI_TIMESTAMP}</a> &nbsp; [<a href="javascript:openWin('${QC_URL}/TT_help.html#update');" title="info about update"><font color=#FFFFFF>?</font></a>]</font></td>-->
    <td nowrap align=right width=17% bgcolor=#999999><font size=2>Configured validity <b>dTime</b>:</td>
    <td nowrap align=left  width=17% bgcolor=#999999><font size=2><b>$DRANGE2</b> hr ($DRANGE1 d)</td>
  </tr>
  <tr bgcolor=$SM_COLOR>
    <td nowrap align=right><font size=2 color=#FFFFFF>Last header: ${DFO_FILE_NAME}.</td>
    <td nowrap align=left><font size=2 color=#FFFFFF>${LAST_HDR}</font></td>
    <td nowrap align=right><font size=2 color=#FFFFFF>$TWI_LEFT1</font></td>
    <td nowrap align=left><font size=2 color=#FFFFFF>$TWI_LEFT2</font></td>
    <td nowrap align=right bgcolor=#999999><font size=2>Configured difference in airmass <b>dAIRM</b>:</td>
    <td nowrap align=left  bgcolor=#999999><font size=2><b>$D_AIRM</b></td>
  </tr>

<!-- quick help-->
  <tr bgcolor=#EEEEEE valign=top>
    <td COLSPAN=3 width=50%><font size=2 color=#666666><i>
Report about the telluric standard star calibrations (TELL) for all science OBs of the indicated date.<br>
&bull; All setup matches are evaluated by time (dTime) and airmass (dAirm) constraint.<br>
&nbsp;&nbsp;&nbsp;&nbsp;<font color="#00FF00"><b>OK</b></font>&nbsp;&nbsp;&nbsp;&nbsp; both constraints are satisfied.<br>
&nbsp;&nbsp;&nbsp;&nbsp;<font color="#FFFF00"><b>NOK</b></font>&nbsp; a match is found but at least one constraint is not satisfied (preference is given to the time constraint!).<br>
&nbsp;&nbsp;&nbsp;&nbsp;<font color="#FF0000"><b>MISS</b></font>&nbsp; no match is found.<br>
&bull; Information in last column during the night:<br>
&nbsp;&nbsp;&nbsp;&nbsp;<b>LEFT+&lt;time_left&gt;</b>: &lt;time_left&gt; remains before the time constraint is violated.<br>
&nbsp;&nbsp;&nbsp;&nbsp;<b>LOST-&lt;elapsed_time&gt;</b>: time constraint expired since &lt;elapsed_time&gt;<br>
&bull; Complete Overview shows in addition tellurics with their airmass and target name. 
    </i></font></td>

    <td COLSPAN=3 width=50% valign=top><font size=2 color=#666666>
EOT

if [ -s $DFO_CONFIG_DIR/info.tellTracker ]
then
	cat $DFO_CONFIG_DIR/info.tellTracker >> $TMP_DIR/output_top
else
	echo "&nbsp;" >> $TMP_DIR/output_top
fi

cat >> $TMP_DIR/output_top <<EOT
</font></td>
  </tr>

<!-- navigation -->
  <tr>
     <td ALIGN=LEFT nowrap><font size=1><br><a class="nav-subitem" href="javascript:openWin('${QC_URL}/TT_help.html');" title="help about tellTracker"><font color=#333333>HELP</font></a> |
      <a href="${QCWEB_URL}/CALCHECK/TELLTRACK" title="find tellTracker result pages for older dates"><font color=${SM_COLOR}>history ...</font></a> |
     <a href="mailto:${QC_ADDRESS}?subject=${DFO_INSTRUMENT} tellTracker" title="mail about this tellTracker result page"><font color=$SM_COLOR>contact</font></a>
     </td>
  </tr>

  <tr bgcolor=#FFFFFF>
    <td colspan="6">
      <table width="100%" border="0" cellpadding="0" cellspacing="0">
        <tr>
          <td height="3"></td>
          <td></td>   
          <td></td>   
          <td></td>   
          <td bgcolor="#FF9966"></td>
          <td></td>
          <td></td>
	  <td></td>
        </tr>
        <tr>
          <td width="3%" align=center nowrap bgcolor="#FFD5B5"><b><font size="1">&nbsp;<a href="${CALCHECK_URL}/calChecker_${DFO_INSTRUMENT}.html" title="calChecker: calibration completeness for science"><font color="#333333">science</font></a>&nbsp;</font></b></td>
	  <td width="1"><font size=1 color=#FFFFFF>.</font></td>
EOT

if [ $CHECK_C4C = YES ]
then
	cat >> $TMP_DIR/output_top <<EOT
          <td width="3%" align=center nowrap bgcolor="#FFD5B5"><b><font size="1">&nbsp;<a href="${CALCHECK_URL}/CAL4CAL/calChecker_${DFO_INSTRUMENT}.html" title="calChecker: calibration completeness for calibrations (only if configured)"><font color="#333333">cal4cal</font></a>&nbsp;</font></b></td>
EOT
else
	cat >> $TMP_DIR/output_top <<EOT
          <td width="3%" align=center bgcolor="#eeeeee" nowrap><font size="1">&nbsp;<font color="#333333"><a title="${DFO_INSTRUMENT} has no checks for calibration completeness for calibrations">no cal4cal</a></font>&nbsp;</font></td>
EOT
fi

cat >> $TMP_DIR/output_top <<EOT
          <td width="1"><font size=1 color=#FFFFFF>.</font></td> 
          <td width="3%" align=center nowrap bgcolor="#FF9966"><b><font size="1">&nbsp;<a href="tellTracker_$DATE.html" title="tellTracker: telluric standards for science [internal link]"><font color="#333333">tellTracker</font></a>&nbsp;</font></b></td>
          <td width="3%"><font size="1" color=#666666>&nbsp;[<a href="javascript:openWin('http://www.eso.org/qc/ALL/TT_help.html');" title="info about tellTracker">?</a>]</font></td>
          <td width="88%" align=right><i><font color="#999999">&nbsp;<font size="1">Date on this monitor changes at 21:00 UT. Current refresh frequency: 15 min during nighttime</font></font></i></td>
        </tr>
        <tr bgcolor="#FF9966" height="3">
      <td colspan="7"></td>
      </tr>
      </table>
    </td>
  </tr>

  <tr bgcolor=#FFFFFF><td>${PREV_HTMLTAG} ${NEXT_HTMLTAG} &nbsp; &nbsp; ${TOP_HTMLTAG} &nbsp; ${BOTTOM_HTMLTAG}</td>
    <td COLSPAN=5 align=right><font size=1 color=#999999><i>
<a id="refresh_link">[page auto-refreshes after $AUTO_REFRESH sec]</a> &nbsp;
   [<a href="javascript:OffLink()" onClick="window.stop()" title="click to provisionally stop browser refresh (will resume on reload)"><font color=#999999>stop</font></a>
  | <a href="javascript:location.reload()" title="click to resume browser refresh"><font color=#999999>on</font></a>]
</i></font>
    </td>
  </tr> 
</table>
<p>
EOT
}

# =====================================================================================================
# 1. $TODAY or for OLD_HDR_UPDATE=YES: download headers (wrapper, from calChecker) 
# =====================================================================================================
if ([ $DATE = $TODAY_DFO ] && [ $NEW_HDR = YES ]) || ([ $DATE != $TODAY_DFO ] && [ $OLD_HDR_UPDATE = YES ])
then
	createWrapper
	if [ $VERBOSE = YES ]
	then
		echo "- download new headers for $DATE ..."
		$TMP_DIR/wrappedHdrDownloader
		echo "  ... done."
	else
		$TMP_DIR/wrappedHdrDownloader 1>/dev/null
	fi
fi
export DFO_HDR_DIR=$DFO_HDR_DIR/TELLTRACK

if [ $DATE = $TODAY_DFO ] && [ $NEW_HDR = NO ] && [ $VERBOSE = YES ]
then
	echo "***INFO: no new headers downloaded."
fi

# =====================================================================================================
# 2. Classification and association
# 2.1 No headers: jump to end
# =====================================================================================================

NO_HDR=NO
if [ ! -s $DFO_HDR_DIR/$DATE ]
then
	DATE_minus=`qcdate $DATE -1`
	DATE_plus=`qcdate  $DATE +1`
	if [ $VERBOSE = YES ]
	then
		echo "No headers found. "
	fi
	createOutput
	mv $TMP_DIR/output_top $TMP_DIR/tellTracker.html
	cat >> $TMP_DIR/tellTracker.html <<EOT
<font size=2><b>No headers found.</b></font>
</body>
</html>

EOT
	NO_HDR=YES
fi

# no headers found: we skip all following except for scp part at the end
# headers found: we continue

# =====================================================================================================
# 2.2 Classification
# =====================================================================================================

if [ $NO_HDR = NO ]
then
	cd $DFO_HDR_DIR/$DATE 
	rm -f $TMP_DIR/tt_list_all

# tellTracker_radec.cfg: fixed cfg file for RA, DEC , LST: formatted output needed because of transformation
# tellTracker.cfg: first part fixed, second user configured
fitsreport -c tellTracker.cfg       `ls | grep .hdr` > $TMP_DIR/tt_list_all
fitsreport -c tellTracker_radec.cfg `ls | grep .hdr` > $TMP_DIR/tt_list_radec

if [ ! -s $TMP_DIR/tt_list_all ]
then
	if [ $VERBOSE = YES ]
	then
		echo "No data found for $DATE. "
	fi
	exit
fi

# =====================================================================================================
# 2.2 calculate timestamps etc.
# =====================================================================================================

# calculate NOW
MJD_TODAY=`qcdate $TODAY`
NOW=`echo $MJD_TODAY $TODAY_HH $TODAY_MM | awk '{printf"%10.5f\n", $1+$2/24+$3/24/60}'`
# for debugging of LEFT/LOSS:
#NOW=55619.02
#MJD_TODAY=55619
#TODAY_H=01
#TODAY_M=12
#echo DEBUG: $NOW $MJD_TODAY $TODAY_H $TODAY_M

# calculate twilight: turned off because unstable
#getTwilight

# MJD_MOR: MJD of morning twilight; converted into $TWI_TIMESTAMP for display
# difference NOW-MJD_MOR in HH and MM
DIFF=`echo $NOW $MJD_MOR | awk '{{print ($2-$1)*24}}'`
DIFF_HH=`echo $DIFF |      awk '{{printf"%i\n",$1}}'`
DIFF_MM=`echo $DIFF $DIFF_HH | awk '{{printf"%i\n",($1-$2)*60}}'`
DIFF_MM=`echo $DIFF_MM | awk '{print sqrt($1*$1)}'`
if [ $DIFF_MM -lt 10 ]
then
        DIFF_MM="0$DIFF_MM"
fi

TWI_DD=`echo $MJD_MOR | awk '{printf"%5.0f\n",$1}'`
TWI_HH=`echo $MJD_MOR $TWI_DD | awk '{printf"%i\n",($1-$2)*24}'`
TWI_MM=`echo $MJD_MOR $TWI_DD $TWI_HH | awk '{printf"%i\n",(($1-$2)*24-$3)*60}'`

if [ $TWI_HH -lt 10 ]
then
	TWI_HH="0$TWI_HH"
fi
if [ $TWI_MM -lt 10 ]
then
	TWI_MM="0$TWI_MM"
fi

DATE_plus=`qcdate $DATE +1`
MJD_DATE=`qcdate $DATE_plus`
TWI_TIMESTAMP="${DATE_plus}T${TWI_HH}:${TWI_MM} (UT)"

# TWI_LIMIT: 11 UT, long enought after true twilight morning
TWI_FRAC=`echo $MJD_TODAY | awk '{printf"%5.5f\n",$1+0.5-1/24-$1}'`
TWI_LIMIT=`echo $MJD_DATE $TWI_FRAC | awk '{printf"%5.5f\n",$1+$2}'`

#TWI_LEFT1=`echo $NOW $TWI_LIMIT | awk '{if ($1 < $2) {print "left:"} else {print "&nbsp;"}}'`
#TWI_LEFT2=`echo $NOW $TWI_LIMIT $TWI_MOR | awk '{if ($1 < $3) {printf"%s %s %s\n","<a title=\"time left\">",diff_hh":"diff_mm,"hr</a>"} else if ($1 < $2) {printf"%s %s %s\n","<a title=\"time left\"><i>",diff_hh":"diff_mm,"hr</i></a>"} else {print "&nbsp;"}}' diff_hh=$DIFF_HH diff_mm=$DIFF_MM`

# not supported anymore because skycalc URL is unstable
TWI_LEFT1="&nbsp;"
TWI_LEFT2="&nbsp;"

# get OB comments and comments
if [ $VERBOSE = YES ]
then
	getObInfo -d $DATE
else
	getObInfo -d $DATE 1>/dev/null 2>/dev/null
fi

mv $TMP_DIR/ob_isql_out  $TMP_DIR/tt_isql_out1
mv $TMP_DIR/ob_isql_out2 $TMP_DIR/tt_isql_out2

# get SM/VM flags
getFlag

# issue tables require less columns than full output --> two tables
rm -f $TMP_DIR/tt_output_issues $TMP_DIR/tt_output2 $TMP_DIR/tt_OB_issues $TMP_DIR/tt_output_details

# prepare HTML output
TD="</font></td><td><font size=2>"
TD0="<td nowrap><font size=2>"
TD1="</font></td><td nowrap><font size=2>"
TD2="</font></td><td nowrap><font size=1>"

TD_OK="</font></td><td bgcolor=$GRN align=right><font size=1>"
TD_NOK="</font></td><td bgcolor=$YLW align=right><font size=1>"
TD_MISS="</font></td><td bgcolor=$RED align=right><font size=1>"
TD_MISS2="</font></td><td bgcolor=$RED colspan=2><font size=1>"
TD_MISS3="</font></td><td bgcolor=$LRED align=right><font size=1>"
TD_MISS4="</font></td><td bgcolor=$LRED colspan=2><font size=1>"
TD_SEP="</font></td><td bgcolor=${HDRSORT_COLOR}><font size=1>&nbsp;"
TD_SEP2="</font></td><td bgcolor=${TELL_COLOR}><font size=1>&nbsp;"

# =====================================================================================================
# 2.3 Association: use $TELL_MATCH to associate $SCI_DATA_TYPE with proper $TELL_DATA_TYPE
# =====================================================================================================

rm -f $TMP_DIR/tt_list_all_tellurics*

# Find SCI DATA_TYPEs and their associated TELL_DATA_TYPEs
grep "^TELL_MATCH" $DFO_CONFIG_DIR/config.tellTracker | awk '{print $2}' > $TMP_DIR/tt_sci_data_types

# start loop over $SCI_DATA_TYPE
for SCI_DATA_TYPE in `cat $TMP_DIR/tt_sci_data_types`
do
	rm -f $TMP_DIR/tt_list_tellurics
	TELL_DATA_TYPE=`grep "^TELL_MATCH" $DFO_CONFIG_DIR/config.tellTracker | grep "[[:space:]]$SCI_DATA_TYPE[[:space:]]" | awk '{print $3}'`
	SCI_DEFSTRING=`grep  "^DATA_TYPE"  $DFO_CONFIG_DIR/config.tellTracker | grep "[[:space:]]$SCI_DATA_TYPE[[:space:]]" | sed "s/\&\&$//" | sed "s/^.*\&\&//"`
	TELL_DEFSTRING=`grep  "^DATA_TYPE" $DFO_CONFIG_DIR/config.tellTracker | grep "[[:space:]]$TELL_DATA_TYPE[[:space:]]" | sed "s/\&\&$//" | sed "s/^.*\&\&//"`

# =====================================================================================================
# 3. ANALYSIS: for each science, per type and setup: check telluric availability
# 3.1 suppress 60. runs
# =====================================================================================================

	cat > $TMP_DIR/tt_sci_exec <<EOT
cat $TMP_DIR/tt_list_all | grep SCIENCE | egrep -v " 60\.| 060\." |\
 ${SCI_DEFSTRING}
EOT
	chmod u+x $TMP_DIR/tt_sci_exec 
	$TMP_DIR/tt_sci_exec > $TMP_DIR/tt_list_science

	if [ ! -s $TMP_DIR/tt_list_science ] && [ $VERBOSE = YES ]
	then
		echo ""
		echo "$SCI_DATA_TYPE: no data found. "
		continue
	fi

# =====================================================================================================
# 3.2 loop across raw files, get all parameters
# =====================================================================================================

	cat > $TMP_DIR/tt_tell_exec <<EOT
cat $TMP_DIR/tt_list_all | grep CALIB |\
 ${TELL_DEFSTRING}
EOT
	chmod u+x $TMP_DIR/tt_tell_exec 
	$TMP_DIR/tt_tell_exec > $TMP_DIR/tt_list_tellurics

	if [ $VERBOSE = YES ]
	then
		echo ""
		echo "$SCI_DATA_TYPE:"
	fi
	MATCH_SETUP=`grep   "^TELL_MATCH" $DFO_CONFIG_DIR/config.tellTracker | grep $SCI_DATA_TYPE | awk '{print $4}'`
	DISPLAY_SETUP=`grep "^TELL_MATCH" $DFO_CONFIG_DIR/config.tellTracker | grep $SCI_DATA_TYPE | awk '{print $5}'`

	cat > $TMP_DIR/tt_find_match <<EOT
grep \$1 $TMP_DIR/tt_list_science | awk '{print $MATCH_SETUP}'
EOT
	chmod u+x $TMP_DIR/tt_find_match

	cat > $TMP_DIR/tt_find_display <<EOT
grep \$1 $TMP_DIR/tt_list_science | awk '{print $DISPLAY_SETUP}' | sed "s| |_|g"
EOT
	chmod u+x $TMP_DIR/tt_find_display

# TELLURICs
	if [ -s $TMP_DIR/tt_list_tellurics ]
	then
		sed "s/tt_list_science/tt_list_tellurics/" $TMP_DIR/tt_find_display > $TMP_DIR/tt_find_display_tell
		chmod u+x $TMP_DIR/tt_find_display_tell
		for TELL in `cat $TMP_DIR/tt_list_tellurics | awk '{print $1}'`
		do
			DISPLAY_SETUP=`$TMP_DIR/tt_find_display_tell $TELL`
                	DISPLAY_SETUP1=`echo $DISPLAY_SETUP | sed "s/ /_/g"`
			grep $TELL $TMP_DIR/tt_list_tellurics | sed "s|^.*|$TELL_DATA_TYPE $DISPLAY_SETUP1 &|" >> $TMP_DIR/tt_list_all_tellurics
		done
	fi

# SCIENCE: loop over raw files; get all required parameters
	for SCI in `cat $TMP_DIR/tt_list_science | awk '{print $1}'`
	do
		MATCH_SETUP=`$TMP_DIR/tt_find_match $SCI`
		MATCH_SETUP1=`echo $MATCH_SETUP | sed "s/ /_/g"`
		DISPLAY_SETUP=`$TMP_DIR/tt_find_display $SCI`
		DISPLAY_SETUP1=`echo $DISPLAY_SETUP | sed "s/ /_/g"`

# this part relies on fixed positions in tellTracker.cfg: MJD_OBS=$5 ... OBS_ID=$9
		MJD_OBS=`grep $SCI $TMP_DIR/tt_list_science | awk '{print $5}'`
		AIRM=`grep    $SCI $TMP_DIR/tt_list_science | awk '{printf"%3.2f\n",($6+$7)/2}'`
		PROG_ID=`grep $SCI $TMP_DIR/tt_list_science | awk '{print $8}'`
		OBS_ID=`grep  $SCI $TMP_DIR/tt_list_science | awk '{print $9}'`

# this part also relies on fixed positions in tellTracker_radec.cfg: LST=$2...DEC=$5
		LST=`grep     $SCI $TMP_DIR/tt_list_radec | awk '{print $2}'`
		RA=`grep      $SCI $TMP_DIR/tt_list_radec | awk '{print $3}'`
		DEC=`grep     $SCI $TMP_DIR/tt_list_radec | awk '{print $4}'`

# UT part of ARCFILE ($SCI), to give some time orientation for issue part
		UT=`echo $SCI | sed "s/-/ /g" | awk '{print $3}' | cut -c4-11`

# getObGrades from calChecker, for this date only
		GRADE=`grep " $OBS_ID " $TMP_DIR/tt_isql_out1 | awk '{print $4}' | tail -1 | sed "s/_//"`
		if [ "Q$GRADE" = Q ]
		then
			GRADE=n/a
		fi

# same colors as in createReport
		case $GRADE in
		  "A"|"B" ) OBCOLOR=$LGRN ;;
                  "C"|"D" ) OBCOLOR=$LRED ;;
                  "X"|"n/a" ) OBCOLOR=$GRY  ;;
                esac

		OBCOMM=`grep " $DATE " $TMP_DIR/tt_isql_out2 | grep " $OBS_ID " | grep -v "NOCOMMENT" | sed "s/^.*UT_START//" | sed "s/COMMENT//" | sed "s/\"/\&ldquo;/g" | sed "s/^.*/&YY/" | sed "s/   //g" | tr "\012" " " | sed "s/YY/ |/"g | sed "s/ | $//"`
		if [ "Q$OBCOMM" != Q ]
		then
			GRADE="<a title=\"$OBCOMM\">${GRADE}!</a>"
		fi
		GRADE="</font></td><td bgcolor=$OBCOLOR><font size=2>$GRADE"

		MODE=`grep "^$PROG_ID " $TMP_DIR/tt_out_flagquery | awk '{print $2}' | sed "s/s/<font color=${SM_COLOR}><b>SM<\/b><\/font>/" | sed "s/v/<font color=${VM_COLOR}>VM<\/font>/" | sed "s/^.*/<b>&<\/b>/"`

# overview:
		TDINI="${TD0}${PROG_ID}${TD}$MODE${TD}$OBS_ID$GRADE${TD}$LST${TD}$RA${TD}$DEC${TD}$AIRM${TD}"
# issues:
		TDINI2="${TD0}${PROG_ID}${TD}$MODE${TD}$OBS_ID$GRADE${TD}$LST${TD}$RA${TD}$DEC${TD}$AIRM${TD}$UT${TD1}"

# match SCIENCE with STD_TELL; tt_inv_tellurics: inventory of all matching tellurics 
		rm -f $TMP_DIR/tt_inv_tellurics $TMP_DIR/tt_tell_results
                # WH 2013-11-05: grep $MATCH_SETUP => grep M1 | grep M2 | grep M3 ...
		# grep " ${MATCH_SETUP}" $TMP_DIR/tt_list_tellurics | awk '{print $1,$5,($6+$7)/2}' > $TMP_DIR/tt_inv_tellurics
                MATCH_COMMAND=""
                for MS in $MATCH_SETUP
                    do
                    MATCH_COMMAND=$MATCH_COMMAND" | grep \"$MS\""
                    done
		COMMAND_STRING="cat  $TMP_DIR/tt_list_tellurics $MATCH_COMMAND | awk '{print \$1,\$5,(\$6+\$7)/2}' > $TMP_DIR/tt_inv_tellurics"
                eval $COMMAND_STRING


# analyze that inventory to find either OK-OK, or best NOK-OK/OK-NOK
		if [ -s $TMP_DIR/tt_inv_tellurics ]
		then
			for TELL in `cat $TMP_DIR/tt_inv_tellurics | awk '{print $1}'`
			do
# absolute numbers
				DIFF_MJD_OBS=`grep $TELL $TMP_DIR/tt_inv_tellurics | awk '{printf"%5.4f\n",sqrt(($2-mjd)*($2-mjd))}' mjd=$MJD_OBS`
				FLAG_MJD_OBS=`echo $DIFF_MJD_OBS | awk '{if ($1 <=drange) {print "OK"} else {print "NOK"}}' drange=$DRANGE`
				DIFF_AIRM=`grep $TELL $TMP_DIR/tt_inv_tellurics | awk '{print sqrt(($3-airm)*($3-airm))}' airm=$AIRM`
				FLAG_AIRM=`echo $DIFF_AIRM | awk '{if ($1 <=diff_airmass) {print "OK"} else {print "NOK"}}' diff_airmass=$D_AIRM`
                                # WH 2013-11-05: tt_list_all_tellurics ==> tt_list_tellurics
				TELL_DATA_TYPE=`grep $TELL $TMP_DIR/tt_list_tellurics | awk '{print $1}'`
				echo "  $TELL $MATCH_SETUP1 $MJD_OBS $DIFF_MJD_OBS $DIFF_AIRM $FLAG_MJD_OBS $FLAG_AIRM $TELL_DATA_TYPE" >> $TMP_DIR/tt_tell_results
			done

# =====================================================================================================
# 3.3 flags OK, NOK, MISS; analyze tt_tell_results: find either OK-OK, or best NOK-OK/OK-NOK 
#     (will break if NOK or OK string exists for other reasons...)
# =====================================================================================================

			if [ -s $TMP_DIR/tt_tell_results ]
			then

# =====================================================================================================
# 3.3.1 OK
# =====================================================================================================

				CHECK_OK=`grep -v NOK $TMP_DIR/tt_tell_results | head -1`

# OK; sort multiple matches by minimum time difference $4
				if [ "Q$CHECK_OK" != Q ]
				then
					grep -v NOK $TMP_DIR/tt_tell_results | sort -k4,4 | head -1 > $TMP_DIR/tt_tell_results_final
					TELL_RESULT1=`cat $TMP_DIR/tt_tell_results_final | awk '{printf"%2.3f\n",$4}'`
					TELL_RESULT2=`cat $TMP_DIR/tt_tell_results_final | awk '{printf"%2.2f\n",$5}'`
					TELL=`cat $TMP_DIR/tt_tell_results_final | awk '{print $1}'`
					echo "${TDINI}$SCI${TD}$SCI_DATA_TYPE${TD1}$DISPLAY_SETUP${TD_SEP}${TD_OK}OK${TD_OK}$TELL${TD_OK}$TELL_RESULT1${TD_OK}$TELL_RESULT2${TD_OK}&nbsp;&nbsp;&nbsp;&nbsp;" >> $TMP_DIR/tt_output2
					if [ $VERBOSE = YES ]
					then
						echo "$SCI $OBS_ID $SCI_DATA_TYPE $DISPLAY_SETUP1 OK $TELL_RESULT1 $TELL_RESULT2" 
						grep -v NOK $TMP_DIR/tt_tell_results | sort -k4,4 | awk '{print "   "$1,$2,$4,$5,$6,$7,$8}'
					fi
					echo "sci: $SCI $OBS_ID $OBS_ID $SCI_DATA_TYPE $DISPLAY_SETUP1 $TELL_RESULT1 $TELL_RESULT2 OK " >> $TMP_DIR/tt_output_details
					grep -v NOK $TMP_DIR/tt_tell_results | sort -k4,4 | awk '{print "   "$1,"tell",obs","sci,$8,$2,$4,$5,$6,$7}' obs=$OBS_ID sci=$SCI >> $TMP_DIR/tt_output_details
					
				else

# =====================================================================================================
# 3.3.2 NOK
# =====================================================================================================

					TIME_LEFT=`grep NOK $TMP_DIR/tt_tell_results | awk '{if ((now-$3)*24*60 < range) {printf("%s%i%s\n","<a title=\"time in mins. left for satisfying time constraint\">LEFT+",range-(now-$3)*24*60,"</a>")} else if ( now < twi_limit ) {printf("%s%i%s\n","<a title=\"time in mins. after constraint expired\">LOST-",(now-$3)*24*60-range,"</a>")} else {printf("%s\n","<a title=\"time constraint expired since more than 12 hrs.\">LOST</a>")}}' now=$NOW range=$RANGE twi_limit=$TWI_LIMIT | head -1`
					TIME_LEFT1=`echo $TIME_LEFT | sed "s/<a title=.*//"`

# we select for closest-in-time 'NOK OK' because this is closest to calChecker logics (commented out: opposite case OK NOK)
					TELL_RESULT=`grep "NOK OK" $TMP_DIR/tt_tell_results | awk '{print $4,$5}' | sort -k1,1 | head -1`
					TELL=`grep "NOK OK" $TMP_DIR/tt_tell_results | awk '{print $1,$4,$5} ' | sort -k2,2 | head -1 | awk '{print $1}'`
					TELL_CASE="NOK OK"
					if [ "Q$TELL_RESULT" = Q ]
					then
						TELL_RESULT=`grep " OK NOK" $TMP_DIR/tt_tell_results | awk '{print $4,$5}' | sort -k1,1 | head -1`
						TELL=`grep " OK NOK" $TMP_DIR/tt_tell_results | awk '{print $1,$4,$5} ' | sort -k2,2 | head -1 | awk '{print $1}'`
						TELL_CASE="OK NOK"
					fi

					if [ "Q$TELL_RESULT" = Q ]
					then
						TELL_RESULT=`grep "NOK NOK" $TMP_DIR/tt_tell_results | awk '{print $4,$5}' | sort -k1,1 | head -1`
						TELL=`grep "NOK NOK" $TMP_DIR/tt_tell_results | awk '{print $1,$4,$5} ' | sort -k2,2 | head -1 | awk '{print $1}'`
						TELL_CASE="NOK NOK"
					fi

					TELL_RESULT1=`echo "$TELL_RESULT" | awk '{printf"%2.3f\n",$1}'`
					TELL_RESULT2=`echo "$TELL_RESULT" | awk '{printf"%2.2f\n",$2}'`

					case $TELL_CASE in
					 "OK NOK"  ) echo "${TDINI}$SCI${TD}$SCI_DATA_TYPE${TD1}$DISPLAY_SETUP${TD_SEP}${TD_NOK}NOK${TD_NOK}$TELL${TD_OK}$TELL_RESULT1${TD_NOK}$TELL_RESULT2${TD_NOK}$TIME_LEFT" >> $TMP_DIR/tt_output2 
					  	     echo "${TDINI2}$DISPLAY_SETUP${TD_SEP}${TD_NOK}NOK${TD_NOK}${TELL_DATA_TYPE}${TD_OK}$TELL_RESULT1${TD_NOK}$TELL_RESULT2${TD_NOK}$TIME_LEFT" >> $TMP_DIR/tt_output_issues ;;
					 "NOK OK"  ) echo "${TDINI}$SCI${TD}$SCI_DATA_TYPE${TD1}$DISPLAY_SETUP${TD_SEP}${TD_NOK}NOK${TD_NOK}$TELL${TD_NOK}$TELL_RESULT1${TD_OK}$TELL_RESULT2${TD_NOK}$TIME_LEFT" >> $TMP_DIR/tt_output2 
					 	     echo "${TDINI2}$DISPLAY_SETUP${TD_SEP}${TD_NOK}NOK${TD_NOK}${TELL_DATA_TYPE}${TD_NOK}$TELL_RESULT1${TD_OK}$TELL_RESULT2${TD_NOK}$TIME_LEFT" >> $TMP_DIR/tt_output_issues ;;
					 "NOK NOK"  ) echo "${TDINI}$SCI${TD}$SCI_DATA_TYPE${TD1}$DISPLAY_SETUP${TD_SEP}${TD_NOK}NOK${TD_NOK}$TELL${TD_NOK}$TELL_RESULT1${TD_NOK}$TELL_RESULT2${TD_NOK}$TIME_LEFT" >> $TMP_DIR/tt_output2 
					 	      echo "${TDINI2}$DISPLAY_SETUP${TD_SEP}${TD_NOK}NOK${TD_NOK}${TELL_DATA_TYPE}${TD_NOK}$TELL_RESULT1${TD_NOK}$TELL_RESULT2${TD_NOK}$TIME_LEFT" >> $TMP_DIR/tt_output_issues ;;
					esac

					echo "$OBS_ID $DISPLAY_SETUP" >> $TMP_DIR/tt_OB_issues 

					if [ $VERBOSE = YES ]
					then
						echo "$SCI $OBS_ID $SCI_DATA_TYPE $DISPLAY_SETUP1 NOK $TELL_RESULT1 $TELL_RESULT2 $TIME_LEFT1"
						grep NOK $TMP_DIR/tt_tell_results | awk '{print "   "$1,$2,$4,$5,$6,$7,$8}'
					fi
					echo "sci: $SCI $OBS_ID $OBS_ID $SCI_DATA_TYPE $DISPLAY_SETUP1 $TELL_RESULT1 $TELL_RESULT2 NOK " >> $TMP_DIR/tt_output_details
					grep NOK $TMP_DIR/tt_tell_results | awk '{print "   "$1,"tell",obs","sci,$8,$2,$4,$5,$6,$7}' sci=$SCI obs=$OBS_ID >> $TMP_DIR/tt_output_details
				fi
			fi
		else

# =====================================================================================================
# 3.3.3 MISS
# =====================================================================================================

			TIME_LEFT=`grep $SCI $TMP_DIR/tt_list_science | awk '{if ((now-$5)*24*60 < range ) {printf("%s%i%s\n","<a title=\"time in mins. left for satisfying time constraint\">LEFT+",range-(now-$5)*24*60,"</a>")} else if (now < twi_limit ) {printf("%s%i%s\n","<a title=\"time in mins. after constraint expired\">LOST-",(now-$5)*24*60-range,"</a>")} else {printf("%s\n","<a title=\"time constraint expired since more than 12 hrs.\">LOST</a>")}}' now=$NOW range=$RANGE twi_limit=$TWI_LIMIT`
			TIME_LEFT1=`echo $TIME_LEFT | sed "s/<a title=.*//"`

# different color codes for LOST and LEFT	
			CHECK_LEFT=`echo $TIME_LEFT | grep LEFT | awk '{print $1}'`
			if [ Q$CHECK_LEFT = Q ]
			then
				echo "${TDINI}$SCI${TD}$SCI_DATA_TYPE${TD1}$DISPLAY_SETUP${TD_SEP}${TD_MISS}MISS${TD_MISS}miss${TD_MISS}&nbsp;${TD_MISS}&nbsp;${TD_MISS}$TIME_LEFT" >> $TMP_DIR/tt_output2
				echo "${TDINI2}$DISPLAY_SETUP${TD_SEP}${TD_MISS}MISS${TD_MISS}${TELL_DATA_TYPE}${TD_MISS2}&nbsp;${TD_MISS}$TIME_LEFT" >> $TMP_DIR/tt_output_issues
			else
				echo "${TDINI}$SCI${TD}$SCI_DATA_TYPE${TD1}$DISPLAY_SETUP${TD_SEP}${TD_MISS3}MISS${TD_MISS3}miss${TD_MISS3}&nbsp;${TD_MISS3}&nbsp;${TD_MISS3}$TIME_LEFT" >> $TMP_DIR/tt_output2
				echo "${TDINI2}$DISPLAY_SETUP${TD_SEP}${TD_MISS3}MISS${TD_MISS3}${TELL_DATA_TYPE}${TD_MISS4}&nbsp;${TD_MISS3}$TIME_LEFT" >> $TMP_DIR/tt_output_issues
			fi

			echo "$OBS_ID $DISPLAY_SETUP" >> $TMP_DIR/tt_OB_issues 

			if [ $VERBOSE = YES ]
			then
				echo "$SCI $OBS_ID $SCI_DATA_TYPE $DISPLAY_SETUP1 MISS $TIME_LEFT1"
			fi
			echo "sci: $SCI $OBS_ID $OBS_ID $SCI_DATA_TYPE $DISPLAY_SETUP1 &nbsp; &nbsp; MISS " >> $TMP_DIR/tt_output_details
			echo "   no tell $OBS_ID,$SCI found &nbsp; &nbsp; &nbsp; MISS MISS" >> $TMP_DIR/tt_output_details 
		fi
	done
done

# =====================================================================================================
# 3.3.4 sort out tt_list_all_tellurics and give them an index 
# =====================================================================================================

if [ -s $TMP_DIR/tt_list_all_tellurics ]
then
        # WH 2013-11-05: sort -u | sort -k3,3   ==> sort -k3,3 -u 
	cat $TMP_DIR/tt_list_all_tellurics | sort -k3,3 -u > $TMP_DIR/tt_list_all_telluric1
        # WH 2013-11-05: list_all_telluric ==> list_all_tellurics 
	mv $TMP_DIR/tt_list_all_telluric1 $TMP_DIR/tt_list_all_tellurics
	cat $TMP_DIR/tt_list_all_tellurics | sort -k3,3 | awk '{if (NR < 10) {print $3,"TELL0"NR} else {print $3,"TELL"NR}}' > $TMP_DIR/tt_list_all_tellurics_index
fi

# =====================================================================================================
# 3.4 turn tt_output_details into HTML output: details page
# =====================================================================================================

if [ -s $TMP_DIR/tt_output_details ]
then
	cat > $TMP_DIR/tellTracker_${DATE}_details.html <<EOT
<html>
<head>
<META HTTP-EQUIV="Refresh" CONTENT="$AUTO_REFRESH">

<title>${DFO_INSTRUMENT} tellTracker: details for date $DATE</title>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery-latest.js"></script>
<script type="text/javascript" src="${JSCRIPT_URL}/jquery.dataTables.js"></script>

<script type="text/javascript" charset="utf-8">
\$(document).ready(function()
{
	\$('#detailedTable').dataTable({
	"aaSorting": [[0,'asc']],
	"aoColumnDefs": [ 
	{ "bSortable": false, "aTargets": [ 0,1 ] } ,
	{ "bVisible":  false, "aTargets": [ 0,3 ]}
	],
	"sDom": '<iftif>',
	"oLanguage": {
         "sSearch": "<b>Search:</b><i> Enter a string.</i> <b>Dependency search:</b><i> Enter OB_ID or science ARCFILE to see all possible telluric matches</i><br> ",
         "sInfo": "Showing _START_ to _END_ of _TOTAL_ files (tellurics may show up multiple times)",
         "sInfoEmpty": "Showing 0 to 0 of 0 files",
         "sInfoFiltered": "(filtered from _MAX_ total files)"
        },
	"bPaginate": false
         });
});
</script>
<!-- styles for datatable -->
<style type="text/css">
table.display {
        text-align:     left;
        font-family:    arial;
        font-size:      small;
        height:         10px;
}
table.display thead tr .header {
        background-color:    #CCCCCC;
        background-image:    url(${IMG_URL}bg.gif);
        background-repeat:   no-repeat;
        background-position: center right;
        height:              30px;
        vertical-align:      middle;
}
table.display thead tr .sorting_desc {
	background-color:    ${HDRSORT_COLOR};
	background: url(${IMG_URL}/desc.gif) no-repeat center right;
}
table.display thead tr .sorting {
	background-color:    ${HDRSORT_COLOR};
	background: url(${IMG_URL}/bg.gif) no-repeat center right;
}
table.display thead tr .sorting_asc {
	background-color:    ${HDRSORT_COLOR};
	background: url(${IMG_URL}/asc.gif) no-repeat center right;
}
</style>
</head>

<body>
<h2>Detailed overview of all possible matches</h2>
<font size=2>
Here we display all possible matches between science files and telluric standar stars. This might be useful in cases
of mixed NOK/OK matches for time and airmass constraints. Listed are: all science files; all telluric matches to the science files; their dTime and dAirm values; and the corresponding flags. Hence, telluric standards may show up multiple times in this table.<p>

You can do a standard string search in this sortable table, as well as an advanced search with display of all 
telluric matches per OBS_ID and/or per science ARCFILE.<p>

<font size=2>
<table id="detailedTable" class="display" style="font-size: x-small; text-align: left">
<thead>
<tr style="font-size: small;" bgcolor=#CCCCCC>
<th>INDEX (hidden)</th>
<th><a title="ARCFILE of raw file">ARCFILE</a></th>
<th><a title="OB ID for science file">OBS_ID</a></th>
<th>OB (hidden)</th>
<th><a title="data type of this file as used on the main result page">DATA TYPE</a></th>
<th><a title="setup as used on the main result page">SETUP</a></th>
<th><a title="time difference SCI-TELL in fractions of day">dTime</a></th>
<th><a title="airmass difference SCI-TELL">dAirm</a></th>
<th><a title="flag for time difference SCI-TELL">flagT</a></th>
<th><a title="flag for airmass difference SCI-TELL">flagA</a></th>
</tr>
</thead>

<tbody>
EOT
# do the proper sorting
	rm -f $TMP_DIR/tt_output_details_sorted
	for SCI in `grep "^sci:" $TMP_DIR/tt_output_details | awk '{print $2}' | sort -k1,1`
	do
		grep "^sci: $SCI" $TMP_DIR/tt_output_details >> $TMP_DIR/tt_output_details_sorted
		grep -v "^sci:" $TMP_DIR/tt_output_details | grep $SCI >> $TMP_DIR/tt_output_details_sorted
	done

	cat $TMP_DIR/tt_output_details_sorted |\
	 sed "s/^   /tell:/g" |\
	 awk '{print NR,$0}' |\
	 sed "s| |</td><td>|g" |\
	 sed "/sci:/s|^.*|<tr bgcolor=#CCCCCC><td>&</td></tr>|" |\
	 sed "s|<td>sci:</td>||" |\
	 sed "/tell:/s|^.*|<tr bgcolor=#eeeeee><td>&</td></tr>|" |\
	 sed "s|tell:|\&nbsp;\&nbsp;\&nbsp;\&nbsp;|" |\
	 sed "s/>MISS</ bgcolor=$RED>MISS</"g |\
	 sed "s/>NOK</ bgcolor=$YLW>NOK</"g |\
	 sed "s/>OK</ bgcolor=$GRN>OK</"g |\
	 sed "s/<td>/<td nowrap>/"g \
 >> $TMP_DIR/tellTracker_${DATE}_details.html

	cat >> $TMP_DIR/tellTracker_${DATE}_details.html <<EOT
</font></i>
</tbody>

<tfoot>
<tr style="font-size: small;" bgcolor=#CCCCCC>
<th>INDEX (hidden)</th>
<th><a title="ARCFILE of raw file">ARCFILE</a></th>
<th><a title="OB ID for science file">OBS_ID<font color=#CCCCCC>__</font></a></th>
<th>OB (hidden)</th>
<th><a title="data type of this file as used on the main result page">DATA TYPE</a></th>
<th><a title="setup as used on the main result page">SETUP</a></th>
<th><a title="time difference SCI-TELL in fractions of day">dTime<font color=#CCCCCC>__</font></a></th>
<th><a title="airmass difference SCI-TELL">dAirm<font color=#CCCCCC>__</font></a></th>
<th><a title="flag for time difference SCI-TELL">flagT<font color=#CCCCCC>__</font></a></th>
<th><a title="flag for airmass difference SCI-TELL">flagA<font color=#CCCCCC>__</font></a></th>
</tfoot>

</table>
</body>
</html>
EOT
fi

# =====================================================================================================
# 4. create HTML output
# 4.1 navigation
# =====================================================================================================

DATE_minus=`qcdate $DATE -1`
DATE_plus=`qcdate  $DATE +1`

PREV_HTMLTAG="<a href=tellTracker_${DATE_minus}.html><img src=${IMG_URL}/left.gif width=16 height=17 border=0 title=\"previous night\"></a>"
NEXT_HTMLTAG="<a href=tellTracker_${DATE_plus}.html><img src=${IMG_URL}/right.gif width=16 height=17 border=0 title=\"next night\"></a>"
TOP_HTMLTAG="<a href=#top title=\"jump to top of page\"><b><font color=${REPORT_COLOR} size=2>top</font></b></a>"
BOTTOM_HTMLTAG="<a href=#bottom title=\"jump to bottom of page\"><b><font color=${REPORT_COLOR} size=2>bottom</font></b></a>"
if [ ! -s $TMP_DIR/tt_output2 ]
then
	TOP_HTMLTAG="&nbsp"
	BOTTOM_HTMLTAG="&nbsp"
fi

AUTO_REFRESH_TAG="<font size=1 color=#999999><i><a>[page auto-refreshes after $AUTO_REFRESH sec]</a> &nbsp; [<a href=\"javascript:OffLink()\" onClick=\"window.stop()\" title=\"click to provisionally stop browser refresh (will resume on reload)\"><font color=#999999>stop</font></a> | <a href=\"javascript:location.reload()\" title=\"click to resume browser refresh\"><font color=#999999>on</font></a>]</i></font>"

# =====================================================================================================
# 4.2 master header and footer for sortable output page
# =====================================================================================================
#NOTE: 
# <td bgcolor=${HDRSORT_COLOR}><font size=1 color=${HDRSORT_COLOR}>.</td>
# the '.' is needed since otherwise the next column (FLAG) fails sorting. No clue why.

cat > $TMP_DIR/tt_header.html <<EOT
<thead>
  <tr bgcolor=#CCCCCC>
    <th><font size=2><a title="run ID as read from the header">PROG_ID</a></th>
    <th><font size=2><a title="ServiceMode or VisitorMode">M</a></th>
    <th><font size=2><a title="OB ID as read from the header">OBS_ID</a></th>
    <th><font size=2><a title="OB grade (A|B|C|D or X) as read from the nightlog database">GRD</a></th>
    <th><font size=2><a title="LST at start as read from keyword 'LST'">LST</a></th>
    <th><font size=2><a title="RA (J2000) of pointing as read from keyword 'RA'">RA</a></th>
    <th><font size=2><a title="DEC (J2000) of pointing as read from keyword 'DEC'">DEC</a></th>
    <th><font size=2><a title="mean airmass during observation">AIRM</a></th>
    <th><font size=2><a title="UT part of ARCFILE">UT</a></th>
    <th><font size=2><a title="ARCFILE of raw file">ARCFILE</a></th>
    <th><font size=2><a title="data type of this file as used on the main result page">DATA TYPE</a></th>
    <th><font size=2><a title="setup as used on the main result page">SETUP</a></th>
    <th bgcolor=${HDRSORT_COLOR} width=1><font size=1 color=${HDRSORT_COLOR}></th>
    <th><a title="result flag (OK/NOK/MISS)">RESULT<br><font size=1>FLAG</font></a></th>
    <th><br><a title="matched telluric std star"><font size=1>TELL</font></th>
    <th><br><a title="type of telluric std star"><font size=1>TELL_DATA_TYPE</font></th>
    <th><br><a title="time difference SCI-TELL in fractions of day"><font size=1>dTime</font></th>
    <th><br><a title="airmass difference SCI-TELL"><font size=1>dAirm</font></th>
    <th><br><a title="time remaining in mins"><font size=1>remain</font></th>
  </tr>
</thead> 
EOT

# master footer for sortable output page (identical to header except for some expansion with ___ for better tablesorter makeup)
cat > $TMP_DIR/tt_footer.html <<EOT
<tfoot>
  <tr bgcolor=#CCCCCC>
    <th><font size=2><a title="run ID as read from the header">PROG_ID</a></th>
    <th><font size=2><a title="ServiceMode or VisitorMode">M</a><font color=#CCCCCC>__</font></th>
    <th><font size=2><a title="OB ID as read from the header">OBS_ID</a><font color=#CCCCCC>__</font></th>
    <th><font size=2><a title="OB grade (A|B|C|D or X) as read from the nightlog database">GRD</a><font color=#CCCCCC>__</font></th>
    <th><font size=2><a title="LST at start as read from keyword 'LST'">LST</a></th>
    <th><font size=2><a title="RA (J2000) of pointing as read from keyword 'RA'">RA</a></th>
    <th><font size=2><a title="DEC (J2000) of pointing as read from keyword 'DEC'">DEC</a></th>
    <th><font size=2><a title="mean airmass during observation">AIRM<font color=#CCCCCC>__</font></a></th>
    <th><font size=2><a title="UT part of ARCFILE">UT</a></th>
    <th><font size=2><a title="ARCFILE of raw file">ARCFILE</a></th>
    <th><font size=2><a title="data type of this file as used on the main result page">DATA TYPE</a></th>
    <th><font size=2><a title="setup as used on the main result page">SETUP</a><font color=#CCCCCC>__</font></th>
    <th bgcolor=${HDRSORT_COLOR} width=1><font size=1></th>
    <th><a title="result flag (OK/NOK/MISS)">RESULT<br><font size=1>FLAG</font></a></th>
    <th><br><a title="matched telluric std star"><font size=1>TELL</font></th>
    <th><br><a title="type of telluric std star"><font size=1>TELL_DATA_TYPE</font></th>
    <th><br><a title="time difference SCI-TELL in fractions of day"><font size=1>dTime</font></th>
    <th><br><a title="airmass difference SCI-TELL"><font size=1>dAirm</font></th>
    <th><br><a title="remaining time in mins"><font size=1>remain</font></th>
</tfoot>
EOT

# header and footer for issue tables
cat $TMP_DIR/tt_header.html | grep -v DATE | grep -v ">ARCFILE<" | grep -v "DATA TYPE" | grep -v ">TELL<" > $TMP_DIR/tt_header_issues.html
cat $TMP_DIR/tt_footer.html | grep -v DATE | grep -v ">ARCFILE<" | grep -v "DATA TYPE" | grep -v ">TELL<" > $TMP_DIR/tt_footer_issues.html

# header and footer for overview table: remove 'UT'
sed -i -e "/>UT</d"  $TMP_DIR/tt_header.html
sed -i -e "/>TELL_DATA_TYPE</d"  $TMP_DIR/tt_header.html
sed -i -e "/>UT</d"  $TMP_DIR/tt_footer.html
sed -i -e "/>TELL_DATA_TYPE</d"  $TMP_DIR/tt_footer.html

# =====================================================================================================
# 4.3 head: javascript etc.
#     (find the complete tablesorter javascript options in calChecker)
# =====================================================================================================

createOutput

cat $TMP_DIR/output_top > $TMP_DIR/tellTracker.html

# =====================================================================================================
# 4.5 results: issues
# 4.5.1 no data: done
# =====================================================================================================

if [ ! -s $TMP_DIR/tt_output2 ]
then
	cat >> $TMP_DIR/tellTracker.html <<EOT
<font size=2><b>No SCIENCE data to check for telluric match.</font>
</body>
</html>
EOT

else

# =====================================================================================================
# 4.5.2 issues: recoverable
# =====================================================================================================

	if [ ! -s $TMP_DIR/tt_output_issues ]
	then
		CHECK_ISSUES=""
	else
		CHECK_ISSUES=`egrep "MISS|NOK" $TMP_DIR/tt_output_issues | head -1 | awk '{print $1}'`
	fi

	rm -f $TMP_DIR/CALCHECK/tt_issue*
	if [ Q$CHECK_ISSUES != Q ]
	then
		rm -f $TMP_DIR/tt_issuesA.html
		cat $TMP_DIR/tt_output_issues | egrep "MISS|NOK" | grep "LEFT" | sed "s|^.*|<tr bgcolor=#CCCCCC>&</font></td></tr>|" > $TMP_DIR/tt_issuesA.html
		cat $TMP_DIR/tt_output_issues | grep "MISS" | grep "LOST-" | sed "s|^.*|<tr bgcolor=#CCCCCC>&</font></td></tr>|" >> $TMP_DIR/tt_issuesA.html

		if [ -s $TMP_DIR/tt_issuesA.html ]
                then
			ISSUE_STRING="&nbsp;"
			BGCOLOR=$LRED
		else
			ISSUE_STRING="NONE"
			BGCOLOR=#CCCCCC
		fi

		cat >> $TMP_DIR/tellTracker.html <<EOT
<table>
  <tr>
    <td bgcolor=$BGCOLOR><font size=3><b>Issues discovered:</b></td>
    <td><font size=2>&nbsp; [<a href=#details>complete overview</a>]</font></td>
  </tr>
  <tr>
    <td bgcolor=$BGCOLOR><font size=2><b><i>a) recoverable:</i></b></td>
    <td bgcolor=#eeeeee><font size=2>&nbsp; ${ISSUE_STRING}</font></td>
  </tr>
</table>
<font size=2><i>
[OBs that could be improved by observing a matching telluric standard star: either NOK matches
  with some validity left, or MISS matches with LEFT, <br>or MISS + LOST if night is not yet over, in the latter case because
  a NOK-LOST might still be better than a MISS-LOST.] </i>
</font><p>
EOT
		if [ -s $TMP_DIR/tt_issuesA.html ]
		then
			cat >> $TMP_DIR/tellTracker.html <<EOT
<table id="issuesTableA" class="tablesorter" BORDER=0 WIDTH="80%">
EOT
			cat $TMP_DIR/tt_header_issues.html  >> $TMP_DIR/tellTracker.html
			echo "<tbody>"               >> $TMP_DIR/tellTracker.html

# aggregate by OBS_ID and setup
			for OBS_ID in `cat $TMP_DIR/tt_OB_issues | awk '{print $1}' | sort -u`
			do
				for DISPLAY_SETUP in `grep $OBS_ID $TMP_DIR/tt_OB_issues | awk '{print $2}' | sort -u`
				do
					grep "$OBS_ID" $TMP_DIR/tt_issuesA.html | grep "$DISPLAY_SETUP" | head -1 >> $TMP_DIR/tellTracker.html
				done
			done

			cat >> $TMP_DIR/tellTracker.html <<EOT
  </tbody>
EOT
			cat $TMP_DIR/tt_footer_issues.html >> $TMP_DIR/tellTracker.html
			cat >> $TMP_DIR/tellTracker.html <<EOT
</table><p>
EOT
		fi

# =====================================================================================================
# 4.5.3 issues: not recoverable
# =====================================================================================================

		rm -f $TMP_DIR/tt_issuesB.html 
		cat $TMP_DIR/tt_output_issues | grep "NOK" | grep "LOST" | sed "s|^.*|<tr bgcolor=#CCCCCC>&</font></td></tr>|" > $TMP_DIR/tt_issuesB.html
		#BWo, v1.4.3: this line prevents propagation of red flag to nav bars, fixed
		#cat $TMP_DIR/tt_output_issues | grep "MISS" | grep -v "LOST-" | grep -v "LEFT" | sed "s|^.*|<tr bgcolor=#CCCCCC>&</font></td></tr>|" >> $TMP_DIR/tt_issuesB.html
		cat $TMP_DIR/tt_output_issues | grep "MISS" | grep "LOST" | sed "s|^.*|<tr bgcolor=#CCCCCC>&</font></td></tr>|" >> $TMP_DIR/tt_issuesB.html

# for calChecker pick-up
		if [ -s $TMP_DIR/tt_issuesB.html ]
                then
			ISSUE_STRING="&nbsp;"
			BGCOLOR=$LRED
# mark as ! in the tab
          		sed -i -e "s|<a href=\"tellTracker_$DATE.html\" title=\"tellTracker: telluric standards for science \[internal link]\"><font color=\"#333333\">tellTracker</font></a>|<font size=\"3\" color=red><b><a title=\"tellTracker issue, not recoverable\">\!</b></a></font>\&nbsp;&|" $TMP_DIR/tellTracker.html
			echo "tellTracker issue for $DATE, not recoverable" > $TMP_DIR/CALCHECK/tt_issue_$DATE
		else
			ISSUE_STRING="NONE"
			BGCOLOR=#CCCCCC
# no  mark
          		sed -i -e "s|<a href=\"tellTracker_$DATE.html\" title=\"tellTracker: telluric standards for science \[internal link\]\"><font color=\"#333333\">tellTracker</font></a>|<font size=\"3\"><b>\&nbsp;</b></font>\&nbsp;&|" $TMP_DIR/tellTracker.html
		fi

		cat >> $TMP_DIR/tellTracker.html <<EOT
<table>
  <tr>
    <td bgcolor=$BGCOLOR><font size=3><b>Issues discovered:</b></td>
    <td><font size=2>&nbsp;[<a href=#details>complete overview</a>]</font></td>
  </tr>
  <tr>
    <td bgcolor=$BGCOLOR><font size=2><b><i>b) not recoverable:</i></b></td>
    <td bgcolor=#eeeeee><font size=2>&nbsp;${ISSUE_STRING}</font></td>
  </tr>
</table>
<font size=2><i>
[OBs that cannot be improved by a new telluric standard since their validity has expired; <br>
depending on circumstances (e.g. MODE, GRADE) it might be reasonable to grade them C or D]</i>
</font><p>
EOT
		if [ -s $TMP_DIR/tt_issuesB.html ]
		then
			cat >> $TMP_DIR/tellTracker.html <<EOT
<table id="issuesTableB" class="tablesorter" BORDER=0 WIDTH="80%">
EOT
			cat $TMP_DIR/tt_header_issues.html  >> $TMP_DIR/tellTracker.html
			echo "<tbody>"               >> $TMP_DIR/tellTracker.html

# aggregate by OBS_ID and setup
			for OBS_ID in `cat $TMP_DIR/tt_OB_issues | awk '{print $1}' | sort -u`
			do
				for DISPLAY_SETUP in `grep "^$OBS_ID " $TMP_DIR/tt_OB_issues | awk '{print $2}' | sort -u`
				do
					grep ">$OBS_ID<" $TMP_DIR/tt_issuesB.html | grep ">$DISPLAY_SETUP<" | head -1 >> $TMP_DIR/tellTracker.html
				done
			done

			cat >> $TMP_DIR/tellTracker.html <<EOT
  </tbody>
EOT
			cat $TMP_DIR/tt_footer_issues.html >> $TMP_DIR/tellTracker.html
			cat >> $TMP_DIR/tellTracker.html <<EOT
</table>
EOT
		fi

		cat >> $TMP_DIR/tellTracker.html <<EOT
<p>
<a name="details"></a>
<hr noshade height=1>
<font size=3><b>Complete overview:</b>&nbsp;</font>
<font size=2>[<a href=#issues>issues</a>] &nbsp; 
[<a href="javascript:openWin('./tellTracker_${DATE}_details.html');" title="all possible telluric matches">all matches</a>]</font>
  &nbsp; $TOP_HTMLTAG &nbsp; $BOTTOM_HTMLTAG &nbsp; $AUTO_REFRESH_TAG

<p><i><font size=2 color=#666666>
Toggle:
  <a href="javascript:void(0);" onclick="fnShowHide(4); fnShowHide(5); fnShowHide(6);">LST+RA+DEC</a>
| <a href="javascript:void(0);" onclick="fnShowHide(7); ">AIRM</a>
| <a href="javascript:void(0);" onclick="fnShowHide(9); ">DATA_TYPE</a>
| <a href="javascript:void(0);" onclick="fnShowHide(4); fnShowHide(5); fnShowHide(6); fnShowHide(7); fnShowHide(9);">all of those</a>

<table id="completeTable" class="display" BORDER=0 WIDTH="80%">
EOT
	else

# =====================================================================================================
# 4.5.4 no issues
# =====================================================================================================

		cat >> $TMP_DIR/tellTracker.html <<EOT
<table><tr><td bgcolor=$LGRN><font size=3><b>No issues discovered, everything is fine!</td></tr></table>
<p><font size=3><b>Complete overview:</b>&nbsp;</font>
<font size=2> [<a href="javascript:openWin('./tellTracker_${DATE}_details.html');" title="all possible telluric matches">all matches</a>]</font>
$TOP_HTMLTAG &nbsp; $BOTTOM_HTMLTAG &nbsp; $AUTO_REFRESH_TAG
<p><i><font size=2 color=#666666>
Toggle:
  <a href="javascript:void(0);" onclick="fnShowHide(4); fnShowHide(5); fnShowHide(6);">LST+RA+DEC</a>
| <a href="javascript:void(0);" onclick="fnShowHide(7); ">AIRM</a>
| <a href="javascript:void(0);" onclick="fnShowHide(9); ">DATA_TYPE</a>
| <a href="javascript:void(0);" onclick="fnShowHide(4); fnShowHide(5); fnShowHide(6); fnShowHide(7); fnShowHide(9);">all of those</a>
<table id="completeTable" class="display" BORDER=0 WIDTH="80%">
EOT
	fi

# =====================================================================================================
# 4.6 overview of details (each single file)
# 4.6.1 header
# =====================================================================================================

	cat $TMP_DIR/tt_header.html >> $TMP_DIR/tellTracker.html
	echo "<tbody>" >> $TMP_DIR/tellTracker.html

# =====================================================================================================
# 4.6.2 SCI results
# =====================================================================================================

# edit tt_output2 to have proper TELL_INDEX
	if [ -s $TMP_DIR/tt_list_all_tellurics ]
	then
		for TELL in `cat $TMP_DIR/tt_list_all_tellurics_index | awk '{print $1}'`
                do
                        # wrong
			TELL_INDEX=`grep $TELL $TMP_DIR/tt_list_all_tellurics_index | awk '{print $2}'`
			sed -i -e "s/$TELL/${TELL_INDEX} /g" $TMP_DIR/tt_output2
		done
	fi

# add to tellTracker.html	
	cat $TMP_DIR/tt_output2 |\
 sed "s|^.*|<tr bgcolor=#CCCCCC>&</font></td></tr>|" \
>> $TMP_DIR/tellTracker.html

# ======================================================================
# 4.6.3 list of tellurics: to be created here, and then added
# ======================================================================

	if [ -s $TMP_DIR/tt_list_all_tellurics ]
	then
			
		for TELL in `cat $TMP_DIR/tt_list_all_tellurics | awk '{print $3}'`
		do
			TELL_INDEX=`grep $TELL $TMP_DIR/tt_list_all_tellurics_index | awk '{print $2}'`

			PROG_ID=`grep  	      $TELL $TMP_DIR/tt_list_all_tellurics | awk '{print $10}'`
			OBS_ID=`grep  	      $TELL $TMP_DIR/tt_list_all_tellurics | awk '{print $11}'`
			TELL_DATA_TYPE=`grep  $TELL $TMP_DIR/tt_list_all_tellurics | awk '{print $1}'`
			TELL_DEF_STRING=`grep $TELL $TMP_DIR/tt_list_all_tellurics | awk '{print $2}'`
			LST=`grep     $TELL $TMP_DIR/tt_list_radec | awk '{print $2}'`
			RA=`grep      $TELL $TMP_DIR/tt_list_radec | awk '{print $3}'`
			DEC=`grep     $TELL $TMP_DIR/tt_list_radec | awk '{print $4}'`
			TARG=`grep    $TELL $TMP_DIR/tt_list_radec | awk '{print $5}'`

			AIRM=`grep    $TELL $TMP_DIR/tt_list_all_tellurics | awk '{printf"%3.2f\n",($8+$9)/2}'`

			GRADE=`grep " $OBS_ID " $TMP_DIR/tt_isql_out1 | awk '{print $4}' | tail -1 | sed "s/_//"`
			if [ "Q$GRADE" = Q ]
			then
				GRADE=Y
			fi

# same colors as in createReport
			case $GRADE in
		  	"A"|"B" ) OBCOLOR=$LGRN ;;
                  	"C"|"D" ) OBCOLOR=$LRED ;;
                  	"X"|"Y" ) OBCOLOR=$GRY  ;;
                	esac

			if [ $GRADE = Y ]
			then
				GRADE=n/a
			fi

			OBCOMM=`grep " $OBS_ID " $TMP_DIR/tt_isql_out2 | grep -v "NOCOMMENT" | sed "s/^.*UT_START//" | sed "s/COMMENT//" | sed "s/\"/\&ldquo;/g" | sed "s/^.*/&YY/" | sed "s/   //g" | tr "\012" " " | sed "s/YY/ |/"g | sed "s/ | $//"`
			if [ "Q$OBCOMM" != Q ]
			then
				GRADE="<a title=\"$OBCOMM\">${GRADE}!</a>"
			fi
			GRADE="</font></td><td bgcolor=$OBCOLOR><font size=2>$GRADE"

			MODE=`grep "^$PROG_ID " $TMP_DIR/tt_out_flagquery | awk '{print $2}' | sed "s/s/<font color=${SM_COLOR}><b>SM<\/b><\/font>/" | sed "s/v/<font color=${VM_COLOR}>VM<\/font>/" | sed "s/^.*/<b>&<\/b>/"`

# check if TELL is already QC processed; if so, get score; replace .fits by  *.ab since AB name could contain _tpl
			AB=`echo $TELL | sed "s/.fits/.ab/"`
			AB1=`echo $TELL | sed "s/.fits/_tpl.ab/"`
			if [ -s $DFO_AB_DIR/${AB} ]
			then
				ABNAME=${AB}
				ABDIR=$DFO_AB_DIR
			elif [ -s $DFO_AB_DIR/${AB1} ]
			then
				ABNAME=${AB1}
				ABDIR=$DFO_AB_DIR
			elif [ -s $DFO_LOG_DIR/$DATE/$AB ]
			then
				ABNAME=${AB}
				ABDIR=$DFO_LOG_DIR/$DATE
			elif [ -s $DFO_LOG_DIR/$DATE/$AB1 ]
			then
				ABNAME=${AB1}
				ABDIR=$DFO_LOG_DIR/$DATE
			else
				ABNAME=none
				SCORE="..."	
			fi

			if [ $ABNAME != none ]
			then
				TLOG=`echo $ABNAME | sed "s/.ab/.tlog/"`
				SCORE_HTML=`echo $ABNAME | sed "s/.ab/.html/"`
				if [ -s $ABDIR/$TLOG ]
				then
					TOTAL_SCORE=`grep "^TOTAL_SCORE"   $ABDIR/$TLOG | awk '{print $2}'`
					TOTAL_NUMBER=`grep "^TOTAL_NUMBER" $ABDIR/$TLOG | awk '{print $2}'`
                                        SCORE="<a href=${QCWEB_URL}/logs/$DATE/${SCORE_HTML} title=\"QC processing score\">$TOTAL_SCORE/$TOTAL_NUMBER</a>"
				else
					SCORE="..."
					TOTAL_SCORE=""
				fi
			fi

			case $TOTAL_SCORE in
			 "0" ) SCORE_COL=$GRN ;;
			 "1" ) SCORE_COL=$YLW ;;
			  *  ) SCORE_COL=$RED ;;
			esac
			if [ "Q$TOTAL_SCORE" = Q ] || [ "Q$SCORE" = "Q..." ]
			then
				SCORE_COL=#CCCCCC
			fi

			TELL="<table><tr><td><font size=1>$TELL</font></td><td bgcolor=$SCORE_COL><font size=1>$SCORE</font></td></tr></table>"

# output
			echo "<tr bgcolor=#eeeeee>${TD0}${PROG_ID}${TD}$MODE${TD}$OBS_ID$GRADE${TD}$LST${TD}$RA${TD}$DEC${TD}$AIRM${TD}$TELL${TD}${TELL_DATA_TYPE}${TD}$TELL_DEF_STRING${TD_SEP2}${TD}&nbsp;${TD2}$TARG${TD2}&nbsp;${TD2}${TELL_INDEX} ${TD2}&nbsp;</font></td></tr>" | sed "s|size=2|size=1|g"  >> $TMP_DIR/tellTracker.html
		done
	fi

# =====================================================================================================
# 4.6.4 footer (indentation correct from here on)
# =====================================================================================================

		cat >> $TMP_DIR/tellTracker.html <<EOT
  </tbody>
EOT
		cat $TMP_DIR/tt_footer.html >> $TMP_DIR/tellTracker.html

# =====================================================================================================
# 4.6.5 signature
# =====================================================================================================

		cat >> $TMP_DIR/tellTracker.html <<EOT
</table>
</font></i>

<table width=100%>
<tr bgcolor=#CCCCCC>
  <td><a href=#top title="jump to top of page"><b><font color=${REPORT_COLOR} size=2>top</font></b></td>
  <td COLSPAN=99 align=right><font size=2 color=#FFFFFF><b>powered by QC</b> [$TOOL_NAME v${TOOL_VERSION}]</font></td>
</tr>
</table>

<a name=bottom></a>
</body>
</html>
EOT
	fi

# end of jump for no headers
fi

# =====================================================================================================
# 5. scp results
# 5.1 check for ${QCWEB_TARG_DIR}
#     for performance, this is done once, and then a flag is written into config.tellTracker
# =====================================================================================================

CHECK_EXIST=`grep "^TELLTRACK_EXIST" $DFO_CONFIG_DIR/config.tellTracker | awk '{print $2}'`
if [ Q$CHECK_EXIST != QYES ]
then
	ssh -o BatchMode=yes ${DFO_WEB_SERVER} "bin/webDircheck ${INSTR_DIR}/${QCWEB_TARG_DIR}" 1>/dev/null
	cat >> $DFO_CONFIG_DIR/config.tellTracker <<EOT

# DO NOT REMOVE: auto-added by tellTracker!
# ======================================================================================
TELLTRACK_EXIST	YES	# flag to indicate that ${INSTR_DIR}/${QCWEB_TARG_DIR} exists on ${DFO_WEB_SERVER}

EOT
fi

# =====================================================================================================
# 5.2 create and scp page for tomorrow
# =====================================================================================================

if [ $DATE = $TODAY_DFO ]
then
	createTomorrow
	scp -r -o BatchMode=yes $TMP_DIR/tellTracker_tomorrow.html ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}/tellTracker_${TOMORROW}.html 1>/dev/null 2>/dev/null
fi

# =====================================================================================================
# 5.3 scp HTML
# =====================================================================================================

if [ $VERBOSE = YES ]
then
	scp -r -o BatchMode=yes $TMP_DIR/tellTracker.html ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}/tellTracker_$DATE.html
	if [ -s $TMP_DIR/tellTracker_${DATE}_details.html ]
	then
		scp -r -o BatchMode=yes $TMP_DIR/tellTracker_${DATE}_details.html ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}/
	fi
else
	scp -r -o BatchMode=yes $TMP_DIR/tellTracker.html ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}/tellTracker_$DATE.html 1>/dev/null 2>/dev/null
	if [ -s $TMP_DIR/tellTracker_${DATE}_details.html ]
	then
		scp -r -o BatchMode=yes $TMP_DIR/tellTracker_${DATE}_details.html ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}/ 1>/dev/null 2>/dev/null
	fi
fi

# =====================================================================================================
# 5.4 DATE=TODAY: scp version without DATE for reference
# =====================================================================================================

if [ $DATE = $TODAY_DFO ]
then
	scp -r -o BatchMode=yes $TMP_DIR/tellTracker.html ${DFO_WEB_SERVER}:${INSTR_DIR}/${QCWEB_TARG_DIR}/tellTracker.html 1>/dev/null 2>/dev/null
fi

# =====================================================================================================
# 6. extract exectime and write into QC1 DB (taken from calChecker)
#    we do this only for operational runs ($DATE=$TODAY)
# =====================================================================================================

if [ $DATE != $TODAY_DFO ]
then
	exit 0
fi

END_TIME=`date -u +%Y-%m-%d"T"%H:%M:%S`

PROCESS_LOAD=`uptime | sed "s/^.*load average://" | awk '{print $1}' | sed "s/,//"`
START_MIN=`echo $START_TIME | sed "s/:/ /g" | sed "s/T/ /" | awk '{print $2*60.+$3+$4/60.}'`
DATE1=`echo $START_TIME | sed "s/T/ /" | awk '{print $1}'`

END_MIN=`echo $END_TIME | sed "s/:/ /g" | sed "s/T/ /" | awk '{print $2*60.+$3+$4/60.}'`
DATE2=`echo $END_TIME | sed "s/T/ /" | awk '{print $1}'`

if [ $DATE1 = $DATE2 ]
then
	TEXEC=`echo $END_MIN $START_MIN | awk '{print $1-$2}'`
else
	TEXEC=`echo $END_MIN $START_MIN | awk '{print $1+1440-$2}'`
fi

# define a number of further parameters and ingest all into the QC1 exec_time table
CIVIL_DATE=`echo $DATE1 | sed "s/-//g"`
LOCAL_TIME=`echo $START_TIME | sed "s/-/ /g" | sed "s/T/ /g" | awk '{print $4}'`

echo "qc1Ingest -table exec_time -civil_date $CIVIL_DATE -local_time $LOCAL_TIME -mjd_obs $TODAY_MJD2 -exec_time $TEXEC -version $TOOL_VERSION -instrument $DFO_INSTRUMENT -hostname $DFO_MACHINE -tool $TOOL_NAME -process_load $PROCESS_LOAD" >> $DFO_TREND_DIR/dfo_exec_time

qc1Ingest -table exec_time -civil_date $CIVIL_DATE -local_time $LOCAL_TIME -mjd_obs $TODAY_MJD2 -exec_time $TEXEC -version $TOOL_VERSION -instrument $DFO_INSTRUMENT -hostname $DFO_MACHINE -tool $TOOL_NAME -process_load $PROCESS_LOAD | grep -v "Connecting to" | grep -v "duplicated rows" | grep -v "Successfully ingested" &

# end
exit
