#!/bin/sh
# PURPOSE:      histoMonitor: collects historical dfo information
# AUTHOR:       Reinhard Hanuschik / ESO
# DATE:         2007-12-12
# VERSIONS:     1.0 -- released
#		1.1 -- daily comments are ingested into DFO db (daily_comments) (2008-03-17)
#		1.1.1- improved isql output format (2008-04-07)
#		1.1.2- bug fixed with comments for not yet finished dates (2008-05-23)
#		1.1.3- bug fixed with navbar links (2008-06-20) [BWo]
#		1.1.4- bug fixed with replacing gif/png links (line 975) (2008-09-17)
#		1.2 -- scp to $DFO_WEB_SERVER (2009-01-08)
#		1.2.1- now calls getStatusAB -H for the final AB report (2009-03-27)
#		1.2.2- auto-updates monthly navigation (current year) if new month created; main page: links edited on safweb1 version; comment_histoMonitor.esh: updates data report as well (2009-09-07)
#		2.0 -- includes packages (2009-10-02)
#		2.0.1- supports partial VM packages (line 931) (2010-04-23) [SMo]
#		2.1 -- improved handling of nights without ABs (now we check for "no data found") (2010-07-30)
#		2.1.1- $DFO_EDITOR replaces hardcoded vi session (2011-04-27)
#		2.1.2- bug fixed with option -d (2011-05-04)
#		2.1.3- prepare for end of pipacks (2011-11-05)
#		2.1.4- replace observations by obs_metadata database (2012-08-24)
#		2.1.5- replace NR link by NLT link (2012-10-02)
#		2.1.6- turned off createNightLog ; bug fix with flag sci_Ingested (2012-10-27)
#		2.1.7- mail sent if new year starts (2013-01-04)
#		2.1.8- NR/NLT distinction re-established; CAL/SCI ingested flags improved (2013-03-18)
#		2.2 -- PHOENIX signal sent if configured (2013-08-02)
#		2.2.1- replace 'switch' by 'change' in awk commands (for OS upgrade) (2014-02-18)
#		2.2.2- fix a bug with this fix (2014-03-10)
#		2.2.3- label "science ABs created and verified" (2015-06-22)
#		2.3 -- have optional file info.histoMonitor in $DFO_CONFIG_DIR (2017-04-10)
#		2.3.1- simplified handling of vnavbar_histoMonitor.html (2017-05-09)
#		2.3.2- handling of NLT_APPLICABLE adapted to getObInfo v2.0 (2017-07-21)
#		2.3.3- added leap days for 2020-02 ... 2028-02 (2020-03-02)
#		2.4 -- decoupled from PHOENIX (2020-10-22)
#
# CONFIG:	config.histoMonitor and config.dfoMonitor (for PHOENIX keys)
#		info.histoMonitor (with optional specific text)
# PARAMETERS:	-y|-m|-d for incremental refreshes; 
#		-p for package refresh
#		-A to enforce AB monitor creation
# TOOLS CALLED: qcdate; getStatusAB; ngasClient; qc1Ingest
# OUTPUT:	histoMonitor.html and related pages under $DFO_MON_DIR/FINISHED; 
#		exported to $DFO_WEB_SERVER
# ==============================================================================
TOOL_VERSION="2.4"

# ===================================================================================
# 0. Initialization
# ===================================================================================

IMG_URL="http://www.eso.org/qc/ALL/img"        	   # URL for images
IMG_URL1="http://www.eso.org/~qc/img"

INSTR_DIR="/home/qc/public_html/${DFO_INSTRUMENT}" # on $DFO_WEB_SERVER
QCBIN_DIR="/home/qc/bin"        		   # on $DFO_WEB_SERVER

WEB_SERVER_URL="http://qcweb.hq.eso.org/~qc"     # QC web server
SCHED_REP_URL="http://archive.eso.org/wdb/wdb/eso/sched_rep_s/query/?tab_period=on&tab_tel=on&tab_progid=on&tab_instrument=on&tab_pi_coi=on&progid="

NLT_URL="http://www.eso.org/gnlt/qcReport/instrumentReport"     # URL for NLT

# list  of keys for custom interface WISQ pack_stat
echo "pack_period
civil_date
instrument
run_ID
N_media
T_media
N_nights
N_OBs
first_night
last_night
N_SCI_RAW
N_CAL_RAW
N_SCI_PRO
N_CAL_PRO
N_CAL_GEN
N_AB" > $TMP_DIR/hm_qc1_keys
KEY_LIST=`cat $TMP_DIR/hm_qc1_keys | sed "s/^.*/\&field_&=&/" | tr "\012" "Y" | sed "s/Y\&/\&/g" | sed "s/Y$//"`

WISQ_URL="http://archive.eso.org/bin/qc1_cgi?table=pack_stat"
CUSTOM_URL="${WISQ_URL}&action=qc1_browse_get&filter_instrument=${DFO_INSTRUMENT}${KEY_LIST}&from=1999-01-01&to=2099-01-01&output=html&sum=true&number=true"

LOG_DIR=/~qc/${DFO_INSTRUMENT}/logs		   # on $DFO_WEB_SERVER
PLT_DIR=/~qc/${DFO_INSTRUMENT}/plots		   # on $DFO_WEB_SERVER
MON_DIR=/~qc/${DFO_INSTRUMENT}/monitor		   # on $DFO_WEB_SERVER
LST_DIR=/~qc/${DFO_INSTRUMENT}			   # on $DFO_WEB_SERVER
PACK_DIR=/~qc/${DFO_INSTRUMENT}/PACK		   # on $DFO_WEB_SERVER

SM_COLOR="#006A9D"      #eso-blue:  color for SM nights
VM_COLOR="#666666"      #dark_grey: color for VM nights
TITLE_COLOR="#006A9D"   #eso-blue:  font color
REPORT_COLOR="#FF6600"  #orange:    color for comment link

PACK_COLOR="#CCCCFF"	#light blue: color for packages
PACK_COLOR2="#E6F7FF"	#color for packages (1st row)
PACK_COLOR3="#EEEEEE"	#color for packages (2nd row)
PACK_COLOR4="#FFFFCC"	#color for VM packages 

TABLE_WIDTH=70%

basename `echo $DFO_PLT_DIR` > $TMP_DIR/hm_pltdir
PLT_DIR_TAG=`cat $TMP_DIR/hm_pltdir`

if [ "Q$DFO_WEB_SERVER" = "Q" ]
then
        echo "*** WARNING: \$DFO_WEB_SERVER not defined, update your .dfosrc."
fi

PHOENIX_ENABLED=`grep "^PHOENIX_ENABLED" $DFO_CONFIG_DIR/config.dfoMonitor | awk '{print $2}'`
PHOENIX_ACCOUNT=`grep "^PHOENIX_ACCOUNT" $DFO_CONFIG_DIR/config.dfoMonitor | awk '{print $2}'`

if [ Q$PHOENIX_ENABLED != QYES ]
then
	PHOENIX_ENABLED=NO
fi

# =========================================================================
# 0.2 help and version
# =========================================================================

YSELECT_MODE=NO
MSELECT_MODE=NO
DSELECT_MODE=NO

CREATE_REPORT=NO
REFRESH_MONITOR=NO
ALL_PERIOD_SELECT=NO

while getopts APd:m:p:y:hv OPTION
do
        case "$OPTION" in
         h ) cat $DFO_DOC_DIR/histoMonitor.h 
             exit 0 ;;
         v ) echo "$TOOL_VERSION"
             exit 0 ;;

         y ) YEAR_SELECT=$OPTARG  ; YSELECT_MODE=YES ; CREATE_REPORT=YES ;;
         m ) MONTH_SELECT=$OPTARG ; MSELECT_MODE=YES ; CREATE_REPORT=YES ;;
         d ) DATE_SELECT=$OPTARG ;  DSELECT_MODE=YES ; CREATE_REPORT=YES ;;

	 p ) PERIOD_SELECT=$OPTARG ;;
	 P ) ALL_PERIOD_SELECT=YES ;;

	 A ) REFRESH_MONITOR=YES ;;
         * ) cat $DFO_DOC_DIR/histoMonitor.h; exit -1 ;;
        esac
done

if [ Q$PERIOD_SELECT != Q ]
then
	CHECK_PERIOD1=`echo $PERIOD_SELECT | cut -c1,1`
	CHECK_PERIOD2=`echo $PERIOD_SELECT | wc -c`
	if [ $CHECK_PERIOD1 != P ] || [ $CHECK_PERIOD2 != 4 ]
	then
		echo "***ERROR: parameter -p must come in format 'Pnn'; is: $PERIOD_SELECT. Exit."
		exit -1
	else
		PERIOD=$PERIOD_SELECT
	fi

	if [ ! -d $DFO_SRV_DIR/FINISHED/$PERIOD ]
	then
		echo "***ERROR: \$DFO_SRV_DIR/FINISHED/$PERIOD not found. Prepare and then try again."
		exit
	fi
fi

if [ $# = 0 ]
then
	 echo "We update all navigation bars for histoMonitor (AB monitors are not refreshed).
Hit return:" 
	read input
fi
	
if [ $REFRESH_MONITOR = YES ] \
  && ( [ $YSELECT_MODE = YES ] || [ $MSELECT_MODE = YES ] )
then
	echo "All AB monitor pages for the selected range will be refreshed. Do you really want this (Y/N) (N)?"
	read REFRESH_YN
	if [ "Q$REFRESH_YN" != "QY" ]
	then
		echo "Think about."
		exit
	fi
fi

# for DSELECT_MODE=YES, refresh is made anyway
if [ $REFRESH_MONITOR = YES ] && [ $DSELECT_MODE = YES ]
then
	REFRESH_MONITOR=NO
fi

LAST_DFO_DATE=`cat $DFO_MON_DIR/DFO_STATUS | grep raw_Complete | awk '{print $2}' | sort | tail -1`
LAST_DFO_DATE1=`qcdate $LAST_DFO_DATE +1`

# ===============================================================================
# 0.3 get configuration
# ===============================================================================

SCI_MAX=`grep "^SCI_MAX"	$DFO_CONFIG_DIR/config.histoMonitor | awk '{print $2}'`
SCI_SCALE=`grep "^SCI_SCALE"	$DFO_CONFIG_DIR/config.histoMonitor | awk '{print $2}'`

# ===============================================================================
# 0.4 job to edit comment file
# maybe the text file becomes obsolete with the DFO_db table!
# ===============================================================================

cat > $DFO_GUI_DIR/comment_histoMonitor.esh <<EOT
echo "Enter DATE for comment (e.g. 2007-03-31):"
read CDATE
CYEAR=\`echo \$CDATE | cut -c1-4\`
CYM=\`echo \$CDATE | cut -c1-7\`
MJD_OBS=\`qcdate \$CDATE\`

if [ ! -d $DFO_LOG_DIR/\$CDATE ]
then
	mkdir $DFO_LOG_DIR/\$CDATE
fi

cat > $TMP_DIR/hm_qcdb_query <<END
SELECT
        convert(varchar(5),id) + char(10) + comment
FROM
        qc1..daily_comments
WHERE
        instrument = "$DFO_INSTRUMENT"
AND
        civil_date = "\$CDATE"
AND
        visible = "Y"
GO
END
rm -f $TMP_DIR/hm_qcdb_results

isql -Uqc -P\`cat $QC1_PWD\` -S${QC1_SERVER} -w999 -i $TMP_DIR/hm_qcdb_query |\
  sed "1,2d" |\
  grep -v affected |\
  sed "/^$/d" |\
  sed "/^   $/d" |\
  sed "s/    //g" \
  > $TMP_DIR/hm_qcdb_results

if [ -s $TMP_DIR/hm_qcdb_results ]
then
	ENTRY_ID=\`cat $TMP_DIR/hm_qcdb_results | head -1 | awk '{print \$1}'\`
	cat $TMP_DIR/hm_qcdb_results | sed "1,1 d" > $DFO_LOG_DIR/\$CDATE/comment_\$CDATE.txt
else
	ENTRY_ID=""
fi

echo "Now we open a '${DFO_EDITOR}' session, enter the comment (one line, 256 chars max.) (hit return):"
read input
eval "$DFO_EDITOR $DFO_LOG_DIR/\$CDATE/comment_\$CDATE.txt"

if [ ! -d $DFO_MON_DIR/FINISHED/\$CYEAR ]
then
	mkdir $DFO_MON_DIR/FINISHED/\$CYEAR
fi

if [ ! -d $DFO_MON_DIR/FINISHED/\$CYEAR/\$CYM ]
then
	mkdir $DFO_MON_DIR/FINISHED/\$CYEAR/\$CYM
fi

cp $DFO_LOG_DIR/\$CDATE/comment_\$CDATE.txt $DFO_MON_DIR/FINISHED/\$CYEAR/\$CYM/
CHECK_LENGTH=\`cat $DFO_LOG_DIR/\$CDATE/comment_\$CDATE.txt  | awk '{print length}'\`
if [ \$CHECK_LENGTH -gt 255 ]
then
	echo "***ERROR: your text is too long. It is saved in $DFO_LOG_DIR/\$CDATE/comment_\$CDATE.txt but not ingested."
	echo "Launch again this comment and edit. Hit return:"
	read input
	exit
fi

rm -f $TMP_DIR/hm_delete $TMP_DIR/tmp_comment
if [ -s $TMP_DIR/hm_qcdb_results ]
then
	echo "delete from qc1..daily_comments where id = \$ENTRY_ID" > $TMP_DIR/hm_delete
	echo "go" >> $TMP_DIR/hm_delete
	isql -Uqc -P\`cat $QC1_PWD\` -S${QC1_SERVER} -w999 -i $TMP_DIR/hm_delete | grep -v affected
	echo "Old entry \$ENTRY_ID deleted."
fi		

# rm apostrophs (make qc1Ingest failing)
cat $DFO_LOG_DIR/\$CDATE/comment_\$CDATE.txt | tr "\012" " " | sed "s/^.*/&\n/" | sed "s/'//" > $TMP_DIR/tmp_comment

qc1Ingest -table daily_comments -civil_date \$CDATE -mjd_obs \$MJD_OBS -year \$CYEAR -month \$CYM -instrument $DFO_INSTRUMENT -comment "\`cat $TMP_DIR/tmp_comment\`"

echo "Refresh histoMonitor for \$CYM, and data plus AB report for \$CDATE, to make comment visible (Y/N) (N) ?"
read REFRESH_YN
if [ \$REFRESH_YN = Y ]
then
	histoMonitor -m \$CYM
	echo "Calling 'createReport -d \$CDATE' ..."
	createReport -d \$CDATE
	echo ""
	echo "calling 'getStatusAB -d \$CDATE -H ..."
	getStatusAB -d \$CDATE -H
fi
exit
EOT
chmod u+x $DFO_GUI_DIR/comment_histoMonitor.esh

# ===============================================================================
# 0.5 procedure getEntry to read comment from daily_comments
# ===============================================================================

getEntry(){
cat > $TMP_DIR/hm_comment_query <<END
SELECT
        comment
FROM
        qc1..daily_comments
WHERE
        instrument = "$DFO_INSTRUMENT"
AND
        civil_date = "$DATE"
AND
        visible = "Y"
GO
END
rm -f $TMP_DIR/hm_comment_results

isql -Uqc -P`cat $QC1_PWD` -S${QC1_SERVER} -w999 -i $TMP_DIR/hm_comment_query |\
  sed "1,2d" |\
  grep -v affected |\
  sed "/^$/d" |\
  sed "/^   $/d" |\
  tr "\012" " "
}

# ==============================================================================
# 0.6 procedure to download headers and extract report
# strategy: report found -> skipped
#           not found and option -A set -> 
#		check for hdr files; all found: OK
#		hdr not found -> try to download headers (ngasClient -H)
#		still not found -> download fits, extract hdr, delete fits
#	    then create Report; hdr are saved, to be used next time 
# ==============================================================================

getHeaders (){
MJD1=`qcdate $DATE | awk '{printf"%10.5f\n",$1+off/24+0.50}' off=$DFO_OFFSET `
MJD2=`qcdate $DATE | awk '{printf"%10.5f\n",$1+off/24+1.50}' off=$DFO_OFFSET `

cat > $TMP_DIR/hm_raw_query <<EOT
SELECT 
        dp_id
FROM
        obs_metadata..data_products
WHERE
	mjd_obs between $MJD1 and $MJD2
AND
        ins_id = "$DFO_INSTRUMENT"
GO
EOT

rm -f $TMP_DIR/hm_raw_out
isql -Uqc -S${ARCH_SERVER} -P`cat $ARCH_PWD` -w 999 -i $TMP_DIR/hm_raw_query |\
 sed "1,2 d" |\
 grep -v affected |\
 sed "/^$/d" \
 > $TMP_DIR/hm_raw_out

N_RAW=`cat $TMP_DIR/hm_raw_out | wc -l`
if [ $N_RAW -gt 0 ]
then
	cd $DFO_HDR_DIR
	if [ ! -d $DFO_HDR_DIR/$DATE ]
        then
                mkdir $DFO_HDR_DIR/$DATE
        fi
        cd $DATE
	echo "      $DATE: check for headers; if missing, attempt to download updated headers ..."

        for R in `cat $TMP_DIR/hm_raw_out`
        do
		if [ ! -s $R.hdr ]
		then
			ngasClient -H $R.hdr 2>&1 1>/dev/null
		fi
		if [ -s $R.hdr ]
		then
			echo "      $R.hdr OK."
		else
			echo "      $R.hdr NOK! We download fits file and extract header from there ..."
			ngasClient -f $R.fits 2>&1 1>/dev/null
			if [ -s $R.fits ]
			then
				dfits $R.fits | sed "/====> file/d" > $R.hdr
				rm $R.fits
				echo "        $R.hdr OK (extracted from fits file, not updated)."
			else
				echo "        $R.hdr NOK! We give up."
			fi
		fi	
	done
fi
}

# ==============================================================================
# 0.7 procedure scpMain: scopy main page to $DFO_WEB_SERVER
# ==============================================================================

scpMain(){
if [ Q$DFO_WEB_SERVER != Q ]
then
	echo "Exporting to ${DFO_WEB_SERVER} ..."
	cat $DFO_MON_DIR/FINISHED/histoMonitor.html |\
	 grep -v "<!--LOCAL-->" \
	 > $TMP_DIR/histoMonitor.html
	 
        scp -o BatchMode=yes $TMP_DIR/histoMonitor.html $DFO_MON_DIR/FINISHED/nautilus.html ${DFO_WEB_SERVER}:${INSTR_DIR}/monitor/FINISHED 
fi
}

# ==============================================================================
# 1. HTML 
# 1.1 prepare
# ==============================================================================

rm -f $TMP_DIR/list_all_periods $TMP_DIR/list_all_dates

if [ ! -d $DFO_MON_DIR/FINISHED ]
then
	mkdir $DFO_MON_DIR/FINISHED
fi

# list of finished periods
ls -d $DFO_SRV_DIR/FINISHED/P[0-9][0-9]* > $TMP_DIR/list_all_periods

# list of finished dates
cd $DFO_LOG_DIR
ls -d 20* > $TMP_DIR/list_all_dates

cat $TMP_DIR/list_all_dates | cut -c1-4 | sort -u > $TMP_DIR/all_years

cat > $TMP_DIR/all_months <<EOT
01
02
03
04
05
06
07
08
09
10
11
12
EOT

cat > $TMP_DIR/all_days <<EOT
01
02
03
04
05
06
07
08
09
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
EOT

MAIN_HISTO=$DFO_MON_DIR/FINISHED/histoMonitor.html
TIMESTAMP=`date -u +%Y-%m-%d"T"%H:%M:%S`

cd $DFO_MON_DIR/FINISHED

# ==============================================================================
# 1.2 create histoMonitor.html
# ==============================================================================

if [ $ALL_PERIOD_SELECT = YES ]
then
	echo "Refresh the complete package history."
	echo "For practical reasons, we first create the main page and then proceed with the package pages. Hit return:"
	read input
fi

echo "
Creating main page \$DFO_MON_DIR/FINISHED/histoMonitor.html ..."

cat > $MAIN_HISTO <<EOT
<html>
<!-- created by histoMonitor v${TOOL_VERSION} -->
<head>
<title>History monitor (instrument: $DFO_INSTRUMENT)</title>
</head>

<body>
<table BORDER=0 WIDTH="$TABLE_WIDTH" class=main>
  <tr VALIGN=TOP BGCOLOR=#CCCCCC>
    <td width=5% rowspan=2>
<!--PLACEHOLDER-->
EOT

cat > nautilus.html <<EOT
<html>
<table width=30% bgcolor=#CCCCCC>
  <tr>
    <td width=1%><img src=${IMG_URL1}/nautilus.gif border=0 align=top></td>
    <td valign=top><font size=1>&copy; www.edition-nautilus.de</td>
  </tr>
  <tr>
    <td COLSPAN=2><font size=2>
"Nautilus is the common name of any marine creatures of the cephalopod family <b>Nautilidae</b>, the sole family of the suborder <b>Nautilina</b>" (says <a href="http://en.wikipedia.org/wiki/Nautilus">Wikipedia</a>).</td>
  </tr>
  <tr>
    <td COLSPAN=2><font size=2> This logo has been chosen for the <em>histoMonitor</em> not only because of its nice association to a biological phenomenon lasting from the ancient past until today. It also features nicely a mathematical symbol (the logarithmic spiral) which may remind of more and more layers (years) of (data) history behind the current surface.<br><br>
[ <a href=histoMonitor.html>back</a> ]
    </td>
  </tr>
</table>
</html>
EOT

# ===================================================================================
# 1.3 Include the vnavbar: simplified with v2.3.1
# ===================================================================================

echo "<font size=2><b>back to:</b><br><a href=$DFO_MON_DIR/dfoMonitor.html>dfoMonitor</a></font> <!--LOCAL-->" >> $MAIN_HISTO

# ===================================================================================
# 1.4 Title etc.
# ===================================================================================

cat >> $MAIN_HISTO <<EOT
    </td>

<!-- title, info -->
    <td width=90% COLSPAN=2>
    <b><font size=+2 color=$TITLE_COLOR> History monitor (instrument: $DFO_INSTRUMENT) </font></b><br>
    <font size=2><tt>last update: $TIMESTAMP UT</tt>
    </b></td>
  </tr>

  <tr VALIGN=TOP BGCOLOR=#CCCCCC>
  <td width=80%>
    <font size=2>This is the status page for the $DFO_INSTRUMENT dfos processing.<p> 
It is based on existing ABs in \$DFO_LOG_DIR directories. For any date with ABs, the monitor displays:<br>
- an overview of file and AB statistics,<br>
- colour coding for nights with science data, SM/VM flags,<br>
- the product ingestion status for mcalibs and science products,<br>
- links to existing nightlogs and data reports,<br>
- links to  the <i>reconstructed</i> AB monitor, with both science and calibration data.<p>
The AB monitor links to the logs, plots and scoring files (if existing).<p>
On the monthly pages, you have the option to enter comments about the night which will then be displayed.
EOT

# add info.histoMonitor if existing
if [ -s $DFO_CONFIG_DIR/info.histoMonitor ]
then
	echo "<hr><p>" >> $MAIN_HISTO
	grep -v "^#" $DFO_CONFIG_DIR/info.histoMonitor >> $MAIN_HISTO
fi

cat >> $MAIN_HISTO <<EOT
    </td>
    <td width=1%><img src=${IMG_URL1}/nautilus.gif border=0 align=top><br>
[<font size=2><a href="nautilus.html" title="more about nautilus">more</a>]
    </td>
  </tr>

  <tr VALIGN=TOP>
    <td COLSPAN=3 bgcolor=#FFFFFF>

<table cellspacing="1" border=0>
  <tr>
    <td> <font size=2> <p>
      <hr noshade>
    </td>
  </tr>

<!-- DFO history table -->
  <tr>
    <td>
      <font size=2><b>1. DFO history</b><p>

      <table cellspacing="1" border=0>
EOT

# ===================================================================================
# 1.5 Year by year = row by row
# ===================================================================================

for Y in `cat $TMP_DIR/all_years`
do
	CHECK_EXIST=`grep $Y $TMP_DIR/list_all_dates`
	if [ "Q$CHECK_EXIST" = "Q" ]
	then
		echo "<tr bgcolor=#CCCCCC><td><font size=2>$Y</td></tr>" >> $MAIN_HISTO
	else
		rm -f $TMP_DIR/list_yearmonths
		echo "<tr bgcolor=#33FF33><td><font size=2>$Y</td>" >> $MAIN_HISTO
    		echo "<td bgcolor=#FFFFFF><font size=2>&nbsp;</font></td>" >> $MAIN_HISTO

		cd $DFO_LOG_DIR
		ls -d ${Y}* | cut -c6-7 > $TMP_DIR/list_yearmonths
		for M in `cat $TMP_DIR/all_months`
		do
			MONTH=`echo $YM | cut -c6-7`
			CHECK_EXIST=`grep "$M" $TMP_DIR/list_yearmonths`
			if [ "Q$CHECK_EXIST" = "Q" ]
			then
				echo "<td bgcolor=#FFFFFF><font size=2 color=#FFFFFF>$Y-$M</td>" >> $MAIN_HISTO
			else
				echo "<td bgcolor=#33FF33><font size=2><a href=\"$Y/${Y}-${M}/${Y}-${M}.html\">$Y-$M</td>" >> $MAIN_HISTO
			fi
		done
		echo "</tr>" >> $MAIN_HISTO
	fi
done

cat >> $MAIN_HISTO <<EOT
     </table>
    </td>
  </tr>

<!-- Package history table -->
  <tr>
    <td> <font size=2> <p>
      <hr noshade>
    </td>
  </tr>
  <tr><td>
    <font size=2><b>2. Package history</b><p>
      <table cellspacing="1" border=0>
EOT

# ===================================================================================
# 1.6 Package history
# ===================================================================================

if [ ! -s $TMP_DIR/list_all_periods ]
then
	echo "***WARNING: No P[0-9][0-9] directories found under \$DFO_SRV_DIR/FINISHED. Can't do package overview."
	echo "<tr bgcolor=#FFFFFF>" >> $MAIN_HISTO
	echo "<td><font size=2> none found </td>" >> $MAIN_HISTO
	echo "</tr>" >> $MAIN_HISTO
else

	for P in `cat $TMP_DIR/list_all_periods`
	do
		P1=`basename $P`
		CHECK_EXIST=`grep "^$P1" $DFO_CONFIG_DIR/dates.histoMonitor`
		if [ "Q$CHECK_EXIST" = Q ]
		then
			LAST_DIGIT=`echo $P1 | cut -c3,3`
			NEW=`grep   "^P.$LAST_DIGIT" $DFO_CONFIG_DIR/dates.histoMonitor | tail -1 | awk '{print $4}'`

			echo "[histoMonitor] New period found which is not yet included in \$DFO_CONFIG_DIR/dates.histoMonitor. 
Pls enter YEAR and MONTH labels corresponding to $P1 (other similar entries are displayed below):"
			grep "^P.$LAST_DIGIT" $DFO_CONFIG_DIR/dates.histoMonitor
			echo "
Enter YEAR for $P1:"
			read YEAR
			echo "
Enter MONTH label for $P1:"
			read MONTH
			echo "$P1	$MONTH $YEAR	$NEW" >> $DFO_CONFIG_DIR/dates.histoMonitor
		fi

		MONTH=`grep "^$P1" $DFO_CONFIG_DIR/dates.histoMonitor | awk '{print $2}'`
		YEAR=`grep  "^$P1" $DFO_CONFIG_DIR/dates.histoMonitor | awk '{print $3}'`
		NEW=`grep   "^$P1" $DFO_CONFIG_DIR/dates.histoMonitor | awk '{print $4}'`

		if [ Q$NEW = QNEW ]
		then
			cat >> $MAIN_HISTO <<EOT
  <tr bgcolor=#FFFFFF>
    <td bgcolor=${PACK_COLOR}><font size=2>$YEAR</font></td>
    <td bgcolor=#FFFFFF><font size=2>&nbsp;</font></td>
    <td bgcolor=${PACK_COLOR}><font size=2><a href=$P1/$P1.html title="$MONTH $YEAR">${P1}</a></font></td>
EOT
 		else 
			cat >> $MAIN_HISTO <<EOT
    <td bgcolor=${PACK_COLOR}><font size=2><a href=$P1/$P1.html title="$MONTH $YEAR">${P1}</a></font></td>
  </tr>
EOT
		fi
	done
fi

echo "
  </tr>
</table>

  </td></tr>

  <tr>
  </table></html>" >> $MAIN_HISTO

cd $DFO_MON_DIR/FINISHED

echo " ... main page created."

scpMain

# ==============================================================================
# 2. PERIOD.html page
# 2.1 selected PERIOD page
# ==============================================================================

if [ Q$PERIOD != Q ]
then
	echo "Collect packing information for period $PERIOD ..."

	cd $DFO_SRV_DIR/FINISHED/$PERIOD
	if [ ! -d $DFO_MON_DIR/FINISHED/$PERIOD ]
	then
		mkdir $DFO_MON_DIR/FINISHED/$PERIOD
	fi
	HISTO_PERIOD="$DFO_MON_DIR/FINISHED/$PERIOD/$PERIOD.html"
	RANGE=`grep "^$PERIOD" $DFO_CONFIG_DIR/dates.histoMonitor | awk '{print $2,$3}'`

	cat > $HISTO_PERIOD <<EOT
<html>
<table cellspacing="1" border=0 width=50%>
  <tr>
    <td COLSPAN=10>
<h2>${DFO_INSTRUMENT} package record: ${PERIOD}</h2>
<font size=2>formal date range: $RANGE</font> <p>
<br>
<font size=2>pack_stat (WISQ): <a href=${CUSTOM_URL}&filter_pack_period=$PERIOD title="customized results for period $PERIOD and $DFO_INSTRUMENT" target=_blank>this period and instrument</a>
| <a href="${WISQ_URL}&action=qc1_browse_table" title="general browser interface" target=_blank>general pack_stat browser</a> </font> <p>
<!--EXTERNAL-->
<font size=2><b> back to:</b>
<a href=../histoMonitor.html>histoMonitor (main)</a> 
  </td>
  <td align=right><img src=${IMG_URL1}/nautilus.gif border=0 align=top><br>
[<font size=2><a href="../nautilus.html" title="more about nautilus">more</a>]
</tr>

<tr><td COLSPAN=20><hr noshade></td></tr>
<tr><td COLSPAN=20>

  <table><tr>
EOT

# period navigation table 
	COUNTER=0
	for P in `cat $TMP_DIR/list_all_periods`
	do
		P1=`basename $P`
		MONTH=`grep "^$P1" $DFO_CONFIG_DIR/dates.histoMonitor | awk '{print $2}'`
		YEAR=`grep  "^$P1" $DFO_CONFIG_DIR/dates.histoMonitor | awk '{print $3}'`
		NEW=`grep   "^$P1" $DFO_CONFIG_DIR/dates.histoMonitor | awk '{print $4}'`
		if [ Q$NEW = QNEW ]
		then
			NEW="<td>&nbsp;</td>"
		fi

		if [ $P1 != $PERIOD ]
		then
			echo "${NEW}<td bgcolor=${PACK_COLOR2}><font size=2><a href=../$P1/$P1.html title=\"$MONTH $YEAR\">$P1</a></td>" >> $HISTO_PERIOD	
		else
			echo "${NEW}<td bgcolor=${PACK_COLOR}><font size=2><a title=\"$MONTH $YEAR\">$P1</a></td>" >> $HISTO_PERIOD
		fi
		COUNTER=`echo $COUNTER | awk '{print $1+1}'`
		if [ $COUNTER = 20 ]
		then
			echo "</tr><tr>" >> $HISTO_PERIOD
			COUNTER=0
		fi
	done

	cat >> $HISTO_PERIOD <<EOT
  </tr></table>

  </td></tr>
EOT

# result table
	cat >> $HISTO_PERIOD <<EOT
  <tr bgcolor=#CCCCCC>
    <td valign=top><font size=2><b>PACKAGE</b><br>(run_id &nbsp; first_data_date)</td>
    <td valign=top><font size=2><b><a title="FULL or PARTIAL">flag</a></td>
    <td valign=top><font size=2><b><a title="archive query sched_rep">PICoI</a></td>
    <td valign=top><font size=2><b><a title="date of QC release to AOG">released</a></td>

    <td valign=top bgcolor=#FFFFFF><font size=2>&nbsp;</td>
    <td valign=top><font size=2><b><a title="README.html file">README</td>
    <td valign=top><font size=2><b><a title="Extracted nightlogs, association info">Observing reports</a></td>
    <td valign=top bgcolor=#FFFFFF><font size=2>&nbsp;</td>
    <td valign=top><font size=2><b><a title="table of contents">TOC</a></td>
    <td valign=top><font size=2><b><a title="statistics and directory list">dir_list</a></td>
    <td valign=top><font size=2><b><a title="releaseDP log">pack_log</a></td>
    <td valign=top bgcolor=#FFFFFF><font size=2>&nbsp;</td>

    <td valign=top align=center><font size=2><b><a title="archive list as shipped with the package">arc_list</a></td>
    <td valign=top><font size=2><b><a title="QC0 report as created by qcm">qcm</a></td>
    <td valign=top bgcolor=#FFFFFF><font size=2>&nbsp;</td>
    <td valign=top><font size=2><b><a title="explore the package structure">directory content</a></td>
</tr>
EOT
	PREVIOUS=0
	INDEX=1
	for PACK in `ls -d [0-9]*`
	do
		FIRST=`echo $PACK | cut -c1,1`
		if [ $FIRST != $PREVIOUS ]
		then
			echo "<tr><td COLSPAN=20><font size=1>&nbsp;</font></td></tr>" >> $HISTO_PERIOD
			PREVIOUS=$FIRST
		fi
		RUN_ID=`echo $PACK | sed "s/_.*//"`
		RUN_ID2=`echo $RUN_ID | sed "s/.$/(&/" | sed "s/.$/&)/"`
		TOC=`ls $PACK    | grep TOC`
		if [ Q$TOC = Q ]
		then
			TOC="  <font size=1><i>no TOC found!</i></font>"
		else
			TOC="  <a href=\"$DFO_SRV_DIR/FINISHED/$PERIOD/$PACK/$TOC\" title=\"Table of contents as sent to archive operations\" target=\"_blank\"><img src=\"${IMG_URL}/text.gif\" width=16 height=17 border=0></a>"
		fi

		README=`ls $PACK | grep README`
		if [ Q$README = Q ]
		then
			README="  <font size=1><a title=\"README introduced in P80\">none</a></i></font>"
		else
			README="  <a href=\"$DFO_SRV_DIR/FINISHED/$PERIOD/$PACK/README.html\" title=\"Data package cover sheet (sent to user)\" target=\"_blank\"><img src=\"${IMG_URL}/blueball.gif\" width=10 height=10 border=0></a>"
		fi

		if [ -d $PACK/GEN_INFO/ObservingReports ]
		then
			NR=`ls $PACK/GEN_INFO/ObservingReports | grep "OBlog.html"`
		else
			NR=""
		fi

		if [ Q$NR = Q ]
                then
                        NR="  <font size=1><i>no NR found!</i></font>"
		else
			NR="  <a href=\"$DFO_SRV_DIR/FINISHED/$PERIOD/$PACK/GEN_INFO/ObservingReports/OBlog.html\" title=\"Nightlogs, OB reports, assoc. report (sent to user)\" target=\"_blank\"><img src=\"${IMG_URL}/blueball.gif\" width=10 height=10 border=0></a>"
		fi
		
		DIR=`ls $PACK | grep list_$PACK`
		if [ Q$DIR = Q ]
                then
                        DIR="  <font size=1><i>no dir_list found!</i></font>"
		else
			DIR="  <a href=\"$DFO_SRV_DIR/FINISHED/$PERIOD/$PACK/list_$PACK\" title=\"directory list and statistics\" target=\"_blank\"><img src=\"${IMG_URL}/redball.gif\" width=10 height=10 border=0></a>"
		fi

		LOG=`ls $PACK | grep pack_$RUN_ID`
		if [ Q$LOG = Q ]
                then
                        LOG="  <font size=1><i>no pack_log found!</i></font>"
		else
			LOG="  <a href=\"$DFO_SRV_DIR/FINISHED/$PERIOD/$PACK/pack_${RUN_ID}\" title=\"packing log (created by releaseDP)\" target=\"_blank\"><img src=\"${IMG_URL}/redball.gif\" width=10 height=10 border=0></a>"
		fi

		if [ ! -d $PACK/GEN_INFO ]
		then
			ARC="  <font size=1><i>no arc_list found!</i></font>"
                        QCM="  <font size=1><i>no qcm_list found!</i></font>"
		else
			ARC=`ls $PACK/GEN_INFO | grep archive_${RUN_ID}.txt`
			if [ Q$ARC = Q ]
                	then
                       		ARC="  <font size=1><i>none!</i></font>"
			else
				ARC="  <a href=\"$DFO_SRV_DIR/FINISHED/$PERIOD/$PACK/GEN_INFO/archive_${RUN_ID}.txt\" title=\"archive list (created by releaseDP)\" target=\"_blank\"><img src=\"${IMG_URL}/greenball.gif\" width=10 height=10 border=0></a>"
			fi

			QCM=`ls $PACK/GEN_INFO | grep qc0_${RUN_ID}.txt`
			if [ Q$QCM = Q ]
                	then
                       		QCM="  <font size=1><i>none!</i></font>"
			else
				QCM="  <a href=\"$DFO_SRV_DIR/FINISHED/$PERIOD/$PACK/GEN_INFO/qc0_${RUN_ID}.txt\" title=\"OB constraint list (created by qcm)\" target=\"_blank\"><img src=\"${IMG_URL}/greenball.gif\" width=10 height=10 border=0></a>"
			fi
		fi

		if [ -s $DFO_MON_DIR/REL_STATUS_$PERIOD ]
		then
			FULL=`grep "${RUN_ID}" $DFO_MON_DIR/REL_STATUS_$PERIOD | awk '{print $7}'`
			if [ "Q$FULL" = Q ]
                        then
                                FULL=`grep "${RUN_ID2}" $DFO_MON_DIR/REL_STATUS_$PERIOD | awk '{print $7}'`
                        fi

			RELEASED=`grep "${RUN_ID}" $DFO_MON_DIR/REL_STATUS_$PERIOD | awk '{print $3}' | tr "\012" "|" | sed "s/|$//" | sed "s/|/| /g"`

			if [ "Q$RELEASED" = Q ]
			then
				RELEASED=`grep "${RUN_ID2}" $DFO_MON_DIR/REL_STATUS_$PERIOD | awk '{print $3}' | tr "\012" "|" | sed "s/|$//" | sed "s/|/| /g"`
			fi
	
# added 'sort -u' to support partial VM packages [Smo]
			VM=`grep "${RUN_ID}" $DFO_MON_DIR/REL_STATUS_$PERIOD | awk '{print $1}' | grep "_VM" | sed "s/^.*_VM/VM/" | sort -u`
		else
			FULL="&nbsp;"
			RELEASED="&nbsp;"
			VM=""
		fi

		case $INDEX in
		 1 ) ROWCOLOR=${PACK_COLOR3};;
		 2 ) ROWCOLOR=${PACK_COLOR2};;
		esac
		if [ Q$VM != Q ]
		then
			ROWCOLOR=${PACK_COLOR4}
		fi

		cat >> $HISTO_PERIOD <<EOT
<tr bgcolor=${ROWCOLOR}>
  <td bgcolor=${PACK_COLOR} nowrap><font size=2><b>$PACK $VM</b></font></td>
  <td align=center><font size=2>$FULL</font></td>
  <td align=center><font size=2><a href="${SCHED_REP_URL}${RUN_ID2}" title="sched_rep info about $RUN_ID" target="_blank">info</a></font></td>
  <td><font size=2>${RELEASED}</td>
  <td valign=top bgcolor=#FFFFFF><font size=2>&nbsp;</td>

  <td align=center>${README}</td>
  <td align=center>${NR}</td>
  <td bgcolor=#FFFFFF><font size=1>&nbsp;</td>

  <td>${TOC}</td>
  <td align=center>${DIR}</td>
  <td align=center>${LOG}</td>
  <td bgcolor=#FFFFFF><font size=1>&nbsp;</td>

  <td align=center>${ARC}</td>
  <td align=center>${QCM}</td>
  <td bgcolor=#FFFFFF><font size=2>&nbsp;</td>
  <td align=center><font size=2><a href="$DFO_SRV_DIR/FINISHED/$PERIOD/$PACK/" title="explore content" target="_blank">go to ...</a></font></td>
</tr>
EOT
		case $INDEX in
		 1 ) INDEX=2 ;;
		 2 ) INDEX=1 ;;
		esac	
	
	done

	echo "</table></html>" >> $HISTO_PERIOD

	echo "Exporting $PERIOD.html to $DFO_WEB_SERVER ..."
	if [ Q$DFO_WEB_SERVER != Q ]
	then
       		ssh -o BatchMode=yes $DFO_WEB_SERVER "$QCBIN_DIR/webDirCreate HISTO_PACK ${DFO_INSTRUMENT} $PERIOD"  1>/dev/null

# edit version for ${DFO_WEB_SERVER}
		cat $HISTO_PERIOD |\
		 sed -e "s|${DFO_MON_DIR}|${MON_DIR}/FINISHED/${PERIOD}|g" |\
		 sed -e "s|${DFO_SRV_DIR}/FINISHED|${PACK_DIR}|g" \
		 > $TMP_DIR/period.html
	
# transfer to $DFO_WEB_SERVER 	
       		scp -o BatchMode=yes $TMP_DIR/period.html ${DFO_WEB_SERVER}:${INSTR_DIR}/monitor/FINISHED/$PERIOD/$PERIOD.html 1>/dev/null
		if [ -s $DFO_MON_DIR/REL_STATUS_$PERIOD ]
		then
       			scp -o BatchMode=yes $DFO_MON_DIR/REL_STATUS_$PERIOD ${DFO_WEB_SERVER}:${INSTR_DIR}/monitor/FINISHED/$PERIOD/ 1>/dev/null
		fi

		if [ -s $DFO_MON_DIR/STATISTICS_PACKAGES_$PERIOD ]
		then
       			scp -o BatchMode=yes $DFO_MON_DIR/STATISTICS_PACKAGES_$PERIOD ${DFO_WEB_SERVER}:${INSTR_DIR}/monitor/FINISHED/$PERIOD/ 1>/dev/null
		fi
	fi
	exit
fi

# ==============================================================================
# 2.2 Refresh all PERIOD pages (for navigation) 
# ==============================================================================

if [ $ALL_PERIOD_SELECT = YES ]
then
	echo "We refresh all package pages, to get the navigation right. OK (Y/N) (Y)?"
	read OK_YN
	if [ Q$OK_YN = QN ]
	then
		exit
	fi

	for P in `cat $TMP_DIR/list_all_periods`
	do
		P1=`basename $P`
		histoMonitor -p $P1 | grep -v "main page"
	done
	echo "All period pages refreshed."
	exit
fi

# ==============================================================================
# 3. YEAR-MONTH.html pages
# ==============================================================================

if [ $YSELECT_MODE = YES ]
then
	case $REFRESH_MONITOR in
	 "NO"  ) echo "Option -y: We refresh the histoMonitor for year $YEAR_SELECT.
Hit return:" ;;
	 "YES" ) echo "Option -y: We refresh the histoMonitor for year $YEAR_SELECT, 
the AB monitor pages are recreated.
Hit return:" ;;
	esac
	read input
fi

if [ $MSELECT_MODE = YES ]
then
	case $REFRESH_MONITOR in
	 "NO"  ) echo "Option -m: We refresh the histoMonitor for month $MONTH_SELECT." ;;
	 "YES" ) echo "Option -m: We refresh the histoMonitor for month $MONTH_SELECT, 
the AB monitor pages are recreated." ;;
	esac
fi

if [ $DSELECT_MODE = YES ]
then
	echo "Option -d: We refresh the histoMonitor for date $DATE_SELECT only.
"
	MSELECT_MODE=YES
	MONTH_SELECT=`echo $DATE_SELECT | cut -c1-7`	
fi

cd $DFO_LOG_DIR

if [ Q$DFO_WEB_SERVER != Q ]
then
        ssh -o BatchMode=yes $DFO_WEB_SERVER "$QCBIN_DIR/webDirCreate MONITOR ${DFO_INSTRUMENT}"  1>/dev/null
fi

# ==============================================================================
# 3.1 YEAR loop
# ==============================================================================

for Y in `cat $TMP_DIR/all_years | sort -r`
do
	cd $DFO_LOG_DIR
	if [ $YSELECT_MODE = YES ]
	then
		if [ $Y != "$YEAR_SELECT" ]
		then
			continue
		fi
	fi

	echo "$Y ..."
	if [ ! -d $DFO_MON_DIR/FINISHED/$Y ]
	then
		mkdir $DFO_MON_DIR/FINISHED/$Y
		echo "histoMonitor: new year created: $Y; you need to run 'histoMonitor' (w/o parameters) to update the navigation bars.
  This should be done off-line since it may be time-consuming." > $TMP_DIR/new_year
		cat $TMP_DIR/new_year
		mail -s "histoMonitor: new year created $Y" $OP_ADDRESS <$TMP_DIR/new_year
	fi
	
	rm -f $TMP_DIR/list_yearmonths
	ls -d 20* | grep $Y > $TMP_DIR/list_yearmonths

# ==============================================================================
# 3.2 MONTH loop
# ==============================================================================

	for YM in `cat $TMP_DIR/list_yearmonths | cut -c1-7 | sort -ur `
	do
		if [ ! -d $DFO_MON_DIR/FINISHED/$Y/$YM ]
		then
# check for PHOENIX_ENABLED: if YES send signal if new month is started
			if [ $PHOENIX_ENABLED = YES ]
			then
				PREVIOUS=`qcdate ${YM}-01 -1`
				PREVIOUS=`echo $PREVIOUS | cut -c1-7`
				echo "histoMonitor: new month created: $YM; you may want to run 'phoenix' on $PHOENIX_ACCOUNT for the previous month:
phoenix -m $PREVIOUS" > $TMP_DIR/new_month
				cat $TMP_DIR/new_month
				mail -s "phoenix: ready to be started for $PREVIOUS" $OP_ADDRESS <$TMP_DIR/new_month
				UPDATE=`date +%Y-%m-%d"T"%H:%M:%S`
				echo "phoenix_Ready $PREVIOUS $UPDATE" >> $DFO_MON_DIR/DFO_STATUS
			fi

			mkdir $DFO_MON_DIR/FINISHED/$Y/$YM
			echo "  New month created; we run 'histoMonitor -y $Y' to update the monthly navigation bars. Hit return:" 
			read input
			histoMonitor -y $Y <<EOT

EOT
		fi

		if [ $MSELECT_MODE = YES ] 
		then
			if [ $YM != "$MONTH_SELECT" ]
			then
				continue
			fi
		fi

		echo "  $YM ..."
		if [ ! -d $DFO_MON_DIR/FINISHED/$Y/$YM ]
		then
			mkdir $DFO_MON_DIR/FINISHED/$Y/$YM
		fi

# check  out for comments in daily_comments
		cat > $TMP_DIR/hm_ym_query <<EOT
SELECT
        id , convert ( varchar(10),civil_date,102 )
FROM
        qc1..daily_comments
WHERE
        instrument = "$DFO_INSTRUMENT"
AND
        month = "$YM"
AND
        visible = "Y"
GO
EOT

		rm -f $TMP_DIR/hm_ym_results
		isql -Uqc -P`cat $QC1_PWD` -S${QC1_SERVER} -i $TMP_DIR/hm_ym_query |\
  sed "1,2d" |\
  grep -v affected |\
  sed "/^$/d" |\
  sed "/^   $/d" |\
  sed "s/\./-/g" | awk '{print $1,$2}' \
  > $TMP_DIR/hm_ym_results

		HISTO_YM=$DFO_MON_DIR/FINISHED/$Y/$YM/${YM}.html
		cat > $HISTO_YM <<EOT
<html>
<script type="text/javascript">
<!--
function openWin(URL) {
  aWindow=window.open(URL,"a","toolbar=yes,width=1300,height=1400,scrollbars=yes,menubar=yes");
  }//-->
</script>

<table cellspacing="1" border=0 width=50%>
  <tr>
    <td COLSPAN=10>
<h2>${DFO_INSTRUMENT} historical record: ${YM}</h2>
<!--EXTERNAL-->
<font size=2><b> back to:</b> 
<a href=../../histoMonitor.html>histoMonitor (main)</a> |
<a href=../../../dfoMonitor.html>dfoMonitor</a>
&nbsp;&nbsp;&nbsp;&nbsp; <a href=$DFO_GUI_DIR/comment_histoMonitor.esh><font color=${REPORT_COLOR}>[enter date comment]</font></a> <!--FILTER-->
<p>

- "no data found": no data found for that date<br>
- "cdb?": products are ingested in NGAS<br>
- "sci?": products are ingested in NGAS (before 2011-10-01); later: associations stored on qcweb<br>
  </td>
  <td align=right><img src=${IMG_URL1}/nautilus.gif border=0 align=top><br>
[<font size=2><a href="../../nautilus.html" title="more about nautilus">more</a>]
</tr>

<tr><td COLSPAN=11><hr noshade></td></tr>
EOT

# ==============================================================================
# 3.3 Year navigation
# ==============================================================================

		cat >> $HISTO_YM <<EOT
  <tr>
    <td COLSPAN=11>

      <table>
        <tr>
EOT
		for YY in `cat $TMP_DIR/all_years`
		do
			if [ $YY != $Y ]
			then
				FIRST_DIR=`ls -d $DFO_MON_DIR/FINISHED/$YY/$YY* | head -1`
				FIRST_DIR=`basename $FIRST_DIR`
				echo "          <td><font size=2><a href=../../$YY/${FIRST_DIR}/${FIRST_DIR}.html>$YY</a></td>" >> $HISTO_YM
			else
				echo "          <td bgcolor=#91C8FF><font size=2>$Y</td>" >> $HISTO_YM
			fi
		done
	
		cat >> $HISTO_YM <<EOT
        </tr>
      </table>
	
    </td>
  </tr>
EOT

# ==============================================================================
# 3.4 month navigation
# ==============================================================================

		cat >> $HISTO_YM <<EOT
  <tr>
    <td COLSPAN=11>

      <table>
        <tr bgcolor=#CCCCCC>
EOT
		for YYMM in `cat $TMP_DIR/list_yearmonths | cut -c1-7 | sort -u`
		do
			if [ $YM != $YYMM ]
			then
				echo "          <td><font size=2><a href=../../$Y/$YYMM/$YYMM.html>$YYMM</a></td>" >> $HISTO_YM
			else
				echo "          <td bgcolor=#91C8FF><font size=2>$YYMM</td>" >> $HISTO_YM
			fi
		done
		cat >> $HISTO_YM <<EOT
        </tr>
      </table>

    </td>
  </tr>
EOT

# ==============================================================================
# 3.5 Columns
# 3.5.1 Header 
# ==============================================================================

		cat >> $HISTO_YM <<EOT
  <tr bgcolor=#CCCCCC>
    <td><font size=2><b>DATE</td>
    <td><font size=2><b><a title="NR: extracted nightlog, or NLT: nightlog tool">NR</a></td>
    <td><font size=2><b>cdb?</td>
    <td><font size=2><b>sci?</td>
    <td><font size=2><b>SM_VM</td>
    <td><font size=2><b>report</td>
    <td COLSPAN=2><font size=2><b>statistics*</td>
    <td><font size=2><b>ABs**</td>
    <td COLSPAN=2><font size=2><b>comments</td>
</tr>
<tr>
EOT
		rm -f $TMP_DIR/hm_list_procHist
		ls -d $DFO_LOG_DIR/${YM}* > $TMP_DIR/hm_list_procHist

		MONTH=`echo $YM | cut -c6-7`
		case $MONTH in
		 02 ) LAST_D=28 ;;
		 04|06|09|11 ) LAST_D=30 ;;
		 * ) LAST_D=31 ;;
		esac
		if [ $YM = 2004-02 ] || [ $YM = 2008-02 ] || [ $YM = 2012-02 ] || [ $YM = 2016-02 ] || [ $YM = 2020-02 ] || [ $YM = 2024-02 ] || [ $YM = 2028-02 ] 
		then
			LAST_D=29
		fi

# ==============================================================================
# 3.5.2 DATE loop
# ==============================================================================

		TODAY_Y=`date +%Y`

		for D in `cat $TMP_DIR/all_days`
		do
			if [ $D -gt $LAST_D ]
			then
				break
			fi
			DATE="${YM}-$D"
			DATE_SHORT=`echo $DATE | sed "s/-//g"`
			DATE_LINK="<td><font size=2><b>$DATE</td>"
			if [ $DATE = $LAST_DFO_DATE ] 
			then
				DATE_LINK="<td nowrap><font size=2><b>$DATE</b><a href=${DFO_MON_DIR}/dfoMonitor.html title=\"current last date in dfoMonitor\"><img src=${IMG_URL}/down.gif width=16 height=17 border=0></a></td>"
			fi

# we assume that all DATEs from 2008 on have their NR created, otherwise the code would become more complex here
			if [ ! -s ${DFO_LST_DIR}/NR_$DATE ] && [ $Y -le 2007 ]
			then
				echo "    $DATE: no nightlog found. "
			fi

# check for NLT old/new scheme
			NLT_APPLICABLE=`grep "^#NLT" $DFO_BIN_DIR/getObInfo | awk '{print $2,$3}' | grep "^$PANL_INSTRUMENT " | awk '{print $2}'`
			if [ Q$NLT_APPLICABLE = Q ]
			then
				NLT_APPLICABLE=20050101
			fi
			NLT_FLAG=YES

			if [ $DATE_SHORT -lt $NLT_APPLICABLE ]
			then
				NLT_FLAG=NO
			fi

			if [ $NLT_FLAG = NO ]
			then
				NR_LINK="<td><a href=${DFO_LST_DIR}/NR_$DATE title=\"night report\" target=\"_blank\"><font size=2>NR</font></a></td> "
			else
               			NR_LINK="<td><a href=\"javascript:openWin('${NLT_URL}?date=${DATE}&instrument=$PANL_INSTRUMENT');\" title=\"instrument night report from NLT\"><font size=2>NLT</font></a>&nbsp;"
			fi

# data report, SM_VM flag
			rm -f $TMP_DIR/hm_list_flag
		
# selected date: delete old report, enforce refreshing new one
			if [ $DSELECT_MODE = YES ]
			then
				if [ $DATE = "$DATE_SELECT" ]
				then
					echo "    $DATE: creation of new report enforced." 
					rm -f ${DFO_LST_DIR}/list_${DFO_INSTRUMENT}_$DATE.txt
				fi
			fi

# none found --> most likely it existed once but was deleted later; download headers, reconstruct report
			TODAY=`date +%Y-%m-%d`
			DATE1=`echo $DATE | sed "s/-//g"`
			TODAY1=`echo $TODAY | sed "s/-//g"`

			if [ ! -s ${DFO_LST_DIR}/list_${DFO_INSTRUMENT}_${DATE}.txt ] && [ $TODAY1 -ge $DATE1 ]
			then
				if [ $CREATE_REPORT = YES ]
				then
					getHeaders
					echo ""
					echo "-Create report for $DATE ..."
					createReport -d $DATE | sed "s/^.*/	&/"
				else
					echo "$DATE: No report found."
				fi
			fi

			SCI_COLOR=#FFFFFF
			REPORT_LINK="<td><font size=2><a href=${DFO_LST_DIR}/REPORT/list_${DFO_INSTRUMENT}_${DATE}_data.html target=_blank><font color=#FF6600>report</font></a></td>"
			if [ -s ${DFO_LST_DIR}/list_${DFO_INSTRUMENT}_${DATE}.txt ]
			then
				cat ${DFO_LST_DIR}/list_${DFO_INSTRUMENT}_$DATE.txt |\
 	                	 awk '{
                 	        	if ( change == 1 && $1 != "Number" && $2 != "" )  { print date, $2 }
                       			if ( $1 == "OBS_PROG_ID" && $2 == "SM/VM" ) { change=1 }
                        		if ( $1 == "Number" ) { change=0 }
                      	     	       }' change=0 date=$DATE | sort -u > $TMP_DIR/hm_list_flag
				if [ -s $TMP_DIR/hm_list_flag ]
				then
					SM_VM=`cat $TMP_DIR/hm_list_flag | awk '{print $2}' | sort -u | tr "\012" " " | sed "s/s/<font color=${SM_COLOR}><b>SM<\/b><\/font>/" | sed "s/v/<font color=${VM_COLOR}>VM<\/font>/"`
					SCI_COLOR=#33FF33
        			else
                			SM_VM="&nbsp;"
					SCI_COLOR=#CCFFCC
        			fi
				SM_VM="<td align=center><font size=2>${SM_VM}</td>"

# statistics: ALL files	(from report)
				rm -f $TMP_DIR/list_STAT $TMP_DIR/list_STAT_SCI
				cat ${DFO_LST_DIR}/list_${DFO_INSTRUMENT}_$DATE.txt |\
				 awk '{
					if ( change == 0 && $1 == "Number" )  { change=1 }
					if ( change == 1 && $1 == "ALL:" )    { print $2; change=0 }
                                      }' change=0 > $TMP_DIR/list_STAT
				if [ -s $TMP_DIR/list_STAT ]
				then
					STAT=`cat $TMP_DIR/list_STAT`
				fi

# ALL science files (from report); for b/w compatibility we support two cases
# new
				STAT_SCI=`grep "SCIENCE:" ${DFO_LST_DIR}/list_${DFO_INSTRUMENT}_$DATE.txt | awk '{print $2}'`
# old
				if [ Q$STAT_SCI = Q ]
                                then
					cat ${DFO_LST_DIR}/list_${DFO_INSTRUMENT}_$DATE.txt |\
				 	 awk '{
					  if ( change == 0 && $1 == "SCIENCE:" )  { change=1 }
					  if ( change == 1 && $1 == "ALL:" )    { print $2; change=0 }
                                              }' change=0 > $TMP_DIR/list_STAT_SCI
				else
					echo $STAT_SCI > $TMP_DIR/list_STAT_SCI
				fi

				if [ -s $TMP_DIR/list_STAT_SCI ]
                                then
                                        STAT_SCI=`cat $TMP_DIR/list_STAT_SCI`
				else
					STAT_SCI=0
                                fi
			else
				SCI_COLOR=#CCCCCC
				SM_VM="<td>&nbsp;</td>"
				STAT=0
				STAT_SCI=0
			fi

# CAL_LINK, SCI_LINK

# criteria: entry in DFO_STATUS (works for the last \approx 3 months) or list_ingest (fine unless deleted) or hdr files in $DFO_CAL_DIR (for old dates; safest)
# CHECK_INGEST=NO: no products exist, nothing to ingest; YES: products ingested; pending: products exist but not yet ingested
# CAL_LINK
			CHECK_INGEST=NO
			CHECK_ENTRY=`grep "^cal_Ingested" $DFO_MON_DIR/DFO_STATUS | grep " $DATE " | head -1`
			if [ "Q$CHECK_ENTRY" != "Q" ]
			then
				CHECK_INGEST=YES
			else
				if [ -s $DFO_LST_DIR/list_ingest_$DATE.txt ] || [ -s $DFO_LST_DIR/list_ingest_CALIB_$DATE.txt ]
				then
					CHECK_INGEST=YES
				elif [ -d $DFO_CAL_DIR/$DATE ]
				then
					CHECK_HDR=`ls $DFO_CAL_DIR/$DATE | grep hdr | head -1`
					CHECK_FITS=`ls $DFO_CAL_DIR/$DATE | grep fits | head -1`
					if [ "Q$CHECK_HDR" != Q ]
					then
						CHECK_INGEST=YES
					elif [ "Q$CHECK_FITS" != Q ]
					then
						CHECK_INGEST=pending
					fi	
				fi
			fi

			case $CHECK_INGEST in
			 "NO" )  	   CAL_LINK="<td bgcolor=#CCCCCC align=center><font size=2>&nbsp;</td>" ;;
			 "YES" ) 	   CAL_LINK="<td bgcolor=#FFFF00 align=center><img src=$IMG_URL/ok.gif width=12 height=15 title=\"calib products ingested\"></td>" ;;
			 "pending" ) CAL_LINK="<td bgcolor=#FFFFCC align=center><img src=$IMG_URL/open.gif width=12 height=15 title=\"no calib products ingested but calib directory existing!\"></td>" ;;
			esac

# SCI_LINK
			CHECK_INGEST=NO
			CHECK_ENTRY=`grep "^sci_Ingested" $DFO_MON_DIR/DFO_STATUS | grep " $DATE " | head -1`
			if [ "Q$CHECK_ENTRY" != "Q" ]
			then
				CHECK_INGEST=YES
			else
				if [ -s $DFO_LST_DIR/list_ingest_$DATE.txt ] || [ -s $DFO_LST_DIR/list_ingest_SCIENCE_$DATE.txt ]
				then
					CHECK_INGEST=YES
# special for SCI after 2011-10-01: check CALSELECTOR directory; if existing, that's good enough for CHECK_INGEST=YES
				elif [ -d $DFO_LOG_DIR/CALSELECTOR/$DATE ]
				then
					CHECK_INGEST=YES
				elif [ -d $DFO_SCI_DIR/$DATE ]
				then
					CHECK_HDR=`ls  $DFO_SCI_DIR/$DATE | grep hdr | head -1`
					CHECK_FITS=`ls $DFO_SCI_DIR/$DATE | grep fits | head -1`
					if [ "Q$CHECK_HDR" != Q ]
					then
						CHECK_INGEST=YES
					elif [ "Q$CHECK_FITS" != Q ]
					then
						CHECK_INGEST=pending
					fi	
				fi
			fi

			case $CHECK_INGEST in
			 NO )  	   SCI_LINK="<td bgcolor=#CCCCCC align=center><font size=2>&nbsp;</td>" ;;
			 YES ) 	   SCI_LINK="<td bgcolor=#66CCFF align=center><img src=$IMG_URL/ok.gif width=12 height=15 title=\"science ABs created and verified\"></td>" ;;

			 pending ) SCI_LINK="<td bgcolor=#AAEEFF align=center><img src=$IMG_URL/open.gif width=12 height=15 title=\"no science ABs found but science directory existing!\"></td>" ;;
			esac

# statistics entry
			BOLD=""
			WIDTH=""
			IMG_LINK="&nbsp;"

			if [ Q$STAT_SCI = Q ]
			then
				STAT_SCI=0
			fi

			if [ $STAT_SCI -gt 0 ] 
			then
				BOLD="<b>"
				WIDTH=`echo $STAT_SCI $SCI_SCALE $SCI_MAX | awk '{if ($1 < $3) {print $1*$2} else {print $3*$2} }'`
				IMG_LINK="<img src=${IMG_URL}/sq1.png width=$WIDTH height=5>"
				if [ $STAT_SCI -gt $SCI_MAX ]
				then
					IMG_LINK="<img src=${IMG_URL}/sq1.png width=$WIDTH height=5> <img src=${IMG_URL}/sq1.png height=10 width=5>"
				fi
			fi

			MAX_WIDTH=`echo $SCI_SCALE $SCI_MAX | awk '{print $1*$2}'`
			if [ ${STAT} -gt 0 ]
			then
				ENTRY_STAT="<td><font size=2>${BOLD}${STAT}/${STAT_SCI}</td><td nowrap>${IMG_LINK}</td>"
			else
				ENTRY_STAT="<td colspan=3 nowrap align=center><font size=1>no data found</font></td>"
			fi

# ========================================================================================
# 3.5.3 create AB monitor, update links
#       FINISHED: if older than 90 days, we always assume $DATE to be finished (no DFO_STATUS info)
#       younger than 90 days: we check DFO_STATUS 
# ========================================================================================

			FINISHED=`grep " $DATE " $DFO_MON_DIR/DFO_STATUS | grep finished`
			MJD_TODAY=`qcdate $TODAY`
			MJD_DATE=`qcdate $DATE`
			DIFF=`echo $MJD_TODAY $MJD_DATE | awk '{print $1-$2}'`

			if [ $DIFF -gt 90 ]
			then
				FINISHED=yes
			fi

# selected date: enforce refreshing AB monitor pages
			if ( [ $DSELECT_MODE = YES ] && [ $DATE = "$DATE_SELECT" ] ) || [ $REFRESH_MONITOR = YES ] 
			then
				rm -f $DFO_MON_DIR/FINISHED/$Y/$YM/status_${DATE}*.html
			fi

			if [ ! -s $DFO_MON_DIR/FINISHED/$Y/$YM/status_${DATE}.html ] && [ "Q$FINISHED" != "Q" ]
			then
				echo ""
				echo "-Create final AB status page for $DATE ..."
				getStatusAB -d $DATE -H | sed "s/^.*/	&/"
				echo ""
			fi

# ==============================================================================
# 3.6 Finish monthly page
# ==============================================================================

			if [ ${STAT} = 0 ]
			then
				AB_LINK=""	
			elif [ -s ${DFO_MON_DIR}/FINISHED/$Y/$YM/status_${DATE}.html ]
			then
				N_AB_SUCCESS=`grep ${DATE} ${DFO_MON_DIR}/STATISTICS_DAILY | awk '{sum +=$15} END {print sum}'`
# earlier version of ABs did not have PROCESS_STATUS; entries in STATISTICS_DAILY might not exist 
				if [ "Q$N_AB_SUCCESS" = "Q" ]
				then
					N_AB_SUCCESS="..."
				fi
# entries in STATISTICS_DAILY might not exist --> we count ABs
				N_AB=`grep ${DATE}         ${DFO_MON_DIR}/STATISTICS_DAILY | awk '{sum +=$14} END {print sum}'`
				if [ "Q$N_AB" = "Q" ]
				then
					N_AB=`ls $DFO_LOG_DIR/$DATE | grep .ab | wc -l`
				fi

				AB_LINK="<td nowrap><font size=2><a href=status_${DATE}.html target=_blank><img src=$IMG_URL/index.gif width=15 height=15 border=0></a> done (${N_AB_SUCCESS}/$N_AB)</td>"
			elif [ $DIFF -lt 90 ]
			then
				AB_LINK="<td nowrap><font size=2>not yet finished</td>"
			else
				AB_LINK="<td nowrap><font size=2>&nbsp;</td>"
			fi

# check out $TMP_DIR/hm_ym_results for entry

			CHECK_EXIST=""
			if [ -s $TMP_DIR/hm_ym_results ]
			then
				CHECK_EXIST=`grep $DATE $TMP_DIR/hm_ym_results`
			fi

			if [ "Q$CHECK_EXIST" != "Q" ]
			then
				COMM1=`getEntry`
				COMM="<td width=\"400p\" COLSPAN=2><font size=1>$COMM1</td>"
			else
				COMM="<td width=\"400p\" COLSPAN=2><font size=1>&nbsp;</td>"
			fi

# check $DATE to not lie in the future			
			if [ $DATE1 -le $TODAY1 ]
			then
				echo "<tr bgcolor=${SCI_COLOR}>${DATE_LINK}${NR_LINK}${CAL_LINK}${SCI_LINK}${SM_VM}${REPORT_LINK}${ENTRY_STAT}${AB_LINK}${COMM}</tr>" >> $HISTO_YM
			else
				echo "<tr bgcolor=${SCI_COLOR}>${DATE_LINK}<td COLSPAN=10 align=left><font size=1 color=#999999><i>date in the future</i></font></td></tr>" >> $HISTO_YM
			fi
		done

		UPDATE=`date +%Y-%m-%d" "%H:%M:%S`
		cat >> $HISTO_YM <<EOT
<tr bgcolor=#CCCCCC><td ALIGN=RIGHT COLSPAN=99><font size=2 COLOR=#FFFFFF> last update: $UPDATE (LT) &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; created on $HOSTNAME by histoMonitor v${TOOL_VERSION}, a <b>dfos</b> tool</font></td></tr>
</table>
<p>
<font size=2>* <b>statistics</b>: taken from reports; ALL/SCIENCE; bar: proportional to SCIENCE number (truncated at $SCI_MAX)<br>
** <b>ABs</b>: successful/all<br>
</html>
EOT

		HISTO_PAGE=`basename $HISTO_YM`
		echo "Exporting $HISTO_PAGE to $DFO_WEB_SERVER ..."
		if [ Q$DFO_WEB_SERVER != Q ]
		then
        		ssh -o BatchMode=yes $DFO_WEB_SERVER "$QCBIN_DIR/webDirCreate HISTO ${DFO_INSTRUMENT} $YM"  1>/dev/null
# cleaned version
			grep -v "<\!\-\-FILTER\-\->" $HISTO_YM |\
			 sed -e "s|${DFO_LST_DIR}|${LST_DIR}|g" |\
			 sed -e "s|${DFO_MON_DIR}|${MON_DIR}|g" |\
			 sed -e "s|/NR_|/NIGHTLOG&|g" |\
			 sed "s/<!--EXTERNAL-->/<b><font size=2>This is the external version, with restricted functionality.<\/b><br>/" > $TMP_DIR/histo1.html
        		scp -o BatchMode=yes $TMP_DIR/histo1.html ${DFO_WEB_SERVER}:${INSTR_DIR}/monitor/FINISHED/$Y/$YM/$HISTO_PAGE 1>/dev/null
		fi
		echo ""
	done
done

# ==============================================================================
# 4. End
# ==============================================================================

echo "... done. Check results under 
$DFO_MON_DIR/FINISHED/histoMonitor.html or 
${WEB_SERVER_URL}/${DFO_INSTRUMENT}/monitor/FINISHED/histoMonitor.html.
"

exit

