#!/bin/sh
# PURPOSE: 	query the qc_metadata..qc_products table for archived products
# AUTHOR:	Reinhard Hanuschik / ESO
# VERSIONS:	1.0 (2007-11-12)
#		1.1 -- improved setup handling; color-coded type (CALIB/SCIENCE/ANCILLARY); mcalib deletion enabled (2008-04-28)
# 		1.2 -- support for ANCILLARY files; file size indicated (2008-08-16)
#		1.2.1- minor change in line :554 (2010-08-02)
#		1.2.2- cdbDelete replaced by dpDelete; calib..data_products with qc_metadata..qc_products (2014-09-04)
#		1.3 -- dataTable used, one output file only (2014-09-04)
#		1.3.1- minor change in result line
#		1.4 -- flag -V: display vproducts (2014-11-14)
#		1.5 -- added: PIPEFILE; checks for double entries in origfile added (2015-04-20)
#		1.5.1- bug fix for ^V. (2015-06-17)
#		1.5.2- accepts r. files (special case of MUSE) (2016-04-06)
#		2.0 -- supporting static calibrations (2017-09-06)
#		2.0.1- bug fix for the search option (2017-09-15)
# PARAMETERS:	-d <date> [-D <date2> -i <instr> -p <pro_catg> -s <setup>]
# OUTPUT:	$TMP_DIR/mcalibDownload for downloads; $TMP_DIR/mcalibDelete for deletion; 
#		$DFO_MON_DIR/productExplorer.html
# CONFIG:	none
# =========================================================================
TOOL_NAME="productExplorer"
TOOL_VERSION="2.0.1"

# =========================================================================
# 0 Initialization
# =========================================================================

PROCATG=ANY
SETUP=ANY

STATIC_MODE=NO		# default
SUPPRESS_VIRTUAL=NO	# hard-coded, but code can work with YES


# colors from AB monitor
SCI_COLOR=#0099FF	#dark_blue
CAL_COLOR=#33FF33	#green
LOG_COLOR=#FFFFCC	#yellow
TITLE_COLOR="#006A9D"   #eso-blue
DEL_COLOR="#CC0000"	#dark-red

# =========================================================================
# 0.1 read options
# =========================================================================

while getopts d:D:i:p:s:Shv OPTION
do
        case "$OPTION" in
         v ) echo $TOOL_VERSION
             exit 0 ;;
         h ) cat $DFO_DOC_DIR/productExplorer.h ; exit 0 ;;
         d ) DATE=$OPTARG ;;
         D ) DATE2=$OPTARG ;;
         i ) INSTR=$OPTARG ;;
	 p ) PROCATG=$OPTARG ;;
	 S ) STATIC_MODE=YES ;;
	 s ) SETUP=$OPTARG ;;
         esac
done

# =========================================================================
# 0.2 initialize
# =========================================================================

if [ $# = 0 ]
then
        cat $DFO_DOC_DIR/productExplorer.h 
	exit 0 
fi

if [ "Q$DATE" = "Q" ] && [ $STATIC_MODE = NO ]
then
	echo "You must specify a DATE."
	exit -1
fi

if [ $STATIC_MODE = YES ] && [ $PROCATG = ANY ]
then
	echo "You must select a pro.catg: call $TOOL_NAME -S -p <pro.catg>. Exit. "
	exit -1
fi

if [ "Q$INSTR" = "Q" ]
then
	INSTR=$DFO_INSTRUMENT
fi

if [ $PROCATG != ANY ] 
then
	PROCATG_STRING="and pro_catg = \"$PROCATG\""
	PROCATG_STRING2="(pro.catg = $PROCATG only)"
else
	PROCATG_STRING=""
	PROCATG_STRING2=""
fi

if [ $INSTR = $DFO_INSTRUMENT ] && [ $PROCATG = ANY ] && [ $SETUP = ANY ] && [ "Q$DATE2" = "Q" ]
then
	SCAN_MODE=STD
elif [ $STATIC_MODE = NO ]
then
	SCAN_MODE=NON-STD
else
	SCAN_MODE=STATIC
	SUPPRESS_VIRTUAL=YES
fi

case $SUPPRESS_VIRTUAL in
 "YES" ) SUPPRESS_STRING="" ; VIRTUAL="and virtual != 1" ;;
 "NO"  ) SUPPRESS_STRING="(vproducts displayed)" ;;
esac

if [ $SCAN_MODE = STD ]
then
	echo "- searching products for $INSTR and $DATE ${SUPPRESS_STRING}..."
elif [ "Q$DATE2" = "Q" ] && [ $SCAN_MODE = "NON-STD" ]
then
	echo "- searching products for $INSTR and $DATE ${PROCATG_STRING2} ${SUPPRESS_STRING}..."
elif [ "Q$DATE2" != "Q" ]
then
	echo "- searching products for $INSTR and $DATE...$DATE2 ${PROCATG_STRING2} ${SUPPRESS_STRING}..."
elif [ $SCAN_MODE = "NON-STD" ] && [ $SETUP = ANY ]
then
	echo "- searching all $PROCATG files for $INSTR (assuming they are static files; all setups) ..."
else
	echo "- searching all $PROCATG files for $INSTR and setup $SETUP (assuming they are static files) ..."
fi

# MJD: DATE plus $DFO_OFFSET
if [ $STATIC_MODE = NO ]
then
	if [ "Q$DATE2" != "Q" ]
	then
		MJD1=`qcdate $DATE  | awk '{printf"%10.5f\n",$1+off/24+0.50}' off=$DFO_OFFSET `
		MJD2=`qcdate $DATE2 | awk '{printf"%10.5f\n",$1+off/24+1.50}' off=$DFO_OFFSET `
		DATE_STRING="$DATE ... $DATE2"
	else
		MJD1=`qcdate $DATE | awk '{printf"%10.5f\n",$1+off/24+0.50}' off=$DFO_OFFSET `
		MJD2=`qcdate $DATE | awk '{printf"%10.5f\n",$1+off/24+1.50}' off=$DFO_OFFSET `
		DATE_STRING="$DATE"
	fi
else
# defaults: 1999-01-01 and 2099-01-01
	MJD1=51179	#1999-01-01 
	MJD2=87704	#2099-01-01
fi

# =========================================================================
# 0.3 size query (taken from extractStat)
# =========================================================================
sizeQuery(){
cat > $TMP_DIR/pe_size_query <<EOT
SELECT DISTINCT
        QP.origfile ,
        NF.file_size ,
	convert(char(1),dp_id),
        "SIZE"
FROM
        ngas..ngas_files NF, qc_metadata..qc_products QP
WHERE
        NF.file_id = QP.dp_id
AND
        instrume = "$DFO_INSTRUMENT"
AND
	mjd_obs between $MJD1 and $MJD2
AND
        NF.file_size != 0
GO
EOT
rm -f $TMP_DIR/pe_filesize
isql -Uqc -S$ARCH_SERVER -w999 -P`cat $ARCH_PWD` -i $TMP_DIR/pe_size_query | grep "SIZE" | awk '{print $2,$3}' > $TMP_DIR/pe_filesize
if [ -s $TMP_DIR/pe_filesize ]
then
	M_SIZE=`cat $TMP_DIR/pe_filesize | grep "M$" | awk '{sum+=$1} END {print sum}'`
	S_SIZE=`cat $TMP_DIR/pe_filesize | grep "S$" | awk '{sum+=$1} END {print sum}'`
	A_SIZE=`cat $TMP_DIR/pe_filesize | grep "A$" | awk '{sum+=$1} END {print sum}'`
	
	M_SIZE=`echo $M_SIZE | awk '{printf"%5.1f", $1/1024/1024}'`
	A_SIZE=`echo $A_SIZE | awk '{printf"%5.1f", $1/1024/1024}'`
	S_SIZE=`echo $S_SIZE | awk '{printf"%5.1f", $1/1024/1024}'`

	ALL_SIZE=`echo "$M_SIZE $S_SIZE $A_SIZE" | awk '{print $1+$2+$3}'`
fi
}

# =========================================================================
# 0.4 Info text for STATIC_MODE=YES
# =========================================================================

cat > $TMP_DIR/pq_static <<EOT
Information for the static mode -S
==================================

1. The static mode is called with the option -S and the choice of a pro.catg. It is special in that it queries the database qc_metadata..qc_products
for the entire lifetime of the instrument.

2. The tool, and the database qc_metadata..qc_products, does not know what a static calibration file is. This knowledge is brought in by you: if you call
the tool in mode -S, the database is queried for all instances of the specified pro.catg, across the lifetime.

3. If you restrict the query to a setup, the query is nevertheless executed for all setups of the specified pro.catgs. 

4. The tool performs several consistency checks and naming convention compliance checks. 

5. What to do if multiple instances (defined by M.name) of the same origfile exist:
- check the mjd-obs; if different, it might be true that these are different files, distinguished e.g. by their pipefile name. If you would dpDelete them, this would be by origfile, meaning both get deleted!
- if one has a well-formed origfile name, and the other a non-standard one, it is likely that the both files are the same. Deletion of the non-standard name should be fine. But ...

6. ... if the non-standard name has been used in early times (before the standard names were created), they are listed in historical ABs, and these would fail. This is in particular true for science ABs. 
Maybe it is maybe better to keep the non-standard and the standard names. calSelector will deliver only one of them.

EOT

# =========================================================================
# 1.  Queries
# 1.1 STATIC_MODE=NO
# =========================================================================

cat > $TMP_DIR/pq_query1 <<EOF
select distinct
	origfile , 
	pipefile ,
	mjd_obs ,
	pro_catg ,
	ins_mode ,
	dp_id 
from 
	qc_metadata..qc_products
where
	mjd_obs between $MJD1 and $MJD2
	and instrume = "$INSTR"
	$VIRTUAL
	$PROCATG_STRING
go
EOF

# =========================================================================
# 1.2 STATIC_MODE=YES
# =========================================================================

cat > $TMP_DIR/pq_query2 <<EOF
select distinct
	origfile , 
	pipefile ,
	mjd_obs ,
	pro_catg ,
	ins_mode ,
	dp_id ,
	certified
from 
	qc_metadata..qc_products
where
	instrume = "$INSTR"
	$PROCATG_STRING
go
EOF

# =========================================================================
# 1.3 execute query
# =========================================================================

if [ $STATIC_MODE = NO ]
then
	rm -f $TMP_DIR/out_pq_query1 $TMP_DIR/pe_results
	isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $TMP_DIR/pq_query1 > $TMP_DIR/out_pq_query1 
else
	rm -f $TMP_DIR/out_pq_query1 $TMP_DIR/pe_results
	isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $TMP_DIR/pq_query2 > $TMP_DIR/out_pq_query1
fi

# =========================================================================
# 1.3 date_comments (from createReport)
# =========================================================================

if [ $SCAN_MODE = STD ]
then
	cat > $TMP_DIR/pe_datecomment_query << EOT
SELECT
        convert(varchar(10),civil_date,102) ,
        comment
FROM
        qc1..daily_comments
WHERE
        civil_date = "$DATE"
AND
        visible = "Y"
AND
        instrument = "$DFO_INSTRUMENT"
GO
EOT
	rm -f $TMP_DIR/pe_datecomment_result_
	echo "- query for date comment (DFO database) ..."
	isql -Uqc -P`cat $QC1_PWD` -S${QC1_SERVER} -w999 -i $TMP_DIR/pe_datecomment_query | sed "1,2 d" | grep -v affected | sed " /^$/d" | sed "s/   //g"  | sed "s/COMM/ /"  | sed "s/^ //" | sed "s/^...........//" > $TMP_DIR/pe_datecomment_result
fi

# =========================================================================
# 2. HTML  output
# 2.1 header
# =========================================================================

CHECK_RESULTS=`grep "(0 rows affected)" $TMP_DIR/out_pq_query1`
if [ "Q$CHECK_RESULTS" != "Q" ]
then
	echo "No data found."
	exit
fi

N_RESULTS=`grep " rows affected)" $TMP_DIR/out_pq_query1 | sed "s/(//" | awk '{print $1}'`
if [ $SCAN_MODE = STD ]
then
	TITLE_DATE="for $DATE"
else
	TITLE_DATE=""
fi

cat > $TMP_DIR/pq_header <<EOT
<html>
<title>Product data report ${TITLE_DATE}</title>
<script type="text/javascript" src="http://www.eso.org/observing/dfo/quality/ALL/jscript/jquery-latest.js"></script>
<script type="text/javascript" src="http://www.eso.org/observing/dfo/quality/ALL/jscript/jquery.dataTables.js"></script>

<!-- dataTable features, like filtering -->
<script type="text/javascript" charset="utf-8">
        \$(document).ready(function()
        {
        \$("#mainTable").dataTable({
        "aaSorting": [[5,'asc']],
        "oLanguage": {
                "sInfo": 	 "<font color=#666666 size=1><i>Showing _START_ to _END_ of _TOTAL_ files</i>",
EOT

if [ $STATIC_MODE = NO ]
then
	cat >> $TMP_DIR/pq_header <<EOT
                "sSearch":	 "<font size=1><b><a title=\"Hide VIRTUAL\" href=\"./productExplorer1.html\"><font color=#666>Hide VIRTUAL</font></a> <font color=#666> | Search:</b></font></font>",
EOT
else
	cat >> $TMP_DIR/pq_header <<EOT
                "sSearch": 	 "<font color=#666 size=1> Search:</b></font></font>",
EOT
fi

cat >> $TMP_DIR/pq_header <<EOT
                "sInfoEmpty": 	 "<font color=#666666 size=1><i>Showing 0 to 0 of 0 files</i>",
                "sInfoFiltered": "(filtered from _MAX_ total files)"
        },
        "bPaginate": false,
        "sDom": '<ift>'
         });
        });
</script>

<!-- styles for datatable -->
<style type="text/css">
table.display {
        text-align:     left;
        font-family:    arial;
        font-size:      small;
        height:         10px;
}
table.display thead tr .header {
        background-color:    #CCCCCC;
        background-image:    url(http://www.eso.org/observing/dfo/quality/ALL/img/bg.gif) no-repeat center right;
        height:              30px;
        vertical-align:      middle;
}
table.display thead tr .sorting_desc {
        background-color:    #8dbdd8;
        background: url(http://www.eso.org/observing/dfo/quality/ALL/img/desc.gif) no-repeat center right;
}
table.display thead tr .sorting {
        background-color:    #8dbdd8;
        background: url(http://www.eso.org/observing/dfo/quality/ALL/img/bg.gif) no-repeat center right;
}
table.display thead tr .sorting_asc {
        background-color:    #8dbdd8;
        background: url(http://www.eso.org/observing/dfo/quality/ALL/img/asc.gif) no-repeat center right;
}

</style>

<table width=100% border=0>
<tr bgcolor=#FFFFCC>
EOT

if [ $SCAN_MODE = STD ]
then
  	echo "  <td COLSPAN=10><h2><font color=$TITLE_COLOR>Product report for $DATE (products ingested into NGAS)</h2></td>" >> $TMP_DIR/pq_header
elif [ $SCAN_MODE = STATIC ]
then
  	echo "  <td COLSPAN=10><h2><font color=$TITLE_COLOR>Product report for static file(s) $PROCATG (products ingested into NGAS)</h2></td>" >> $TMP_DIR/pq_header
else
  	echo "  <td COLSPAN=10><h2><font color=$TITLE_COLOR>Product report (special query)</h2></td>" >> $TMP_DIR/pq_header
fi

cat >> $TMP_DIR/pq_header <<EOT
</tr>
EOT

# special info for non-STD queries
if [ $SCAN_MODE = "NON-STD" ]
then
	cat >> $TMP_DIR/pq_header <<EOT
<tr bgcolor=#FFFFCC><td align=right width=10><font size=2>INSTRUMENT:</td><td> $INSTR </tr>
<tr bgcolor=#FFFFCC><td align=right><font size=2>DATE:</td><td><font size=2>$DATE_STRING </tr>
<tr bgcolor=#FFFFCC><td align=right><font size=2>PRO_CATG:</td><td><font size=2>$PROCATG </tr>
<tr bgcolor=#FFFFCC><td align=right><font size=2>SETUP:</td><td><font size=2>"%$SETUP%"  </tr>
<tr bgcolor=#FFFFCC><td align=right><font size=2>mjd_obs:</td><td><font size=2>$MJD1 ... $MJD2 </tr>
<tr bgcolor=#FFFFCC><td colspan=2>&nbsp;</td></tr>
EOT
elif [ $SCAN_MODE = "STATIC" ]
then
	cat >> $TMP_DIR/pq_header <<EOT
<tr bgcolor=#FFFFCC><td align=right width=10><font size=2>INSTRUMENT:</td><td> $INSTR </tr>
<tr bgcolor=#FFFFCC><td align=right><font size=2>DATE:</td><td><font size=2>Entire lifetime </tr>
<tr bgcolor=#FFFFCC><td align=right><font size=2>PRO_CATG:</td><td><font size=2>$PROCATG </tr>
<tr bgcolor=#FFFFCC><td align=right><font size=2>SETUP:</td><td><font size=2>"%$SETUP%"  </tr>
<tr bgcolor=#FFFFCC><td colspan=2>&nbsp;</td></tr>
EOT
fi

# number and size of files
if [ $SCAN_MODE = STD ]
then
	sizeQuery

	cat >> $TMP_DIR/pq_header <<EOT
<tr>
  <td bgcolor=#FFFFCC>

<table border=0>
  <tr bgcolor=#CCCCCC>
    <td> </td><td><font size=2>CAL</td><td><font size=2>SCI</td><td><font size=2>ANC</td><td><font size=2>ALL</td>
  </tr>
  <tr bgcolor=#CCCCCC> 
    <td><font size=2><b>number:</b></td><td COLSPAN=4><font size=2>$N_RESULTS </td>
  </tr>
  <tr bgcolor=#CCCCCC>
    <td><font size=2><b>size (MB):</b></td><td><font size=2>$M_SIZE</td><td><font size=2>$S_SIZE</td><td><font size=2>$A_SIZE</td><td><font size=2>$ALL_SIZE</td>
</tr>
</table>

  </td>
</tr>
EOT

	if [ -s $TMP_DIR/pe_datecomment_result ]
	then
		cat >> $TMP_DIR/pq_header <<EOT
<tr bgcolor=#FFFFCC> <td align=left COLSPAN=10><font size=2><b>Date comment:</b>
EOT
		cat $TMP_DIR/pe_datecomment_result >> $TMP_DIR/pq_header
	else
		cat >> $TMP_DIR/pq_header <<EOT
<tr bgcolor=#FFFFCC> <td align=left COLSPAN=10><font size=2>&nbsp;
EOT
	fi
	echo "</td></tr>" >> $TMP_DIR/pq_header
else
	cat >> $TMP_DIR/pq_header <<EOT
<tr bgcolor=#FFFFCC> <td align=right width=10><font size=2>N_files:</td><td><font size=2>$N_RESULTS (before setup filtering, if specified)</td></tr>
EOT
fi

case $STATIC_MODE in
 "NO"  ) QUERY=$TMP_DIR/pq_query1 ;;
 "YES" ) QUERY=$TMP_DIR/pq_query2 ;;
esac
	
cat >> $TMP_DIR/pq_header <<EOT
<tr bgcolor=#FFFFCC> <td COLSPAN=10><font size=2><a href=$QUERY><font color=${TITLE_COLOR}><b>[read query]</a>
| <font size=2><a href=$TMP_DIR/mcalibDownload><font color=${TITLE_COLOR}><b>[read download file]</a> 
| <font size=2><a href=$TMP_DIR/mcalibDelete><font color=${DEL_COLOR}><b>[read deletion file]</a>
EOT

if [ $STATIC_MODE = YES ]
then
	cat >> $TMP_DIR/pq_header <<EOT
| <font size=2><a href=$TMP_DIR/pq_static><font color=${TITLE_COLOR}><b>[info for static files]</a>
EOT
fi

cat >> $TMP_DIR/pq_header <<EOT
</td></tr>
</td></tr>
</table>

<table border=0>
<tr bgcolor=#FFFFFF>
<td colspan=6><font size=2>
*type: CALIB or SCIENCE (before 2011-10) or ANCILLARY (before 2014-05) 
EOT

if [ $SUPPRESS_VIRTUAL = NO ]
then
	cat >> $TMP_DIR/pq_header <<EOT
or VIRTUAL (for calSelector datasets)
EOT
fi

cat >> $TMP_DIR/pq_header <<EOT
<br>
**setup: derived from the origfile name, applying the QC rules for products<br>
***dp_id: 
EOT

if [ $SUPPRESS_VIRTUAL = NO ]
then
	cat >> $TMP_DIR/pq_header <<EOT
either arcfile or predicted pipefile name (for VIRTUAL)
EOT
else
	cat >> $TMP_DIR/pq_header <<EOT
arcfile
EOT
fi

cat >> $TMP_DIR/pq_header <<EOT
<br>
default sorting by mjd_obs </font></td></tr>
</table>

EOT

rm -f $DFO_MON_DIR/productExplorer*.html
cat $TMP_DIR/pq_header > $DFO_MON_DIR/productExplorer.html

cat >> $DFO_MON_DIR/productExplorer.html <<EOT
<table id="mainTable" class="display" border=0>
<thead> 
 <tr bgcolor=#CCCCCC>
   <th><font size=2>type*<font color=#CCCCCC>__</font></th>
   <th><font size=2>origfile</th>
   <th><font size=2>setup**</th>
   <th><font size=2>pipefile</th>
   <th><font size=2>mjd_obs</th>
   <th><font size=2>pro_catg </th>
   <th><font size=2>ins_mode<font color=#CCCCCC>__</font></th>
   <th><font size=2>dp_id***</th>
 </tr>
</thead>

<tbody>
EOT

# =========================================================================
# 2.2 main part
# =========================================================================
# analyse results
cat $TMP_DIR/out_pq_query1 |\
 grep -v "\-\-\-" |\
 grep -v affected |\
 grep -v origfile |\
 sed "/^$/d" |\
 sed "s/000 / /" \
> $TMP_DIR/pe_results

# first 2 chars instrument? 
CHECK_INSTR=`cat $TMP_DIR/pe_results | awk '{print substr($1,1,2)}' | egrep -v "^V.|^r." | sort -u | wc -l`
if [ "Q$CHECK_INSTR" != Q1 ] && [ "Q$CHECK_INSTR" != Q0 ]
then
	echo "***WARNING: multiple or non-standard instrument tags found:"
	cat $TMP_DIR/pe_results | awk '{print substr($1,1,2)}' | grep -v "^V." | sort -u
	echo "Hit return to continue:"
	read input
fi

# next group: 4letter code? 
rm -f $TMP_DIR/pe_checkcodes
cat $TMP_DIR/pe_results | egrep -v "^ V.|^ r." | sed "s/_/ /g" | awk '{print $2}' | sort -u > $TMP_DIR/pe_checkcodes
for C in `cat $TMP_DIR/pe_checkcodes`
do
	CHECK_CODE=`echo $C | wc -c`
	if [ $CHECK_CODE != 5 ]
	then
		echo "***WARNING: '$C' is a non-standard product code!" 
		case $STATIC_MODE in
		 "NO"  ) echo "Hit return to continue:"; read input ;;
		 "YES" ) : ;;
		esac
	fi
done

if [ $STATIC_MODE = YES ]
then
	echo "Check warnings if any. Next we check for multiple ingestions of the same file. Hit return to continue:"
	read input
fi

# find correct setup string, depending on S|M|A|V
rm -f $TMP_DIR/productExplorer $TMP_DIR/pe_multiple
for F in `cat $TMP_DIR/pe_results | awk '{print $1}'`
do
	CHECK_MULTI=`grep " $F " $TMP_DIR/pe_results | awk '{print $6}' | wc -l`
	if [ "Q$CHECK_MULTI" != Q1 ]
	then
		echo $F >> $TMP_DIR/pe_multiple
		echo "***ERROR: origfile $F has more than one ingested version:"
		grep " $F " $TMP_DIR/pe_results | awk '{print $6}'
		echo "This should not happen! Data will be marked in red. "
		case $STATIC_MODE in
		 "NO"  ) echo "Hit return to continue:"; read input ;;
		 "YES" ) : ;;
		esac
	fi
	CNAME=`grep " $F " $TMP_DIR/pe_results | awk '{print $6}'`	
	CTYPE=`echo $CNAME | cut -c1,1 | sort -u`

	if [ $CTYPE = "M" ]
	then
		CSETUP=`echo $F | sed "s/_/ /" | sed "s/_/ /" | sed "s/_/ /" | awk '{print $4}' | sed "s/.tfits//" | sed "s/.fits//"`
		CTAG=CALIB
	elif [ $CTYPE = "S" ]
	then
		CSETUP=`echo $F | sed "s/_/ /" | sed "s/_/ /" | sed "s/_/ /" | sed "s/_/ /" | awk '{print $5}' | sed "s/.tfits//" | sed "s/.fits//"`
		CTAG=SCIENCE
	elif [ $CTYPE = "V" ]
	then
		CSETUP="nbsp;"
		CTAG=VIRTUAL
	else
		CSETUP=`echo $F | sed "s/_/ /"g | awk '{print $2}'`
		CTAG=ANCILLARY
	fi
	grep " $F " $TMP_DIR/pe_results | awk '{print $2,$3,$4,$5,$6}' | sed "s/^.*/$CTAG $F $CSETUP &/" | sed "s/nbsp;/\&nbsp;/" >> $TMP_DIR/productExplorer
done

if [ $SETUP != ANY ]
then
	grep "$SETUP" $TMP_DIR/productExplorer > $TMP_DIR/productExplorer1
else
	mv $TMP_DIR/productExplorer $TMP_DIR/productExplorer1
fi

cat $TMP_DIR/productExplorer1 |\
 sort -n -k4,4 |\
 sed "s| |</font></td><td><font size=1>|g" |\
 sed "/^SCIENCE/s|^.*|<tr bgcolor=${SCI_COLOR}><td><font size=1>&</font></td></tr>|" |\
 sed "/^CALIB/s|^.*|<tr bgcolor=${CAL_COLOR}><td><font size=1>&</font></td></tr>|" |\
 sed "/^VIRTUAL/s|^.*|<tr bgcolor=${LOG_COLOR}><td><font size=1>&</font></td></tr>|" |\
 sed "/^ANCILLARY/s|^.*|<tr bgcolor=${LOG_COLOR}><td><font size=1>&</font></td></tr>|" \
>> $DFO_MON_DIR/productExplorer.html

if [ -s $TMP_DIR/pe_multiple ]
then
	for F in `cat $TMP_DIR/pe_multiple`
	do
		sed -i -e "/$F/s|font size|font color=red size|g" $DFO_MON_DIR/productExplorer.html
	done	
fi

# =========================================================================
# 2.3 footer
# =========================================================================

cat >> $DFO_MON_DIR/productExplorer.html <<EOT
</tbody>

<tfoot>
<tr BGCOLOR=#CCCCCC> <td ALIGN=RIGHT COLSPAN=99> <font COLOR=#FFFFFF size=2> created by productExplorer v${TOOL_VERSION}, a <b>dfos</b> tool</font> </td> </tr> 
</tfoot>
</table>
EOT

# =========================================================================
# 2.4 version without VIRTUAL
# =========================================================================
# relies on "<font size=1>VIRTUAL"!
grep -v "font size=1>VIRTUAL" $DFO_MON_DIR/productExplorer.html |\
 sed "s/Hide VIRTUAL/Display VIRTUAL/"g |\
 sed "s/productExplorer1.html/productExplorer.html/" |\
 sed "s/or VIRTUAL (for calSelector datasets)//" |\
 sed "s/either arcfile or predicted pipefile name (for VIRTUAL)/arcfile name/" \
> $DFO_MON_DIR/productExplorer1.html

# =========================================================================
# 3 Download files
# 3.1 prepare text file to download mcalibs ...
# =========================================================================

if [ $SUPPRESS_VIRTUAL = NO ]
then
	VIRTUAL_TEXT="# Virtual products cannot be downloaded and are suppressed here."
else
	VIRTUAL_TEXT=""
fi

cat > $TMP_DIR/mcalibDownload <<EOT
#!/bin/sh
# Download: Edit and execute this file (\$TMP_DIR/mcalibDownload). 
# Columns after the filename can be used for filtering (they get ignored by ngasClient).
$VIRTUAL_TEXT
# Downloads into \$DFO_CAL_DIR/\$DOWNLOAD
# You may want to change the download directory (but you don't need to do so):
DOWNLOAD="DOWNLOAD"
#

if [ ! -s \$DFO_CAL_DIR/\$DOWNLOAD ]
then
	mkdir $DFO_CAL_DIR/\$DOWNLOAD
fi
cd $DFO_CAL_DIR/\$DOWNLOAD

EOT

# =========================================================================
# 3.2 bulk download
# =========================================================================

cat $TMP_DIR/out_pq_query1 |\
 grep -v "\-\-\-" |\
 grep -v affected |\
 grep -v origfile |\
 sed "/^$/d" |\
 awk '{print $1,$2,$3,$4,$5}' |\
 grep -v "^V\." |\
 sed "s/^.*/ngasClient -c &/" > $TMP_DIR/mcalibDownload1

if [ $SETUP != ANY ]
then
	grep "$SETUP" $TMP_DIR/mcalibDownload1 >> $TMP_DIR/mcalibDownload
else
	cat $TMP_DIR/mcalibDownload1 >> $TMP_DIR/mcalibDownload
fi

chmod u+x $TMP_DIR/mcalibDownload

# =========================================================================
# 4 Delete entries
# 4.1 prepare text file to delete mcalibs ...
# =========================================================================

if [ $SUPPRESS_VIRTUAL = NO ]
then
	VIRTUAL_TEXT="# Virtual products cannot be deleted and are suppressed here."
else
	VIRTUAL_TEXT=""
fi
rm -f $TMP_DIR/mcalibDelete
cat > $TMP_DIR/mcalibDelete <<EOT
#!/bin/sh
# Delete: Edit and execute this file ($TMP_DIR/mcalibDelete)
$VIRTUAL_TEXT
echo "
To run in simulation mode: call \$TMP_DIR/mcalibDelete.
To run in enforced mode:   edit entries to become 'dpDelete <file> -force'
If you just hit return, tool will only run in SIMULATION mode!
Hit return:"
read input

# ENTRIES:
EOT

cat $TMP_DIR/pe_results | awk '{print $1}' | grep -v "^V\." | sed "s/^.*/dpDelete &/" >> $TMP_DIR/mcalibDelete

chmod u+x $TMP_DIR/mcalibDelete

# =========================================================================
# 5 End: refresh browser display
# =========================================================================

$DFO_GUI_DIR/refresh_browser $DFO_MON_DIR/productExplorer.html

echo "... done. Find results in $DFO_MON_DIR/productExplorer.html.
- For download, edit and execute \$TMP_DIR/mcalibDownload .
- For deletion, edit and execute \$TMP_DIR/mcalibDelete .
"

exit

