#!/bin/sh
# PURPOSE:     	Replace FITS files in $DFO_CAL_DIR/<date> etc. with HDR files
# AUTHOR:       John Pritchard / ESO
# VERSIONS:     0.5 -- CREATION: November 2007
#		1.0 -- released: January 2008
#		1.1 -- Added ngas_files check and broke many file checks into 500 file chunks [JDP] (2008-06-17)
#		1.2 -- Cleans BOTH CALIBs and SCIENCE for the specified night [JDP] (2008-07-28)
#		1.3 -- Includes ancillary fits files in $DFO_LOG_DIR and $DFO_PLT_DIR (2008-09-22)
#		1.3.1- saves extension from ancillary fits (for packing by headers) (2008-10-09)
#		1.3.2- bug fixed with tag for ORIGFILE (2008-10-13)
#		1.3.3- cosmetic improvement which avoids unnecessary WARNINGS [JDP] (2009-07-13)
#		1.3.4- improved handling of large number of files [RHa] (2010-01-05)
#		1.4 -- supports cleanup of $DFO_MON_DIR/MCAL_DOWNLOAD [RHa] (2011-06-09)
#		2.0 -- supports 'explore' mode: complete scan of all directories [RHa] (2011-06-16)
#		2.1 -- added: override option -O to force cleaning up [RHa] (2011-11-28)
#		2.1.1- calib..data_products replaced by qc_metadata..qc_products (2014-09-03)
#		2.1.2- adding ORIGFILE no longer needed with dpIngest (2014-09-16)
#		3.0 -- support IDPs; new option -n for EXPLORE&PHOENIX (2015-09-25)
#		3.1 -- support for ANC files removed (2015-10-19)
#		3.2 -- updated to support new phase3 infrastructure; database name: phase3-->phase3v2 (2016-04-27)
#		3.3 -- have CLEANUP_PLUGIN for PHOENIX; better checks for phase3 ingestion (2016-06-22)
# 		3.3.1- fix bug in line 403 (2019-02-04)
# PARAMETERS:   -d <date> 
#               
# OUTPUT:       Headers in $DFO_CAL_DIR/<date>|$DFO_SCI_DIR
# CONFIG:       config.ingestProducts, contains name of CLEANUP_PLUGIN (optional, for PHOENIX only)
# NOTES:	supports PHOENIX
# =========================================================================
TOOL=cleanupProducts
TOOL_VERSION=3.3.1

# ==========================================================================
# 0. Initialize
# 0.1 Define help
# ==========================================================================

if [ Q$THIS_IS_PHOENIX = Q ]
then
	THIS_IS_PHOENIX=NO
fi

if [ Q$THIS_IS_MCAL != QYES ]
then
	THIS_IS_MCAL=NO
fi

# needed for PHOENIX MCALIB cleanup; THIS_IS_MCAL defined in .dfosrc_X
if [ $THIS_IS_PHOENIX = YES ] && [ Q$THIS_IS_MCAL = QYES ]
then
	THIS_IS_PHOENIX=NO
fi

cat > $TMP_DIR/cleanupProducts_help <<EOT
USAGE:
	cleanupProducts -v | -h | -d <date> | -E [-n] | -O
        Replaces ingested CALIB and SCIENCE fits files of the specified date with HDR files.
	Use only after successful product ingestion and finishNight.
	Date: date to process
	Option -E: explore all earlier dates for hidden fits files
	       -E -n: same, without interrupt on the command line (for PHOENIX historical batch)
	Option -O: override warning "not yet finished", you know what you do
EOT

if [ $THIS_IS_PHOENIX = YES ]
then
	cat >> $TMP_DIR/cleanupProducts_help <<EOT
	
	For usage within PHOENIX: 
	Replaces ingested IDPs and their associated fits files in the \$DFO_SCI_DIR/<date>/conv directory with HDR
	files. The \$DFO_SCI_DIR/<date> directory is emptied. Graphical and text files in the conv directory are  
   	replaced with symlinks. You can optionally specify a CLEANUP_PLUGIN in config.ingestProducts that is executed
	at the end.
EOT
fi

# ==========================================================================
# 0.2 get options
# ==========================================================================

EXPLORE=NO
INTERRUPT=YES
INT_FLAG=""
OVERRIDE=NO	#hidden option for -E

case $THIS_IS_PHOENIX in
 "NO"  ) TYPE=CAL ;;	#hidden option for -E
 "YES" ) TYPE=SCI ;;
esac

while getopts d:EOt:hvn OPTION
do
	case "$OPTION" in
	 v ) echo $TOOL_VERSION
             exit 0 ;;
         h ) cat $TMP_DIR/cleanupProducts_help
             exit 0 ;;
	 d ) export DATE=${OPTARG} ;;
	 E ) EXPLORE=YES ;;
	 n ) INTERRUPT=NO ; INT_FLAG=-n ;;
	 O ) OVERRIDE=YES ;;
	 t ) TYPE=${OPTARG} ;;
	 ? ) cat $TMP_DIR/cleanupProducts_help
             exit 0 ;;
	esac
done

if [ "Q$DATE" = "Q" ] && [ $EXPLORE = NO ]
then
	cd $DFO_LOG_DIR
	LAST_DATE=`ls -d [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] | tail -1`
	echo "You must enter a date (last available: $LAST_DATE). Exit."
	echo ""
	cat $TMP_DIR/cleanupProducts_help
	exit -1
fi

CLEANUP_PLUGIN=NONE
if [ $THIS_IS_PHOENIX = YES ]
then
	CLEANUP_PLUGIN=`grep "^CLEANUP_PLUGIN"  $DFO_CONFIG_DIR/config.ingestProducts | awk '{print $2}'`
	if [ Q$CLEANUP_PLUGIN = Q ]
	then
		CLEANUP_PLUGIN=NONE
	fi
fi

# ==========================================================================
# 0.3 removeFromJobs: remove executed jobs from JOBS_CLEANUP
# ==========================================================================

removeFromJobs(){
if [ -s $DFO_JOB_DIR/JOBS_CLEANUP ]
then
	sed -i -e "/^cleanupProducts -d ${DATE}/d" $DFO_JOB_DIR/JOBS_CLEANUP
else
	cat > $DFO_JOB_DIR/JOBS_CLEANUP <<EOT
#!/bin/sh
# This is the jobs file for cleanup products (after successful run of ingestProducts)

EOT
fi
chmod u+x $DFO_JOB_DIR/JOBS_CLEANUP
}

# ==========================================================================
# 1. EXPLORE=YES: check whole directory tree for left-behind fits files
# This is a special  mode which uses the normal mode in a wrapper style
# 1.1 Organize
# ==========================================================================

if [ $EXPLORE = YES ]
then
	case $TYPE in
	 "CAL" ) TYPE_DIR=$DFO_CAL_DIR ;;
	 "SCI" ) TYPE_DIR=$DFO_SCI_DIR ;;
	esac

	if [ $TYPE = CAL ]
	then
		echo "You have entered option -E for exploring all dates in \$DFO_CAL_DIR|\$DFO_SCI_DIR for left-behind fits files. 
The tool will lead you through a dialogue. For all dates found, there will be the check for archive ingestion before actually deleting a fits file.

1. Check out \$DFO_CAL_DIR. Hit return:"
	elif [ $TYPE = SCI ]
	then
		case $THIS_IS_PHOENIX in
		 "NO"  ) echo "2. We check remaining dates in \$DFO_SCI_DIR. Hit return:" ;;
		 "YES" ) echo "Exploration mode for historical IDP batches can take long. Pls. be patient ..."
			 if [ $INTERRUPT = NO ] && [ $EXPLORE = YES ] 
			 then
				echo "Confirm that you don't want acknowledgment after each date (expert mode) (Y/N) (N):"
				read CONFIRM
				if [ Q$CONFIRM != QY ]
				then
					echo "No confirmation. Start again without flag -n."
					exit
				else
					echo "Confirmed. You will still be asked twice for other issues but then not again."
				fi
			 fi
			 echo ""
			 echo "[1. No check for \$DFO_CAL_DIR (maintained by phoenix).]"
			 echo "2. We check remaining dates in \$DFO_SCI_DIR. There is no check for the 'finished' flag. Hit return:" ;;
		esac
	fi
	read input
	cd $TYPE_DIR

# ==========================================================================
# 1.2 Find dates
# 1.2.1 Already finished dates
# ==========================================================================

	rm -f $TMP_DIR/cp_all_fits $TMP_DIR/cp_all_dates $TMP_DIR/cp_finished_dates $TMP_DIR/cp_notfinished_dates
	for YY in `ls -d 20* 2>/dev/null | cut -c1-4 | sort -u`
	do
		ls ${YY}*/*fits 2>/dev/null >> $TMP_DIR/cp_all_fits
		if [ $THIS_IS_PHOENIX = YES ] && [ $TYPE = SCI ]
		then
			ls ${YY}*/conv/*fits 2>/dev/null >> $TMP_DIR/cp_all_fits
		fi
	done

	if [ ! -s $TMP_DIR/cp_all_fits ]
	then
		echo "No left-behind fits files found."
	else
		cat $TMP_DIR/cp_all_fits | sed "s|/| |" | awk '{print $1}' | sort -u > $TMP_DIR/cp_all_dates
		if [ $THIS_IS_PHOENIX = NO ]
		then
			echo "a) Finished dates with left-behind fits files:"
			for D in `cat $TMP_DIR/cp_all_dates`
			do
				CHECK_FINISHED=`grep "^finished $D" $DFO_MON_DIR/DFO_STATUS | head -1` 
				if [ "Q$CHECK_FINISHED" != Q ]
				then
					echo $D >> $TMP_DIR/cp_finished_dates
				fi
			done
		else
			echo "a) Dates with left-behind fits files:"
			cp $TMP_DIR/cp_all_dates $TMP_DIR/cp_finished_dates
		fi 

		if [ -s $TMP_DIR/cp_finished_dates ]
		then
			cat $TMP_DIR/cp_finished_dates
			echo "For these dates we now cleanup all fits files (they would otherwise occupy disk space)."
			echo "We call 'cleanupProducts -d <date>' and then come back to the other dates."
			if [ $INTERRUPT = NO ]
			then
				echo "You can interrupt the workflow anytime and come back later (this is the last time that we ask for confirmation; hit return):"
			else
				echo "You can interrupt the workflow anytime and come back later (hit return):"
			fi
			read input

# call cleanupProducts in normal way
			for D in `cat $TMP_DIR/cp_finished_dates`
			do
				echo "$D ..."
				cleanupProducts -d $D $INT_FLAG | sed "s/^.*/	&/"
			done
		else

# ==========================================================================
# 1.2.2 Unfinished dates
# ==========================================================================

			echo "... none found. 

b) We continue with unfinished dates (the ones for which we couldn't find a 'finished' entry in DFO_STATUS 
but other entries do exist). 
Hit return:"
			read input

			for D in `cat $TMP_DIR/cp_all_dates`
			do
				CHECK_ANY=`grep " $D " $DFO_MON_DIR/DFO_STATUS`
				if [ "Q$CHECK_ANY" != Q ]
				then
					echo $D >> $TMP_DIR/cp_notfinished_dates
				fi
			done

			if [ -s $TMP_DIR/cp_notfinished_dates ]
			then
				cat $TMP_DIR/cp_notfinished_dates
				echo "
Please investigate these dates one-by-one, we cannot decide what to do.
Maybe these dates are still under dfo workflow and not yet finished, or
from historical dates for which no entry exists in DFO_STATUS. You can 
then call the tool with option -O[verride].

Or, you might have recently reprocessed some ABs in which case you could 
either remove all entries in DFO_STATUS or add the 'finished' flag.

We remove them from the list of TBD dates and continue with the remaining ones.
Hit return:"
				read input
				for D in `cat $TMP_DIR/cp_notfinished_dates`
				do
					sed -i -e "/$D/d" $TMP_DIR/cp_all_dates
				done
			else
				echo "... none found."
			fi
		
# ==========================================================================
# 1.2.3 Remaining dates
# ==========================================================================

			if [ -s $TMP_DIR/cp_all_dates ]
			then
				echo "
c) We continue with the remaining dates. 
Hit return to display the list of dates (\$TMP_DIR/cp_all_dates):"
				read input
				cat $TMP_DIR/cp_all_dates
				echo "You can interrupt anytime and come back later.
Hit return to continue:"
				read input

# call cleanupProducts in override mode (don't care about 'finished' flag since it's outdated)

				for D in `cat $TMP_DIR/cp_all_dates`
				do
					echo "$D ..."
					cleanupProducts -d $D -O | sed "s/^.*/	&/"
				done
			else
				echo "... none found."
			fi

		fi
	fi

# ==========================================================================
# 1.3 Done --> fork for next $TYPE
# ==========================================================================

	case $TYPE in
	 "CAL" ) echo "We have now finished cleaning up for the dates found in $TYPE_DIR.  
Hit return:"
		 read input
		 cleanupProducts -E -t SCI $INT_FLAG ;;
	 "SCI" ) echo "We have now finished cleaning up for the dates found in $TYPE_DIR. You may want to call this mode again for verification. Exit.
Hit return:"
		 read input
		 exit ;;
	esac
	exit
fi

# ==========================================================================
# 2. Daily cleanup
# 2.1 Check that the night is already finished (otherwise statistics would be corrupted)
# 2.1.1 DFOS
# ==========================================================================

rm -f $TMP_DIR/cp_query.* 
if [ $THIS_IS_PHOENIX = NO ]
then
	CHECK_FINISHED=`grep "^finished" $DFO_MON_DIR/DFO_STATUS | grep " $DATE "`

	if [ "Q$CHECK_FINISHED" = "Q" ] && [ $OVERRIDE = NO ]
	then
		echo "
***WARNING: date $DATE is not yet finished, or has no data, can't be cleaned up.
Either finish the date first and retry (E[xit]), or O[verride] warning and continue (E):"

		read CONTI_YN
		if [ Q$CONTI_YN != QO ]
		then
			exit
		fi	
		OVERRIDE=YES
		echo "***INFO: override option set, OK to cleanup."
	elif [ $OVERRIDE = YES ]
	then
		echo "***INFO: no finished flag found but override option set, OK to cleanup."
	else
		echo "
***INFO: date $DATE has been finished already, OK to cleanup."
	fi
fi

# ==========================================================================
# 2.1.2 PHOENIX 
# ==========================================================================

if [ $THIS_IS_PHOENIX = YES ]
then
# PHOENIX: check for 'finished' flag and for existence of $DFO_SCI_DIR/<date>/conv
# (unclear if correct)
	if [ ! -d $DFO_SCI_DIR/$DATE/conv ]
	then
		echo "***WARNING: no $DFO_SCI_DIR/$DATE/conv found, can't be cleaned up. Exit."
		exit
	fi
	CHECK_FINISHED=`grep "^finished $D" $DFO_MON_DIR/DFO_STATUS | head -1` 
	if [ "Q$CHECK_FINISHED" = "Q" ]
	then
		echo "
***WARNING: date $DATE is not yet finished, can't be cleaned up. Exit."
		exit
	fi
fi
	
# ==========================================================================
# 2.2 prologue
# ==========================================================================

case $THIS_IS_PHOENIX in
 "NO"  ) SOURCE=NGAS ; DIR=DATE ; ADD="" ;;
 "YES" ) SOURCE=phase3 ; DIR=DATE/conv ; ADD="4) Replace all graphical files by symlinks" ;;
esac

echo "For each FITS file in the $DIR directories specified on the command line we:
  1) Check for existence of that file in $SOURCE (successful ingestion)
  2) Extract the HEADER
  3) Delete the FITS file
  $ADD"

if [ $CLEANUP_PLUGIN != NONE ]
then
	echo "  5) Call $CLEANUP_PLUGIN"
fi

# ==========================================================================
# 2.3 Create file list; check that some FITS files exist
# ==========================================================================
# list_all_nonanc_files: DFOS 	 --> all FITS files in DFO_CAL_DIR and DFO_SCI_DIR
#                        PHOENIX --> all FITS files in DFO_SCI_DIR/<date>/conv (IDPs and non-IDPs)
# (list_all_anc_files existed for historical reasons, no support anymore)

someExist=0
rm -f $TMP_DIR/list_all_files $TMP_DIR/list_all_nonanc_files

case $THIS_IS_PHOENIX in
 "NO"  ) ls $DFO_CAL_DIR/$DATE/ 2>/dev/null | grep ".fits$" | sed "s|^.*|$DFO_CAL_DIR/$DATE/&|" >  $TMP_DIR/list_all_files
	 ls $DFO_SCI_DIR/$DATE/ 2>/dev/null | grep ".fits$" | sed "s|^.*|$DFO_SCI_DIR/$DATE/&|" >> $TMP_DIR/list_all_files ;;
 "YES" ) ls $DFO_SCI_DIR/$DATE/conv/ 2>/dev/null | grep ".fits$" | sed "s|^.*|$DFO_SCI_DIR/$DATE/conv/&|" > $TMP_DIR/list_all_files ;;
esac

if [ -s $TMP_DIR/list_all_files ]
then
	cp $TMP_DIR/list_all_files $TMP_DIR/list_all_nonanc_files
fi

if [ -s $TMP_DIR/list_all_files ]
then
	for L in `cat $TMP_DIR/list_all_files`
	do
		L1="`eval echo ${L}`"
		if [ -e "${L1}" ]
		then
	    		someExist=1
			break
		fi
	done
fi

if [ $someExist -eq 0 ]
then
	case $THIS_IS_PHOENIX in
	 "NO"  ) echo "*** INFO: No FITS files found in \$DFO_CAL_DIR/$DATE | \$DFO_SCI_DIR/$DATE. Hit return:" ;;
	 "YES" ) echo "*** INFO: No FITS files found in \$DFO_SCI_DIR/$DATE/conv. " ;;
	esac

	if [ $THIS_IS_PHOENIX = YES ]
	then
		rm -f $TMP_DIR/cp_list_files
 		ls $DFO_SCI_DIR/$DATE/ 2>/dev/null | grep ".fits$" | sed "s|^.*|$DFO_SCI_DIR/$DATE/&|" > $TMP_DIR/cp_list_files
		if [ -s $TMP_DIR/cp_list_files ]
		then
			echo ""
			echo "... FITS files found in $DFO_SCI_DIR/$DATE:"
			for F in `cat $TMP_DIR/cp_list_files`
			do
				F1=`basename $F`
				rm $F
				echo "	$F1 deleted (not archived)"
			done
		fi
		if [ $INTERRUPT = YES ]
		then
			echo "Hit return:"
			read input
		fi
	fi
	removeFromJobs
	exit 0
else
	NUMBER_FILES=`cat $TMP_DIR/list_all_files | wc -l`
	echo "*** INFO: $NUMBER_FILES FITS files found. "
fi

# ==========================================================================
# 2.4 Check the Archive for the files ...
# 2.4.1 DFOS
# ==========================================================================

if [ $THIS_IS_PHOENIX = NO ]
then
	echo "Checking CDB for files..."
	cat > $TMP_DIR/cp_query.$$ <<EOF
select distinct
        qp.origfile , 
        qp.mjd_obs ,
        qp.pro_catg ,
        qp.ins_mode ,
        qp.dp_id 
from 
        qc_metadata..qc_products qp, ngas..ngas_files f
where
        qp.origfile in (
EOF
	i=0
	COMMA="  "
	for L in `cat $TMP_DIR/list_all_files`
	do
		(( i++ ))
		L1="`eval echo ${L}`"
		CDBFILE=`basename $L1`
		cat >> $TMP_DIR/cp_query.$$ <<EOF
       ${COMMA}"${CDBFILE}"
EOF
		COMMA=", "
		if [ $i -eq 500 ]
		then
			cat >> $TMP_DIR/cp_query.$$ <<EOF
         ) and qp.dp_id=f.file_id
go
select distinct
        qp.origfile , 
        qp.mjd_obs ,
        qp.pro_catg ,
        qp.ins_mode ,
        qp.dp_id 
from 
        qc_metadata..qc_products qp, ngas..ngas_files f
where
        qp.origfile in (
EOF
		  	i=0
			COMMA="  "
		fi
	done
	cat >> $TMP_DIR/cp_query.$$ <<EOF
         ) and qp.dp_id=f.file_id
go
EOF

	rm -f $TMP_DIR/cp_query.out.$$
	isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $TMP_DIR/cp_query.$$ -o $TMP_DIR/cp_query.out.$$
fi

# ==========================================================================
# 2.4.2 PHOENIX 
# ==========================================================================

if [ $THIS_IS_PHOENIX = YES ]
then
	echo "Checking phase3v2 database for files..."
	cat > $TMP_DIR/cp_query.$$ <<EOF
select
	archived,
	archive_id,
	name
from 
	phase3v2..files
where
	archived = 1
and
        name in (
EOF
	i=0
	COMMA="  "
	for L in `cat $TMP_DIR/list_all_files`
	do
		(( i++ ))
		L1="`eval echo ${L}`"
		IDPFILE=`basename $L1`
		cat >> $TMP_DIR/cp_query.$$ <<EOF
       ${COMMA}"${IDPFILE}"
EOF
		COMMA=", "
		if [ $i -eq 500 ]
		then
			cat >> $TMP_DIR/cp_query.$$ <<EOF
         )
go

select
	archived,
	archive_id,
	name
from 
	phase3v2..files
where
	archived = 1
and
        name in (
EOF
		  	i=0
			COMMA="  "
		fi
	done
	cat >> $TMP_DIR/cp_query.$$ <<EOF
         )
go
EOF

	rm -f $TMP_DIR/cp_query.out.$$
	isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $TMP_DIR/cp_query.$$ -o $TMP_DIR/cp_query.out.$$
fi

# ==========================================================================
# 2.5 If file is in NGAS/phase3v2 make the HDR file, then delete FITS
# 2.5.1 DFOS: calib and science products
# ==========================================================================

if [ $THIS_IS_PHOENIX = NO ] && [ -s $TMP_DIR/list_all_nonanc_files ]
then
	for L in `cat $TMP_DIR/list_all_nonanc_files`
	do
		L1="`eval echo ${L}`"
		CDBFILE="`basename $L1`"

# this doesn't work safely: there could be a '.' in the filename...
		HDRFILE=`echo $L1 | sed "s/\.fits/.hdr/" | sed "s/\.tfits/.hdr/"`

		if [[ `grep ${CDBFILE} $TMP_DIR/cp_query.out.$$` ]]
		then
			dfits $L1 | grep -v "<====" | grep -v "^HISTORY" > ${HDRFILE}
			if [ $? -ne 0 ] && [ -e "${HDRFILE}" ]
			then
				echo "*** ERROR: error creating HDR for ${CDBFILE}"
			else
				rm -f $L1
				echo "*** INFO: ${CDBFILE}	replaced by header."
			fi
		else
			echo "*** WARNING: ${L1}	not yet archived, can't delete."
		fi
	done
fi

# ==========================================================================
# 2.5.2 PHOENIX: science products
# ==========================================================================

if [ $THIS_IS_PHOENIX = YES ] && [ -s $TMP_DIR/list_all_nonanc_files ]
then
	echo "Extracting hdr files, removing fits files, symlinking graphics in \$DFO_SCI_DIR/$DATE/conv ..."
	ERROR_LOG=$TMP_DIR/cp_errorlog
	rm -f $ERROR_LOG
# FITS files: replaced by HDR 
	for L in `cat $TMP_DIR/list_all_nonanc_files`
	do
		L1="`eval echo ${L}`"
		IDPFILE="`basename $L1`"
# IDPROOT is composed of $3=OBS_ID, $4=timestamp
		IDPROOT=`echo $IDPFILE | sed "s/_/ /g" | awk '{print $3"_"$4}'`
		HDRFILE=`echo $L1 | sed "s/\.fits/.hdr/"`

# now check if *ALL* fits files are ingested; otherwise exit here
		if [[ `grep ${IDPFILE} $TMP_DIR/cp_query.out.$$` ]]
		then
			echo "	${IDPFILE}	archived ok."
		else
			echo "	${IDPFILE}	not (yet) archived or archiving failed, can't clean up." | tee -a $ERROR_LOG
			echo "*** ERROR: we cannot cleanup since at least one file failed phase3 ingestion. Check and re-try. Exit."
			exit -1
		fi

		if [[ `grep ${IDPFILE} $TMP_DIR/cp_query.out.$$` ]]
		then
			dfits -x 0 $L1 | grep -v "<====" | grep -v "^HISTORY" > ${HDRFILE}
			if [ $? -ne 0 ] && [ -e "${HDRFILE}" ]
			then
				echo "*** ERROR: error creating HDR for ${IDPFILE}" | tee -a $ERROR_LOG
			else
				rm -f $L1
# for $DFO_SCI_DIR: we delete all fits files that contain the OB_ID and timestamp, no matter which 4letter code, no matter which
# setup string. This will remove: intermediate products, plus the UVES case of format changes between $DFO_SCI_DIR and conv.
				rm -f $DFO_SCI_DIR/$DATE/*${IDPROOT}*fits $DFO_SCI_DIR/$DATE/*${IDPROOT}*logs
			fi
		else
			echo "*** ERROR: ${L1}	not (yet) archived or archiving failed, can't delete." | tee -a $ERROR_LOG
		fi
	done

# GRAPHICS: replaced by symlink to $DFO_PLT_DIR/$DATE; all gif/jpg/ps/png
	rm -f `ls $DFO_SCI_DIR/$DATE/conv/* | grep "r.${DFO_FILE_NAME}" | egrep "\.gif|\.ps|\.jpg|\.png"`
	ln -sf `ls $DFO_PLT_DIR/$DATE/*     | grep "r.${DFO_FILE_NAME}" | egrep "\.gif|\.ps|\.jpg|\.png"` $DFO_SCI_DIR/$DATE/conv/

	echo "*** INFO: ingested FITS files replaced by header."
	echo "*** INFO: graphical files replaced by symbolic links; text files untouched."
# TEXT: just leave there

# mail if error found
	if [ -s $ERROR_LOG ]
	then
		echo "***ERROR found:"
		cat $ERROR_LOG
		mail -s "$TOOL: error found for $DATE" $OP_ADDRESS <$ERROR_LOG 
	else
		echo "No error found."
	fi

# files in $DFO_SCI_DIR? Delete
	rm -f $TMP_DIR/cp_list_files*
 	ls $DFO_SCI_DIR/$DATE/ 2>/dev/null | grep ".fits$" | sed "s|^.*|$DFO_SCI_DIR/$DATE/&|" > $TMP_DIR/cp_list_files
 	ls $DFO_SCI_DIR/$DATE/ 2>/dev/null | grep ".fits$" > $TMP_DIR/cp_list_files1
	if [ -s $TMP_DIR/cp_list_files ]
	then
		echo "... FITS files found in $DFO_SCI_DIR/$DATE:"
		cat  $TMP_DIR/cp_list_files1
		echo "File deletion: if all went fine with the ingestion, these FITS files are orphans (not IDPs, not ancillary files) and can be deleted.
If there was an ingestion error, investigate carefully, your process might have missed ingesting them. Should we delete them (Y/N) (Y)?"
		read DELETE_YN
		if [ "Q$DELETE_YN" = QN ]
		then
			echo "OK, files not deleted, please investigate and then try again. Exit"
			exit
		fi

		for F in `cat $TMP_DIR/cp_list_files`
		do
			F1=`basename $F`
			rm $F
			echo "$F1 deleted (not archived)"
		done
	fi

# now call CLEANUP_PLUGIN
	if [ $CLEANUP_PLUGIN != NONE ]
	then
		source $DFO_BIN_DIR/$CLEANUP_PLUGIN
	fi
fi

# ==========================================================================
# 3. Check MCAL_DOWNLOAD: DFOS only
#    These are files which have been downloaded *after* a previous, regular 
#    call of cleanupProducts, since they were needed by processAB
#    Most likely these are mcalibs. Certainly they are in NGAS. Hence we don't 
#    check for NGAS availability again.
# ==========================================================================

if [ -s $DFO_MON_DIR/MCAL_DOWNLOAD ] && [ $THIS_IS_PHOENIX = NO ]
then
	CHECK_EXIST=`cat $DFO_MON_DIR/MCAL_DOWNLOAD | grep -v "^#" | grep fits | head -1`
	if [ Q$CHECK_EXIST != Q ]
	then
		echo ""
		echo "- We clean up $DFO_MON_DIR/MCAL_DOWNLOAD list. Hit return:"
		read input
		for F in `cat $DFO_MON_DIR/MCAL_DOWNLOAD | grep -v "^#" | grep fits  | sort -u`
		do
			F1=`echo $F | sed "s/.fits/.hdr/"`
			if [ ! -s $F1 ] && [ -s $F ]
			then
				echo "***ERROR: $F1 does not exist, we cannot delete $F. Please check. Hit return:"
				read input
				continue
			fi
			
			rm -f $F
			grep -v "$F" $DFO_MON_DIR/MCAL_DOWNLOAD > $DFO_MON_DIR/MCAL_DOWNLOAD1
			mv $DFO_MON_DIR/MCAL_DOWNLOAD1 $DFO_MON_DIR/MCAL_DOWNLOAD
			echo "*** INFO: ${F}	removed."
		done
	fi
fi

# ==========================================================================
# 4. Remove entry in JOB file JOBS_CLEANUP
# ==========================================================================

removeFromJobs

# ==========================================================================
# 5. End
# ==========================================================================

rm -f $TMP_DIR/cp_query.out.$$
rm -f $TMP_DIR/cp_query.$$

if [ $OVERRIDE = NO ]
then
	UPDATE=`date +%Y-%m-%d"T"%H:%M:%S`
	echo "cal_Cleaned $DATE $UPDATE" >> $DFO_MON_DIR/DFO_STATUS
fi

if [ $INTERRUPT = NO ] && [ $THIS_IS_PHOENIX = YES ]
then
	echo "... done."
	echo ""
else
	echo "... done. Hit return:"
	read input
fi

exit
