#!/bin/sh
# NAME:		finishNight
# PURPOSE: 	- clean up at the end of the daily workflow
#		- provide statistics about reduced and calib data
# DATE:		2004-11-19
# AUTHOR:	Reinhard Hanuschik, ESO-DMD
#
# VERSIONS: 	1.0 -- copied from UVES version 1.13 and turned into a dfos tool (2004-11-19)
#		1.1 -- calls extractStat, has logging (2005-05-10)
#		1.1.1- removes dags from $CONDOR_DAG_DIR (2005-06-10)
#		1.1.2- removes AB logs in $DFO_AB_DIR (2005-12-15)
#		1.1.3- checks for open ingest jobs (2006-01-23)
#		1.1.4- no call of extractStat if LSPACK_MODE set (2006-02-23)
#		1.1.5- no check for DRS_TYPE required anymore (2006-03-03)
#		1.1.6- now checks for $BQSROOT instead of $CONDOR_DAG_DIR (2006-03-13)
#		1.1.7- introducing DELETE_RAW (2006-03-15)
#		1.1.8- qcdate replacing mjd2date; dfosLog suspended (2006-05-16)
#		1.1.9- removes $HIDE_DIR/$DATE (2006-11-22)
#		1.2 -- calls createReport and histoMonitor in the end (2008-01-08)
#		1.3 -- option -c added (cleanup only, no statistics updated) (2008-01-30)
#		1.3.1- also cleans up status files on $DFO_WEB_SERVER (2008-12-16)
#		1.3.2- improved deletion of CONDOR workspace (2010-05-04)
#		1.3.3- stabilized DFO_STATUS (2010-09-20)
#		1.3.4- bug fix with N_LINES1 (2010-09-30)
#		1.3.5- LSPACK_MODE and ING_FILE_NAME removed; check for sci_Harvested (2011-10-11)
#		1.3.6- config file obsolete; RAW data always deleted (2012-09-26)
#		1.3.7- delete QC cascades (line :204) (2013-05-13)
#		1.4 -- common support for DFOS and PHOENIX installations (2014-11-17)
#		1.4.1- checks for harvestAB entry dropped (2015-06-22)
#		1.4.2- extends condor work dir deletion to PHOENIX case of QCSCIENCE_all (sect 2.4) (2017-09-18)
#		1.5 -- adds completeness check at the end, in order to catch missing headers for science files (2017-12-11)
#		1.5.1- line 259ff edited to include step2 processing (2018-06-01)
#		1.5.2- suppress qcdate error for DEEP_MODE (2019-01-03)
# 		1.5.3- some cosmetics for case 'no hdr' (2020-03-24)
#		1.5.4- nasty bug fix for DEEP_MODE and new runID format (2021-05-11) [RHA]
#
# CONFIG:	none
#		also: config.filterRaw for $HIDE_DIR; config.phoenix for QCJOB_CONCENTRATION
# TOOLS:	extractStat, qcdate; createReport, histoMonitor
# COMMENTS: 	supporting PHOENIX
# ==============================================================================
TOOL_VERSION=1.5.4
N_LINES=5000	#number of lines to keep in $DFO_MON_DIR/DFO_STATUS

INSTR_DIR="/home/qc/public_html/${DFO_INSTRUMENT}"      # on $DFO_WEB_SERVER
QCBIN_DIR="/home/qc/bin"        			# on $DFO_WEB_SERVER

# ==============================================================================
# 0. preparation and initialization
# 0.1 check for directory variables which should be set globally in .dfosrc
# ==============================================================================

CHECK=`printenv | grep DFO`

if [ "Q$CHECK" = "Q" ]
then
	echo "*** ERROR: DFO variables not defined. Check ~/.dfosrc and restart."
        exit 1
fi

if [ "Q$DFO_WEB_SERVER" = "Q" ]
then
        echo "*** WARNING: \$DFO_WEB_SERVER not defined, update your .dfosrc."
fi

# =========================================================================
# 0.2 security
#     check params
# =========================================================================

if [ $# = 0 ]
then
        cat $DFO_DOC_DIR/finishNight.h | more
        exit 0
fi

CLEANUP_ONLY=NO
while getopts d:chv OPTION
do
	case "$OPTION" in
	 v ) echo $TOOL_VERSION
	     exit 0 ;;
	 h ) cat $DFO_DOC_DIR/finishNight.h | more
	     exit 0 ;;
	 d ) DATE=$OPTARG ;;
	 c ) CLEANUP_ONLY=YES ;;
	 ? ) cat $DFO_DOC_DIR/finishNight.h | more
	     exit 0 ;;
	esac
done

CHECK=`echo $DATE | wc -c`
if [ $CHECK != 11 ]
then
        echo "***ERROR: wrong date format: $DATE; should be: 2004-04-04; start again."
        exit -1
fi

# ==============================================================================
# 1. Check for header complenetess, open ingestion job; get statistics
# 1.1 Header completeness
# ==============================================================================

if [ $THIS_IS_PHOENIX = NO ]
then
# isql query for files? no, hdr could be moved (incomplete last tpl); just call hdr download again, slow but safest
	rm -f $TMP_DIR/list_all_hdr_orig $TMP_DIR/list_all_hdr_now
	CHECK_HDR_ORIG=`ls $DFO_HDR_DIR/$DATE/*hdr 2>/dev/null | wc -l`
	ls $DFO_HDR_DIR/$DATE/*hdr 2>/dev/null > $TMP_DIR/list_all_hdr_orig	
# mv existing dir to .save, like dataLoader
	if [ -s $DFO_HDR_DIR/$DATE ]
	then
		rm -rf $DFO_HDR_DIR/${DATE}.save
		mv $DFO_HDR_DIR/$DATE $DFO_HDR_DIR/${DATE}.save
	fi
	echo "- Calling HdrDownloader -d $DATE as completeness check ..."
	HdrDownloader -d $DATE | sed "s/^.*/	&/"
	CHECK_HDR_NOW=`ls $DFO_HDR_DIR/$DATE/*hdr 2>/dev/null | wc -l`
	ls $DFO_HDR_DIR/$DATE/*hdr 2>/dev/null > $TMP_DIR/list_all_hdr_now
	if [ $CHECK_HDR_ORIG = $CHECK_HDR_NOW ]
	then
		echo "  ... headers complete, continue with finishNight."
		rm -rf $DFO_HDR_DIR/${DATE}.save
	else
		diff $TMP_DIR/list_all_hdr_orig $TMP_DIR/list_all_hdr_now
		echo "  ... discrepancy found, there are new headers which have not previously been downloaded.
  New downloads are in \$DFO_HDR_DIR/${DATE}, previous ones in \$DFO_HDR_DIR/${DATE}.save.
  Exit and check. Hit return:"
		read input
		exit 
	fi
fi	

# ==============================================================================
# 1.2 Check for open ingestion job
# ==============================================================================

if [ -s $DFO_JOB_DIR/JOBS_INGEST ]
then
	CHECK_JOB=`egrep "^ingestProducts.*$DATE" $DFO_JOB_DIR/JOBS_INGEST`
	if [ "Q$CHECK_JOB" != "Q" ]
	then
		echo "***WARNING: Entry found in $DFO_JOB_DIR/JOBS_INGEST; exit for checking (E) or continue (C)? (E)"
		read CONTI_YN 
		if [ "Q$CONTI_YN" != "QC" ]
		then
			echo "Exit."
			exit
		fi
	fi
fi

# ==============================================================================
# 1.3 get statistics
# ==============================================================================

if [ $CLEANUP_ONLY = NO ]
then
	echo "***INFO: calling extractStat -d $DATE ..."
	extractStat -d $DATE
else
	echo "***INFO: option -c, clean-up only, no update of statistics done. Ok (Y/N) (N)?"
	read OK_YN
	if [ "Q$OK_YN" != "QY" ]
	then
		echo "Think about it."
		exit
	fi
fi

# ==============================================================================
# 2. Clean up
# 2.1 delete dfs_product directories for DATE
# ==============================================================================

if [ -d $DFS_PRODUCT ]
then
	rm -rf $DFS_PRODUCT/[A-Z]*/$DATE
fi

# ==============================================================================
# 2.2 Delete all $DATE related files in: $DFO_JOB_DIR/$DFO_MON_DIR/$DFO_GUI_DIR
# ==============================================================================

cd $DFO_JOB_DIR
rm -f *${DATE}*

cd $DFO_MON_DIR
rm -f `ls | grep ${DATE} | grep -v "DFO_"`

cd $DFO_GUI_DIR
rm -f *${DATE}*

cd $TMP_DIR
rm -f *${DATE}*
rm -f script*.sh

cd /tmp
rm -f *${DATE}*
rm -f *esh		# these are created by the browser 

# qc1update lists  are just logs and are not needed after finishing
cd $DFO_LST_DIR
rm -f list_qc1update*${DATE}*

# ==============================================================================
# 2.3 Delete all left-over ABs and AB logs in $DFO_AB_DIR, unless DEEP_MODE
# ==============================================================================

if [ Q$DEEP_MODE != QYES ]
then
	DATEplus1=`qcdate $DATE 1 2>/dev/null`
	if [ Q$DATEplus1 = Q ] || [ Q$DATEplus1 = QERROR ]
	then
		echo "***ERROR: something went wrong with the calculation of DATEplus1 from $DATE. Exit."
		exit
	fi

# calculate turnover between two DATEs
	TURNOVER=`echo 12 $DFO_OFFSET | awk '{print $1+$2}'`

	HOUR=$TURNOVER

	while [ $HOUR -lt 24 ]
	do
		rm -f ${DFO_AB_DIR}/${DFO_FILE_NAME}.*${DATE}T${HOUR}*
		rm -f ${DFS_LOG}/${DFO_FILE_NAME}.*${DATE}T${HOUR}*
		HOUR=`echo $HOUR | awk '{print $1+1}'`
	done

	HOUR=00

	while [ $HOUR -lt $TURNOVER ]
	do
		rm -f ${DFO_AB_DIR}/${DFO_FILE_NAME}.*${DATEplus1}T${HOUR}*
		rm -f ${DFS_LOG}/${DFO_FILE_NAME}.*${DATEplus1}T${HOUR}*
		HOUR=`echo $HOUR | awk '{print $1+1}'`
		if [ $HOUR -lt 10 ]
		then
			HOUR=`echo $HOUR | awk '{print "0"$1}'`
		fi
	done
fi

# ==============================================================================
# 2.4 delete all CONDOR dags for $DATE
# ==============================================================================

if [ Q$THIS_IS_PHOENIX != QYES ]
then
	THIS_IS_PHOENIX=NO
fi

if [ $THIS_IS_PHOENIX = YES ]
then
	QCJOB_CONCENTRATION=`grep "^QCJOB_CONCENTRATION" $DFO_CONFIG_DIR/config.phoenix | awk '{print $2}'`
	if [ Q$QCJOB_CONCENTRATION != QYES ]
	then
		QCJOB_CONCENTRATION=NO
	fi
else
	QCJOB_CONCENTRATION=NO
fi

if [ "Q$BQSROOT" != "Q" ]
then
	CONDOR_DAG_DIR=$BQSROOT/condor
	if [ -d $CONDOR_DAG_DIR ]
	then
# the * accounts for possible complex processing scenarios (step2 etc.)
		rm -rf $CONDOR_DAG_DIR/CALIB*_${DATE}*   $CONDOR_DAG_DIR/QCCALIB*_${DATE}*
		rm -rf $CONDOR_DAG_DIR/SCIENCE*_${DATE}* $CONDOR_DAG_DIR/QCSCIENCE*_${DATE}*

# also check for QCSCIENCE directories from QCJOB_CONCENTRATION and monthly jobs in $DFO_JOB_DIR
		if [ $QCJOB_CONCENTRATION = YES ]
		then
			DATE_SHORT=`echo $DATE | cut -c1-7`
			rm -rf $CONDOR_DAG_DIR/QCSCIENCE_${DATE_SHORT}_all*
			rm -f  $DFO_JOB_DIR/conc_JOBS_PHOENIX_${DATE_SHORT} $DFO_JOB_DIR/execQC_SCIENCE_${DATE_SHORT}
		fi	
	fi
fi

# ==============================================================================
# 2.5 manage other files: 
#     - delete multiple fitsreport logs
#     - keep DFO_STATUS to the last N_LINES entries
# ==============================================================================

rm -f $DFO_MON_DIR/fitsreport.log.*

cp $DFO_MON_DIR/DFO_STATUS $DFO_MON_DIR/DFO_STATUS.save
N_ENTRIES=`wc $DFO_MON_DIR/DFO_STATUS | awk '{print $1}'`
if [ $N_ENTRIES -gt $N_LINES ]
then
	N_DELETE=`echo $N_ENTRIES $N_LINES | awk '{print $1-$2}'`
	sed -i -e "1,$N_DELETE d" $DFO_MON_DIR/DFO_STATUS
	CHECK_ERROR=`wc $DFO_MON_DIR/DFO_STATUS | awk '{print $1}'`
	if [ $CHECK_ERROR = 0 ]
	then
		echo  "***ERROR: \$DFO_MON_DIR/DFO_STATUS seems to have been deleted. We activate $DFO_MON_DIR/DFO_STATUS.save. "
		cp $DFO_MON_DIR/DFO_STATUS.save $DFO_MON_DIR/DFO_STATUS
	fi
fi

echo "
  Cleaned: jobs and lists under \$DFO_JOB_DIR 
           orphan ABs under \$DFO_AB_DIR, alogs, tlogs, rblogs under \$DFS_LOG
           residual directories under \$DFS_PRODUCT
           lists under \$DFO_MON_DIR"

# ==============================================================================
# 2.6 Delete raw files
# ==============================================================================

if [ -s $DFO_CONFIG_DIR/OCA/config.filterRaw ]
then
	HIDE_DIR=`grep "^HIDE_DIR" $DFO_CONFIG_DIR/OCA/config.filterRaw | awk '{print $2}'`
fi

if [ -d $DFO_RAW_DIR ]
then
	cd $DFO_RAW_DIR
	rm -rf $DATE
	echo "  Deleted: $DFO_RAW_DIR/$DATE and all data therein. Hit return:"
	read input
else
	echo "***ERROR: No \$DFO_RAW_DIR found!"
fi

if [ "Q$HIDE_DIR" != "Q" ] && [ -d $HIDE_DIR ]
then
	cd $HIDE_DIR
	if [ -d $DATE ]
	then
		rm -rf $DATE
		echo "  Deleted: $HIDE_DIR/$DATE and all data therein. Hit return:"
		read input
	fi
fi

# ==============================================================================
# 3. END
# 3.1 set DFO flag
# ==============================================================================

UPDATE=`date +%Y-%m-%d"T"%H:%M:%S`
DFO_STATUS="finished"
echo "$DFO_STATUS $DATE $UPDATE" >> $DFO_MON_DIR/DFO_STATUS

# ==============================================================================
# 3.2 update data report and history
# ==============================================================================

if [ $CLEANUP_ONLY = NO ]
then
	echo "  Update data report ..."
	echo "***INFO: calling createReport -d $DATE ..."

	createReport -d $DATE
else
	echo "Do you want to update the data report (Y/N) (N)?"
	read UPDATE_YN
	if [ "Q$UPDATE_YN" = "QY" ]
	then
		createReport -d $DATE
	fi
fi

# histoMonitor call: suppressed for PHOENIX installations
if [ $THIS_IS_PHOENIX = NO ]
then
	YYMM=`echo $DATE | cut -c1-7`
	echo ""
	echo "  Update nautilus (histoMonitor) ..."
	echo "***INFO: calling histoMonitor -m $YYMM ..."

	histoMonitor -m $YYMM <<EOT
	
EOT
fi

# cleanup status files in monitor directory on $DFO_WEB_SERVER
if [ "Q$DFO_WEB_SERVER" != "Q" ]
then
	ssh -o BatchMode=yes $DFO_WEB_SERVER "$QCBIN_DIR/webFinishNight ${DFO_INSTRUMENT} $DATE " 1>/dev/null
fi

# ==============================================================================
# 3.3 update dfoMonitor for option -c
# ==============================================================================
# 
if [ $CLEANUP_ONLY = YES ]
then
	echo "Now we update dfoMonitor ..."
	dfoMonitor
fi

exit
