#!/bin/sh
# PURPOSE:     	download of updated headers or fits files, per DATE
# AUTHOR:       Reinhard Hanuschik / ESO
# VERSIONS:     1.0 -- April 2008
#		1.0.1- bug fix with dataclient (2010-01-28) 
#		1.1 -- download of product headers or fits files; DFO_STATUS stabilized (2010-09-21)
#		1.1.1- minor bug fixed (see :202) (2011-01-28) [DDo]
#		1.2 -- query in line 208 adapted to obs_metada tables (2011-06-17)
#		1.2.1- replace observations by obs_metadata database (2012-08-24)
#		1.2.2- bug fix for product headers and fits files (2012-12-09)
#		1.3 -- dataclient replaced by HdrDownloader (2014-04-28)
#		1.3.1- supports optional config file HdrDownloader.cfg (2014-05-21)
#		1.3.2- dropped: config file (2015-01-13)
#		1.4 -- query to NGAS database changed due to change of format of ingestion_date column (2024-03-05)
#		1.4.1- do not require user confirmation if header directory is already existing (2025-02-03)
#
# PARAMETERS:   -t h [raw headers] or r [raw fits files] | P [product headers] or c [product fits files] 
#		-d <date>
# OUTPUT:     	raw:  updated headers in $DFO_HDR_DIR/<date> or fits files in $DFO_RAW_DIR | 
#		prod: updated headers or fits files in $DFO_CAL_DIR|$DFO_SCI_DIR/<date>
# CONFIG:       none
# OTHER TOOLS:	ngasClient plus hotfly; HdrDownloader for headers
# =========================================================================
TOOL_VERSION=1.4.1

# taken from ngasMonitor
FTP_HOSTID="ngau1"      # current host_id of host receiving ftp data

# ==========================================================================
# 0. Initialize
# 0.1 Define help
# ==========================================================================

cat > $TMP_DIR/dataLoader_help <<EOT
USAGE:
	dataLoader -v | -h | -d <date> [-t h|r|c|P]

	load hdr or fits files (raw or products) per <date>; data go through hotfly and have updated headers

	date: 		DFO date for which headers/fits files are downloaded
	-t h:		headers (raw)	   (go to \$DFO_HDR_DIR/<date>)
	-t r:		fits files (raw)   (go to \$DFO_RAW_DIR/<date>)

	-t P:		headers (products) (go to \$DFO_CAL_DIR/<date> and \$DFO_SCI_DIR/<date>)
	-t c:		fits files (prod.) (go to \$DFO_CAL_DIR/<date> and \$DFO_SCI_DIR/<date>)
EOT

# ==========================================================================
# 0.2 get options
# ==========================================================================

while getopts d:t:hv OPTION
do
	case "$OPTION" in
	 v ) echo $TOOL_VERSION
             exit 0 ;;
         h ) cat $TMP_DIR/dataLoader_help
             exit 0 ;;
	 d ) DATE=$OPTARG ;;
	 t ) TYPE=$OPTARG ;;
	 ? ) cat $TMP_DIR/dataLoader_help
             exit 0 ;;
	esac
done

if [ $# = 0 ]
then
	cat $TMP_DIR/dataLoader_help; exit
fi

if [ "Q$DATE" = "Q" ]
then
	echo "***ERROR: You must enter a date. Exit."
	exit -1
fi

if [ Q$TYPE = Q ]
then
	echo "***ERROR: You must enter a file type (h | r | P | c)."
	exit
fi

if [ Q$TYPE != Qh ] && [ Q$TYPE != Qr ] && [ Q$TYPE != QP ] && [ Q$TYPE != Qc ]
then
	echo "***ERROR: No support for file type \"$TYPE\". Exit."
	exit -1
fi

# ==========================================================================
# 0.3 availability query (from ngasMonitor v3.1)
# ==========================================================================
check4avail(){
cat > $TMP_DIR/dl_insql1 <<EOT
SELECT 
	convert(char(40), file_id),
	substring(ingestion_date,1,4)+'.'+substring(ingestion_date,6,2)+'.'+substring(ingestion_date,9,2),
	substring(ingestion_date,12,8)
FROM
        ngas..ngas_files f ,
        ngas..ngas_disks d ,
        ngas..ngas_hosts h
WHERE
        file_id in (
EOT
cat $TMP_DIR/dl_file_list | sed "s/^.*/\"&\" ,/" | sed "s/.fits//" | sed "$,$ s/,//" >> $TMP_DIR/dl_insql1

cat >> $TMP_DIR/dl_insql1 <<EOT
)
AND
	h.domain = "hq.eso.org"
AND
        d.disk_id = f.disk_id
AND
        h.host_id = d.host_id
AND
        d.host_id != "${FTP_HOSTID}"
GO
EOT
rm -f $TMP_DIR/dl_outsql
isql -Uqc -P`eval "cat ${ARCH_PWD}"` -S${ARCH_SERVER} -w 999 -i $TMP_DIR/dl_insql1 -o $TMP_DIR/dl_outsql1
cat $TMP_DIR/dl_outsql1 |\
 grep -v affected |\
 sed "1,2 d" |\
 sed "/^$/d" |\
 sort -u |\
 awk '{print $1}' \
> $TMP_DIR/dl_outsql
}

# ==========================================================================
# 0.9 prologue
# ==========================================================================

if [ $TYPE = h ]
then
	echo "
We download all $DFO_FILE_NAME updated headers for date $DATE. They come sorted with broken template shifting."
elif [ $TYPE = r ]
then
	echo "
We download and update all raw fits files for date $DATE. They come sorted as in \$DFO_HDR_DIR/$DATE
(i.e. with broken template shifting)."
elif [ $TYPE = P ]
then
	echo "
We download all product headers for date $DATE. CAL headers are delivered in \$DFO_CAL_DIR/$DATE,
SCI headers in \$DFO_SCI_DIR/$DATE."
elif [ $TYPE = c ]
then
	echo "
We download all product files for date $DATE. 
CAL products are delivered to \$DFO_CAL_DIR/$DATE,
SCI products to \$DFO_SCI_DIR/$DATE."
fi

# ==========================================================================
# 1. Headers
# 1.1 Headers exist already: make a backup (in case HdrDownloader fails)
# ==========================================================================

if [ $TYPE = h ]
then
	if [ -s $DFO_HDR_DIR/$DATE ]
	then
		rm -rf $DFO_HDR_DIR/${DATE}.save
		mv $DFO_HDR_DIR/$DATE $DFO_HDR_DIR/${DATE}.save
		echo "*** INFO: old directory \$DFO_HDR_DIR/${DATE} found, saved as \$DFO_HDR_DIR/${DATE}.save."
		echo "          Make sure to delete afterwards."
		#echo "          Make sure to delete afterwards. Hit return:"
		#read input
	fi

# ==========================================================================
# 1.2 call HdrDownloader
# ==========================================================================

	echo "- Calling HdrDownloader -d \$DATE for header download from the repository ..."

# redirect status flag output to $TMP_DIR
	rm -f $TMP_DIR/list_hdr_$DATE.txt
	MON_DIR_ORIG=$DFO_MON_DIR
	export DFO_MON_DIR=$TMP_DIR

	HdrDownloader -d $DATE

	export DFO_MON_DIR=$MON_DIR_ORIG
	rm -f $TMP_DIR/list_hdr_$DATE.txt

	CHECK_EXIST=`ls $DFO_HDR_DIR/$DATE 2>/dev/null | grep hdr | head -1`
	UPDATE=`date +%Y-%m-%d"T"%H:%M:%S`

	if [ Q$CHECK_EXIST = Q ]
	then
		echo "- No headers found in the keyword repository."
		echo "hdr_Notexist $DATE $UPDATE dataLoader" >> $DFO_MON_DIR/DFO_STATUS
		exit
	else
		echo "hdr_Exist $DATE $UPDATE dataLoader" >> $DFO_MON_DIR/DFO_STATUS
	fi

	echo "  ... done."
	exit
fi

# ==========================================================================
# 2. Raw fits file download, calling ngasClient
# ==========================================================================

if [ $TYPE = r ]
then
	if [ ! -s $DFO_RAW_DIR/$DATE ]
	then
		mkdir $DFO_RAW_DIR/$DATE
		echo "... \$DFO_RAW_DIR/$DATE created." 
	fi

	if [ ! -s $DFO_MON_DIR/list_hdr_$DATE.txt ]
	then
		echo "- No \$DFO_MON_DIR/list_hdr_$DATE.txt found. We create that list from headers ..."
		echo "  (or run 'dataLoader -t h -d $DATE' first if you aren't sure about completeness)."

		if [ ! -d $DFO_HDR_DIR/$DATE ]
		then
			echo "  ... no \$DFO_HDR_DIR/$DATE found. Run 'dataLoader -t h -d $DATE' first and then re-try."
			exit
		fi
		cd $DFO_HDR_DIR/$DATE
		ls | grep hdr > $DFO_MON_DIR/list_hdr_$DATE.txt
		echo "  ... created from header files."
	fi

	cd $DFO_RAW_DIR/$DATE
	echo "- downloading hotfly'ed fits files with \$DFO_MON_DIR/list_hdr_$DATE.txt ..."

# check availability
	rm -f $TMP_DIR/dl_file_list
        cat $DFO_MON_DIR/list_hdr_$DATE.txt | awk '{print $1}' | sed "s/hdr/fits/" > $TMP_DIR/dl_file_list
	check4avail
	if [ ! -s $TMP_DIR/dl_outsql ]
	then
		echo "  no file yet available."
	else
        	for F in `cat $TMP_DIR/dl_file_list`
        	do
			F1=`echo $F | sed "s/.fits//"`
			CHECK_AVAIL=`grep $F1 $TMP_DIR/dl_outsql`
			if [ Q$CHECK_AVAIL = Q ]
			then
				echo "  $F not yet available."
			elif [ -s $F ]
			then
				echo "  $F already existing, no download."
			else
				echo "  $F downloading ..."
               		 	ngasClient -f $F | sed "s/^.*/   &/" | grep -v "downloaded into"
			fi
        	done
	fi

	echo "  ... done."
fi

# ==========================================================================
# 3. Product hdr/fits file download
# ==========================================================================

if [ $TYPE = c ] || [ $TYPE = P ]
then

# call productExplorer -d $DATE to have reference list
	echo "- We call productExplorer first..."
	productExplorer -d $DATE | egrep -v "For download|For deletion" | sed "/^$/d" | sed "s/^.*/   [productExplorer] &/"
	if [ $TYPE = c ]
	then
		echo "
- Now we start downloading the product fits files. First we download
  CALIB products. Then we continue with SCIENCE products. We do not 
  download ANCILLARY fits files (use the download file $TMP_DIR/mcalibDownload
  if you need them)."
		rm -f $TMP_DIR/dl_sciproc $TMP_DIR/dl_mcalib
		echo "  1. CALIB products ..."
		grep "ngasClient" $TMP_DIR/mcalibDownload | grep " M.${DFO_INSTRUMENT}" | awk '{print $1,$2,$3}' > $TMP_DIR/dl_mcalib
		if [ -s $TMP_DIR/dl_mcalib ]
		then
			chmod u+x $TMP_DIR/dl_mcalib
			if [ ! -d $DFO_CAL_DIR/$DATE ]
			then
				mkdir $DFO_CAL_DIR/$DATE
			fi
			cd $DFO_CAL_DIR/$DATE
			$TMP_DIR/dl_mcalib
		else
			echo "  ... no mcalib products found."
		fi

		echo "  2. SCIENCE products ..."
		grep "ngasClient" $TMP_DIR/mcalibDownload | grep " S.${DFO_INSTRUMENT}" | awk '{print $1,$2,$3}' > $TMP_DIR/dl_sciproc
		if [ -s $TMP_DIR/dl_sciproc ]
		then
			chmod u+x $TMP_DIR/dl_sciproc
			if [ ! -d $DFO_SCI_DIR/$DATE ]
			then
				mkdir $DFO_SCI_DIR/$DATE
			fi
			cd $DFO_SCI_DIR/$DATE
			$TMP_DIR/dl_sciproc
		else
			echo "  ... no science products found."
		fi
		
		exit
	fi

	if [ $TYPE = P ]
	then
		echo "
- Now we start downloading the product headers. First we download
  CALIB product headers. Then we continue with SCIENCE product headers. We do not 
  download ANCILLARY file headers."
		rm -f $TMP_DIR/dl_sciproc $TMP_DIR/dl_mcalib
		echo "  1. CALIB product headers (to $DFO_CAL_DIR/$DATE)"
		grep "ngasClient" $TMP_DIR/mcalibDownload | grep " M.${DFO_INSTRUMENT}" | awk '{print $1" -P "$3}' > $TMP_DIR/dl_mcalib

		if [ -s $TMP_DIR/dl_mcalib ]
		then
			chmod u+x $TMP_DIR/dl_mcalib
			if [ ! -d $DFO_CAL_DIR/$DATE ]
			then
				mkdir $DFO_CAL_DIR/$DATE
			fi
			cd $DFO_CAL_DIR/$DATE

# have better output than the standard output from hotfly
			for HDR  in `cat $TMP_DIR/dl_mcalib | awk '{print $3}'`
			do
				H1=`echo $HDR | sed "s/fits/hdr/"`				
				echo "    $H1"
				`eval "grep $HDR $TMP_DIR/dl_mcalib"` 1>/dev/null
			done
		else
			echo "  ... no mcalib products found."
		fi

		echo ""
		echo "  2. SCIENCE product headers (to $DFO_SCI_DIR/$DATE) ..."
		grep "ngasClient" $TMP_DIR/mcalibDownload | grep " S.${DFO_INSTRUMENT}" | awk '{print $1" -P "$3}' > $TMP_DIR/dl_sciproc
		if [ -s $TMP_DIR/dl_sciproc ]
		then
			chmod u+x $TMP_DIR/dl_sciproc
			if [ ! -d $DFO_SCI_DIR/$DATE ]
			then
				mkdir $DFO_SCI_DIR/$DATE
			fi
			cd $DFO_SCI_DIR/$DATE
# have better output than the standard output from hotfly
			for HDR  in `cat $TMP_DIR/dl_sciproc | awk '{print $3}'`
			do
				H1=`echo $HDR | sed "s/fits/hdr/"`				
				echo "    $H1"
				`eval "grep $HDR $TMP_DIR/dl_sciproc"` 1>/dev/null
			done
		else
			echo "  ... no science products found."
		fi
	fi
	echo "
  ... done."
fi

exit

