#!/bin/sh
# PURPOSE:	provide a wrapper for the DFS tool ngamsCClient
#		ngamsCClient offers direct download of a file into the QC system.
#		Header and fits files come with 'hotfly' update.
# AUTHOR:	Reinhard Hanuschik / ESO
# VERSIONS:	1.0 -- MAY 2006
#		1.1 -- enabled to download tarballs (2006-06-12)
#		1.1.1- has choice of NGAS servers (2006-07-20)
#		1.2 -- -H: download updated headers (2006-11-27)
#		1.2.1- removed ngau1 from server list (2007-03-09)
#		1.3 -- -f: uses hotfly to update headers; -H: hotfly; archived hdr: now -W (2008-03-04)
#		1.3.1- updated feedback text (line 124) (2008-04-15)
#		1.3.2- modified syntax for hotfly (2008-06-13)
#		1.3.3- suppress current error messages of hotfly (2008-07-16)
#		1.3.4- returns EXIT_CODE properly (2008-08-14)
#		1.3.5- help updated to reflect ancillary file downloads (2008-09-18)
#		1.3.6- tool queries NGAS for availability before launching request (2008-09-25)
#		1.4 -- -P: download updated product header (2008-10-22) [IPe]
#		1.4.1- supports tile (un)compression (for VCAM files) (2008-12-02) [WHu]
#		1.4.2- enforces uncompression (for non-VCAM files, :230) (2009-03-13)
#		1.4.3- bug fixed with nc_out_calib_$$ and nc_query_$$ (2009-10-14)
#		1.4.4- another bug fixed with nc_out_ngas_$$ (2009-10-23)
#		1.4.5- another bug fixed with nc files (2010-01-10)
#		1.4.6- introducing timeout parameter for ngasmCClient (2010-10-28)
#		1.4.7- no need to mv .fz file (line 253) (2012-08-24)
#		1.5  - check for hotfly being necessary; FTP_HOSTID terminated (2012-10-25)
#		1.5.1- bug fix to remove properly nc_out_update_$$ and nc_update_$$ (2012-11-05)
#		1.5.2- yet another bug fix related to nc_out_update and nc_update (2012-11-27)
#		1.5.3- check for proper hotfly installation dropped (2013-01-17)
#		1.6 -- check  if file has already been downloaded (for -f and -c); if yes, don't download, unless -F is set (2013-02-11)
#		1.6.1- jewel2 replaced by jewel61 (2013-10-24)
#		1.7 -- dynamic retrieval of $NGAMS_SERVERS; new option -U to fill ngasClient.cfg (2013-11-06)
#		1.7.1- replace 'switch' by 'change' in awk commands (for OS upgrade) (2014-02-18)
#		1.7.2- fix a bug with this fix (2014-03-10)
#		1.7.3- dpQuery replacing cdbQuery; check for hotfly removed (comes with dfs) (2014-09-04)
#		1.8 -- retrieval of IDPs (2015-09-21)
#		1.8.1- deletes hdr if fits downloaded (RAW, MCAL; relevant for OPSHUB) (2018-08-02)
#		1.8.2- expand jewel to jewel.hq.eso.org for OPSHUB (2018-10-22)
# 		1.8.3- option -P (product header PHDR): skip download if file existing (2018-11-14)
#		1.8.4- 'case' replaced by 'switch' in awk command (2019-04-16) 
#		1.8.5- bug fix in checkUpdate function (didn't work properly before) (2020-03-03)
#		1.8.6- change of database server from SAFIQR1 to SAFIQ in checkUpdate (2023-02-06)
#		1.9  - handle both the DFOS-v1 and the qcFlow naming schemes for MCAL and PHDR download (2023-06-05)
#		1.10 - de-commissioning of ngamsCClient (2024-02-26)
#		1.10.1- enable hotfly update for master calibrations after NGAS upgrade (2024-03-18)
#		1.10.2- add aliases for curl and openssl (2025-06-30)
#
# OUTPUT:	file is downloaded into directory from where tool is called.
#		tool output goes to stdout
#
# CONFIG:	none
# TOOLS CALLED:	hotfly; funpack (tile uncompression tool); dpQuery
# =========================================================================
TOOL_VERSION=1.10.2

TAR_NGAMS_HOST=jewel81

CONNECT_TIMEOUT=20 #timeout for server connection in seconds

ARC_URL="http://archive.eso.org"

DPTRACK_PWD=qc_08ps	# dbcm database
DPTRACK_SRV=SAFIQ	# dbcm database

# v1.10.2. Fix needed after change of security certificate
alias curl='/usr/bin/curl'
alias openssl='/opt/sybase-16.0/OCS-16_0/bin/openssl'

cat > $TMP_DIR/ngas_help <<EOT
USAGE: 
ngasClient -f GIRAF.2005-09-21T15:28:43.421.fits         retrieve a raw file with updated header (only if not yet downloaded)
ngasClient -H MIDI.2008-02-22T14:55:34.810.hdr|.fits   	 retrieve a raw header, updated
							 (you can use .hdr or .fits extension)

ngasClient -c GI_PAES_060501A_Medusa2_H525.8nm_o11.fits  retrieve a product file, science or calib (only if not yet downloaded)
ngasClient -i GI_SIDP_200139477_2004-06-26T10:59:44.686_F86_Med1_L543.1_o3.fits
							 retrieve an IDP file (any type: fits, gif, txt etc.)

ngasClient -P GI_PAES_060501A_Medusa2_H525.8nm_o11.fits  retrieve a product header
							 (NOTE: specify the cdbname, even though you want a header only)

'Exotic' options:
ngasClient -t FEROS.2005-11-07T01:01:35.890.tar		 retrieve a product tar ball
ngasClient -W FEROS.2006-11-12T08:34:25.285.hdr      	 retrieve a header from archive, hotfly'ed in many cases, 
							  no extensions, slow!
ngasClient -v, -h: as usual
ngasClient -f <file> -F / -c <file> -F			 Force: always download, even if file exists already 
EOT

# ==========================================================================
# 0. preparation
# 0.1 check for DFO variables
# ==========================================================================

CHECK=`printenv | grep DFO`

if [ "Q$CHECK" = "Q" ]
then
	echo "*** ERROR: define DFO variables in ~/.dfosrc and re-start."
	exit -1
fi

if [ "Q$HOTFLY_DBALIAS" = "Q" ]
then
	echo "***ERROR: include 'export HOTFLY_DBALIAS=HOTFLY_OP' in your .qcrc file, then source and try again."
	exit
fi

# check if this is an OPSHUB installation (environment variable THIS_IS_OPSHUB=YES is set by .dfosrc)
if [ Q$THIS_IS_OPSHUB != QYES ]
then
	THIS_IS_OPSHUB=NO
fi

# ==========================================================================
# 0.2 check for ngamsCClient
# ==========================================================================

# OBSOLETE

# ==========================================================================
# 0.3 read params
# ==========================================================================

if [ $# = 0 ]
then
        cat $TMP_DIR/ngas_help
        exit 0
fi

FORCE=NO
UPDATE=NO

GETOPT="f:c:i:P:t:H:W:Fvh"

while getopts $GETOPT OPTION
do
	case "$OPTION" in
	 v ) echo $TOOL_VERSION ; exit 0 ;;
	 h ) cat $TMP_DIR/ngas_help ; exit 0 ;;
	 f ) DMODE=RAW 
	     FILE=$OPTARG
	     FILEID=`echo $FILE | sed "s/\.fits//" | sed "s/\.tfits//"` ;;
	 c ) DMODE=MCAL
	     FILE=$OPTARG ;;
	 i ) DMODE=IDP
	     FILE=$OPTARG ;;
	 t ) DMODE=TAR
	     FILE=$OPTARG ;;
	 H ) DMODE=HOTFLY	# raw header (standard method)
	     FILE=$OPTARG ;;
	 W ) DMODE=HDR		# raw header (depricated)
	     FILE=$OPTARG ;;
	 P ) DMODE=PHDR		# product header
	     FILE=$OPTARG ;;
	 F ) FORCE=YES ;;
	 * ) cat $TMP_DIR/ngas_help ; exit 0 ;;
	esac
done

# =========================================================================
# 0.4 procedure checkAvail 
#     check availability in NGAS for single raw file (adapted from createReport)
# =========================================================================
checkAvail(){
echo "- check availability ..."
cat > $TMP_DIR/nc_ngas_$$ <<EOT
SELECT DISTINCT
        convert (char(40), file_id )
FROM
        ngas..ngas_files f ,
        ngas..ngas_disks d ,
        ngas..ngas_hosts h
WHERE
        file_id = "$FILEID"
AND
        h.domain = "hq.eso.org"
AND
        d.disk_id = f.disk_id
AND
        h.host_id = d.host_id
GO
EOT
rm -f $TMP_DIR/nc_out_ngas_$$
isql -Uqc -P`eval "cat ${ARCH_PWD}"` -S${ARCH_SERVER} -w 999 -i $TMP_DIR/nc_ngas_$$ -o $TMP_DIR/nc_out_ngas_$$
}

# =========================================================================
# 0.5 query in dp_tracking (useful to decide if hotfly is needed)
# If this query returns nothing, there is no need to call hotfly (which would 
# cost some time)
# =========================================================================

# OBSOLETE

# =========================================================================
# 0.6 query for product
# =========================================================================
getDP(){
cat > $TMP_DIR/nc_query_$$ <<EOT
SELECT
	dp_id
FROM
	qc_metadata..qc_products
WHERE
	origfile = "$FILE"
GO
EOT
rm -f $TMP_DIR/nc_out_calib_$$ 
isql -Uqc -P`eval "cat ${ARCH_PWD}"` -S${ARCH_SERVER} -w 999 -i $TMP_DIR/nc_query_$$ -o $TMP_DIR/nc_out_calib_$$
}

# =========================================================================
# 0.7 query for ngas servers (for download!)
# =========================================================================

# OBSOLETE

# =========================================================================
# 0.8 create dynamic cfg file
# =========================================================================

# OBSOLETE

# =========================================================================
# 0.9 UPDATE=YES
# =========================================================================

# =========================================================================
# 1. Download 
# ==========================================================================

case $DMODE in
 "RAW"  ) case $FORCE in
	   "YES" ) rm -f $FILE ;;
	   "NO"  ) if [ -s $FILE ]
		   then
			echo "- file exists already. Force download with option -F. Exit."
			exit
		   fi ;;
	  esac

	  checkAvail $FILEID
	  if [ -s $TMP_DIR/nc_out_ngas_$$  ]
	  then
		CHECK_AVAIL=`grep $FILEID $TMP_DIR/nc_out_ngas_$$ | awk '{print $1}'`
		if [ Q$CHECK_AVAIL != Q ]
		then
			echo "- download file ..."
			curl -nOJs --connect-timeout $CONNECT_TIMEOUT "https://dataportal.eso.org/dataPortal/file/${FILEID}"
			rm -f $TMP_DIR/nc_out_ngas_$$ $TMP_DIR/nc_ngas_$$
# avoid having .hdr and .fits in the system, could confuse ABbuilder

			if ( [ -s $FILE ] || [ -s ${FILE}.Z ] || [ -s ${FILE}.fz ] ) && [ -s ${FILEID}.hdr ]
			then
				rm -f ${FILEID}.hdr
			fi
		else
			echo "***ERROR: $FILE currently not available in NGAS. Exit."
			rm -f $TMP_DIR/nc_out_ngas_$$ $TMP_DIR/nc_ngas_$$
			exit -1
		fi
	  else
		echo "***ERROR: $FILE currently not available in NGAS. Exit."
		rm -f $TMP_DIR/nc_out_ngas_$$ $TMP_DIR/nc_ngas_$$
		exit -1
	  fi ;;
 "MCAL" ) case $FORCE in
	   "YES" ) rm -f $FILE ;;
	   "NO"  ) if [ -s $FILE ]
		   then
			echo "- file exists already. Force download with option -F. Exit."
			exit
		   fi ;;
	  esac
	  # v1.9: handle both DFOS-v1 and qcFlow naming scheme
	  CHECK_NAME=`echo $FILE | grep "^M\."`
	  if [ "Q$CHECK_NAME" = "Q" ]
	  then
		#dpQuery -f $FILE -o ./ 
		# new with v1.10.1:
		#dpQuery -f $FILE | awk '!/---|origfile/{system("curl -n -s -o "$1" https://dataportal.eso.org/dataPortal/file/"$5"")}'
		# new with v1.10.2:
		dpQuery -f $FILE | awk '!/---|origfile/{system("/usr/bin/curl -n -s -o "$1" https://dataportal.eso.org/dataPortal/file/"$5"")}'
	  else
		FILEID=`echo $FILE | sed "s/.fits//"`
		curl -nOJs --connect-timeout $CONNECT_TIMEOUT "https://dataportal.eso.org/dataPortal/file/${FILEID}"
	  fi
	  #
	  if [ -s ${FILE}.Z ]
	  then
		  gunzip ${FILE}.Z
	  fi
	  FILEHDR=`echo $FILE | sed "s/tfits/hdr/" | sed "s/fits/hdr/"`
	  if ( [ -s $FILE ] || [ -s ${FILE}.Z ] ) && [ -s $FILEHDR ]
	  then
		rm $FILEHDR
	  fi ;;

 "IDP" )  case $FORCE in
	   "YES" ) rm -f $FILE ;;
	   "NO"  ) if [ -s $FILE ]
		   then
			echo "- file exists already. Force download with option -F. Exit."
			exit
		   fi ;;
	  esac
	  #dpQuery --idp -f $FILE -o ./ ;;
	  # changed with v1.10.1:
	  #dpQuery --idp -f $FILE | awk '!/---|origfile/{system("curl -n -s -o "$1" https://dataportal.eso.org/dataPortal/file/"$2"")}' ;;
	  # changed with v1.10.2:
	  dpQuery --idp -f $FILE | awk '!/---|origfile/{system("/usr/bin/curl -n -s -o "$1" https://dataportal.eso.org/dataPortal/file/"$2"")}' ;;
 "TAR"  ) rm -f $FILE
	  curl -nOJs --connect-timeout $CONNECT_TIMEOUT "https://dataportal.eso.org/dataPortal/file/${FILE}" ;;
 "HOTFLY" ) rm -f $FILE
	  FILE=`echo $FILE | sed "s/.fits/.hdr/"`
	  hotfly -h $FILE -s $HOTFLY_DBALIAS ;;
 "HDR"  ) rm -f $FILE
	  echo "NOTE: This header source is available for single-extension files only, and only after a couple of days after acquisition."
	  echo "Downloading ..."
	  HDR=`echo $FILE | sed "s/.hdr//" | sed "s/.fits//"`
	  rm -f $HDR
	  eval "wget -q ${ARC_URL}/hdr?DpId=$HDR" ;;
 "PHDR" ) HDR_FILE=`echo ${FILE} | sed "s/tfits/hdr/" | sed "s/fits/hdr/"`
	  if [ -s $HDR_FILE ]
	  then
		echo "*** INFO: header file $HDR_FILE already existing, skipped."
		exit
	  fi

	  # v1.9: handle both DFOS-v1 and qcFlow naming scheme
	  CHECK_NAME=`echo $FILE | grep "^M\."`
	  if [ "Q$CHECK_NAME" = "Q" ]
	  then
	  	getDP $FILE
	  	CHECK_EXIST=`grep [A-Z] $TMP_DIR/nc_out_calib_$$ | wc | awk '{print $1}'`
	  	if [ $CHECK_EXIST != 0 ]
	  	then
			DP_ID=`grep [A-Z] $TMP_DIR/nc_out_calib_$$ | awk '{print $1}' | sed "s/ //g" | sort -u`
			HDR_FILE=`echo ${DP_ID}.hdr`
			hotfly -h $HDR_FILE -s $HOTFLY_DBALIAS
	  		FILE=`echo $FILE | sed "s/.tfits/.hdr/" | sed "s/.fits/.hdr/"`
			mv $HDR_FILE $FILE
			rm -f $TMP_DIR/nc_out_ngas_$$ $TMP_DIR/nc_ngas_$$
	  	fi
	  else
		hotfly -h $HDR_FILE -s $HOTFLY_DBALIAS
		FILE=$HDR_FILE
	  fi
esac

rm -f $TMP_DIR/nc_out_calib_$$ $TMP_DIR/nc_query_$$ $TMP_DIR/nc_out_update_$$

# =========================================================================
# 2. uncompress and update header
# ==========================================================================

if [ $DMODE = TAR ] || [ $DMODE = RAW ]
then
# uncompress .Z raw files
	if [ -s ${FILE}.Z ]
	then
		echo "- uncompress file ..."
		uncompress -f ${FILE}.Z
	fi	

# uncompress .fz raw files: requires funpack
	if [ -s ${FILE}.fz ]
	then
		echo "- funpack file ..."
		CHECK_FUNPACK=`which funpack 2>/dev/null | wc -w`
		if [ "$CHECK_FUNPACK" = 0 ]
		then
			echo "***ERROR: funpack not found. Can't decompress."
		else
			funpack -F ${FILE}.fz
		fi
	fi	
fi	

# =========================================================================
# 3. HDR: remove HTML tags
# ==========================================================================

if [ $DMODE = "HDR" ]
then
	if [ -s "hdr?DpId=$HDR" ]
	then
        	cat "hdr?DpId=$HDR" | awk '{
  if ( change == 1 ) { print }
  if ( $1 == "<pre>" ) { change=1 }
  if ( $1 == "END" )   { change=2 }
  }' change=0 > $HDR.hdr
		rm "hdr?DpId=$HDR"
    	    	echo "... done."
	else
       		echo "... unsuccessful, or not found."
	fi
fi

# =========================================================================
# 4. end
# ==========================================================================

if [ -s ${FILE} ]
then
	echo "$FILE downloaded into $PWD."
elif [ -s ${FILE}.fz ]
then
	echo "$FILE.fz downloaded into $PWD but not unpacked, tool funpack required."
	rm -f $TMP_DIR/nc_out_calib_$$ $TMP_DIR/nc_query_$$ $TMP_DIR/nc_out_ngas_$$ $TMP_DIR/nc_ngas_$$
	exit -1
else
	echo "***ERROR: $FILE download unsuccessful."
	rm -f $TMP_DIR/nc_out_calib_$$ $TMP_DIR/nc_query_$$ $TMP_DIR/nc_out_ngas_$$ $TMP_DIR/nc_ngas_$$
	exit -1
fi

rm -f $TMP_DIR/nc_out_calib_$$ $TMP_DIR/nc_query_$$ $TMP_DIR/nc_out_ngas_$$ $TMP_DIR/nc_ngas_$$
exit 0

