#!/bin/sh
# PURPOSE: 	getObInfo: get OB grades and comments from both old and new nightlog systems 
# AUTHOR: 	Reinhard Hanuschik and Isabelle Percheron / ESO 
# VERSIONS: 	1.0 -- exported from createReport (2012-09-21)
#		1.1 -- date loop added for calChecker (2012-09-25)
#		1.2 -- other instruments migrated to NLT; modified date format; bug fix with mode -C (2012-10-02)
#		1.2.1- comment-->public_comment, plus internal_comment added (2012-10-11)
#		1.2.2- bug fix: PANL_INSTRUMENT replaced by INSTRUMENT2 (2013-01-11)
#		1.2.3- KMOS added (2013-03-13) 
#		1.2.4- minor change for calChecker v4.2 (2013-11-15)
#		1.2.5- MUSE and SPHERE added (2014-01-30)
#		1.2.6- bug fixes, and check for gsspInterface (2014-10-22)
#		1.3 -- 400 chars for COMMENT, new combined query for grade+comment; PIONIER included (2015-08-31)
#		1.3.1- GRAVITY added (2015-11-15)
#		2.0 -- new queries for old nightlog scheme, since PANL2 database disappeared (2017-07-21)
#	
# PARAMETERS:   -d <date>; -i <instrument>
# OUTPUT:	
# CONFIG FILE: 	NONE
# TOOLS CALLED:	NONE
# COMMENTS:	NLT configuration: grep for NEW NIGHTLOG TOOL CONFIGURATION
# 		That configuration is the master, also for other tools like histoMonitor.
# NOTE:		The UT_start apparently is the truncated key OBS.START.
# ==========================================================================
# currently we have all new instruments, for backward compatibility; can be removed in a future version 2.0+
TOOL_NAME="getObInfo"
TOOL_VERSION="2.0"
LENGTH=400

# NIGHTLOG TOOL CONFIGURATION (lists only instruments which have a mixed history; no need to add new instruments after 2012-10-02)
# This configuration is the master, also for other tools like histoMonitor!

#NLT	FLAMES		20120401
#NLT	UVES		20120401
#NLT	XSHOOTER	20120401
#NLT	ISAAC		20120929
#NLT	VIMOS		20120929
#NLT	VISIR		20120929

#NLT	FORS1		20090401
#NLT	CRIRES		20121002
#NLT	FORS2		20121002
#NLT	HAWKI		20121002
#NLT	NACO		20121002
#NLT	SINFONI		20121002
#NLT	AMBER		20121002
#NLT	MIDI		20121002

# can be removed once the other tools (calChecker, createReport etc.) have been fixed:
#NLT	KMOS		20121002
#NLT	PIONIER		20121002
#NLT    SPHERE          20121002
#NLT    GRAVITY         20121002
#NLT    VIRCAM          20050101
#NLT    OMEGACAM        20050101

# ==========================================================================
# 0. Initialization
# 0.1 check for DFO variables
# ==========================================================================

CHECK=`printenv | grep DFO`
if [ "Q$CHECK" = "Q" ]
then
        echo "*** ERROR: define DFO variables in ~/.dfosrc and re-start."
        exit
fi

TODAY=`date -u +%Y-%m-%d`
NLT_TODAY=`date -u +%d-%m-%Y`
TODAY1=`qcdate $TODAY +1 | sed "s/-/ /g" | awk '{print $3,$2,$1}' | sed "s/ /-/g"`
TODAY2=`date -u +%d`
NLT_LABEL="$TODAY2/$TODAY1"

cat > $TMP_DIR/getObInfo.txt <<EOT
USAGE:	getObInfo [ -d <date> | -C] [-i <instrument>] [-V]] [-h | -v] 
	-- utility tool to retrieve OB related information (grades, date and start times, comments) 
	   from NLT database (normal case), or associations database (old system) 

        date: 	    DFO date; note that e.g. $TODAY is collected in nightlog $NLT_TODAY (labelled "$NLT_LABEL").
	instrument: \$PANL_INSTRUMENT, or valid \$DFO_FILE_NAME for old nightlog info
	-C:	    calChecker mode (a loop over all calChecker dates)
	-V: 	    verbose output (for debugging)
	Output is in $TMP_DIR/ob_isql_out, $TMP_DIR/ob_isql_out2 and $TMP_DIR/ob_isql_out3.
EOT

# ================================================================
# 0.2 get options
# ===============================================================

if [ $# = 0 ] 
then
        cat $TMP_DIR/getObInfo.txt
        exit 0
fi

VERBOSE=NO
INSTRUMENT2=$PANL_INSTRUMENT
CAL_MODE=NO
OTHER_INS=NO

while getopts d:i:CVvh OPTION
do
	case "$OPTION" in
	 v ) echo $TOOL_VERSION
	     exit 0 ;;
	 h ) cat $TMP_DIR/getObInfo.txt
	     exit 0 ;;
	 d ) DATE=$OPTARG ;;
	 C ) CAL_MODE=YES ;;
	 i ) INSTRUMENT2=$OPTARG ; OTHER_INS=YES ;;
	 V ) VERBOSE=YES ;;
	 ? ) cat $TMP_DIR/getObInfo.txt
	     exit 0;;
	esac
done

# If called from createReport, the exported $JTMP_DIR is used; otherwise, $TMP_DIR is used 
if [ Q$JTMP_DIR = Q ] 
then
# if -C called from command line:
	CHECK_TMP=`echo $TMP_DIR | grep CALCHECK`
	if [ $CAL_MODE = YES ] && [ Q$CHECK_TMP = Q ]
	then
		TMP_DIR=$TMP_DIR/CALCHECK
	fi
	JTMP_DIR=$TMP_DIR
fi

if [ $VERBOSE = YES ]
then
	echo "[$TOOL_NAME] TMP_DIR used:	$JTMP_DIR"
fi

# ==========================================================================
# 0.3 check date format
# ==========================================================================

if [ "Q$DATE" != "Q" ]
then
	if [ `echo $DATE | wc -c` != "11" ]
	then
        	echo "*** ERROR: wrong date format: $DATE; should be: 2005-12-12; start again"
        	exit
	fi

	YEAR=`echo $DATE  | cut -c1-4`
	MONTH=`echo $DATE | cut -c6-7`
	DAY=`echo $DATE   | cut -c9-10`

	NLT_DATE="${DAY}-${MONTH}-${YEAR}"
	if [ $VERBOSE = YES ]
	then
		echo "[$TOOL_NAME] DATE: $DATE	(NLT_DATE: $NLT_DATE)"
	fi
elif [ $CAL_MODE = NO ]
then
	echo "*** ERROR: you must specify a date."
	exit
fi

# ==========================================================================
# 0.4 read NLT configuration from this tool
# ==========================================================================

NLT_APPLICABLE=`grep "^#NLT" $DFO_BIN_DIR/$TOOL_NAME | awk '{print $2,$3}' | grep "^$PANL_INSTRUMENT " | awk '{print $2}'`
if [ Q$NLT_APPLICABLE = Q ]
then
	NLT_APPLICABLE=20050101
fi

if [ $CAL_MODE = NO ]
then
	DATE_SHORT=`echo $DATE | sed "s/-//g"`
	if [ $DATE_SHORT -lt $NLT_APPLICABLE ] && [ $OTHER_INS = YES ] 
	then
# unsupported because we cannot match DFO_FILE_NAME and PANL_INSTRUMENT, and DFO_FILE_NAME is configured only in its own account.
		echo "[$TOOL_NAME] ERROR: we can retrieve OB information for $INSTRUMENT2 before $NLT_APPLICABLE only from its own account. Exit."
		exit
	fi

	if [ $VERBOSE = YES ]
	then
		if [ $DATE_SHORT -lt $NLT_APPLICABLE ]
		then
			echo "[$TOOL_NAME] DFO_FILE_NAME:	$DFO_FILE_NAME"
		else
			echo "[$TOOL_NAME] PANL_INSTRUMENT:	$INSTRUMENT2"
		fi
		echo "[$TOOL_NAME] PANL REF.DATE:	$NLT_APPLICABLE"
	fi
fi

# ==========================================================================
# 1. get OB grades and comments
# ==========================================================================

rm -f $JTMP_DIR/ob_isql* $JTMP_DIR/ob_isql_out*

if [ $CAL_MODE = NO ]
then
	echo $DATE > $JTMP_DIR/ob_list_dates
else
	if [ -s $JTMP_DIR/cc_list_dates3 ]
	then
		cp $JTMP_DIR/cc_list_dates3 $JTMP_DIR/ob_list_dates
	else
		cp $JTMP_DIR/cc_list_dates2 $JTMP_DIR/ob_list_dates
	fi
fi

# ==========================================================================
# 1.1 old info: associations..night_logs
# ==========================================================================

for D in `cat $JTMP_DIR/ob_list_dates`
do
	DATE2=`qcdate $D +1`
	cat >> $JTMP_DIR/ob_isql1 <<EOT
SELECT
	"$D" ,
	ob_id ,
	dp_id ,
	ob_grade
FROM
	associations..night_logs
WHERE
	dp_id like "${DFO_FILE_NAME}.${D}T2%" or dp_id like "${DFO_FILE_NAME}.${DATE2}T0%" or dp_id like "${DFO_FILE_NAME}.${DATE2}T1%"
GO
EOT

	cat >> $JTMP_DIR/ob_isql2 <<EOT
SELECT
	"$D" ,
	ob_id,
	 convert(varchar(30), dp_id), 
	"COMMENT"+convert(varchar($LENGTH), comments)
FROM
	associations..night_logs
WHERE
	dp_id like "${DFO_FILE_NAME}.${D}T2%" or dp_id like "${DFO_FILE_NAME}.${DATE2}T0%" or dp_id like "${DFO_FILE_NAME}.${DATE2}T1%"
GO
EOT
done

# ==========================================================================
# 1.2 standard scheme: night_log database; 3 queries: one for grade, one for
#     public_comment, one for internal_comment; plus a combined query
# ==========================================================================

for D in `cat $JTMP_DIR/ob_list_dates`
do
	DATE1=`qcdate $D +1`
	DATEdot=`echo $D | sed "s/-/\./g"`
	DATE1dot=`echo $DATE1 | sed "s/-/\./g"`

	cat >> $JTMP_DIR/ob_isql1_nlt <<EOT
use night_log
go

SELECT
	"$D" as DATE,
	ob_id ,	
	convert(varchar(5),start_time,108) as START ,
	convert( char(2), grade ) 
FROM
	calchecker_metadata
WHERE
	(
	 convert(varchar(10),start_time,102) = "$DATEdot" and convert(varchar(5),start_time,108) >= "21:00"
OR	
	 convert(varchar(10),start_time,102) = "$DATE1dot" and convert(varchar(5),start_time,108) < "21:00"
	)
AND
	name in ("$INSTRUMENT2")
ORDER by "START"
go
EOT

	cat >> $JTMP_DIR/ob_isql2_nlt <<EOT
use night_log
go

SELECT
	"$D" as DATE,
	ob_id ,
	"UT_START"+convert(varchar(5),start_time,108) as START ,
	"COMMENT"+convert(varchar($LENGTH), public_comment ) 
FROM
	calchecker_metadata
WHERE
	(
	 convert(varchar(10),start_time,102) = "$DATEdot" and convert(varchar(5),start_time,108) >= "21:00"
OR
	 convert(varchar(10),start_time,102) = "$DATE1dot" and convert(varchar(5),start_time,108) < "21:00"
	)
AND
	name in ("$INSTRUMENT2")
ORDER by "START"
go
EOT

	cat >> $JTMP_DIR/ob_isql3_nlt <<EOT
use night_log
go

SELECT
	"$D" as DATE,
	ob_id ,
	"UT_START"+convert(varchar(5),start_time,108) as START ,
	"ICOMM"+convert(varchar($LENGTH), internal_comment ) 
FROM
	calchecker_metadata
WHERE
	( 
	 convert(varchar(10),start_time,102) = "$DATEdot" and convert(varchar(5),start_time,108) >= "21:00"
OR	
	 convert(varchar(10),start_time,102) = "$DATE1dot" and convert(varchar(5),start_time,108) < "21:00"
        )
AND
	name in ("$INSTRUMENT2")
ORDER by "START"
go
EOT

# this query combines grade and comments and is better for createReport
	cat >> $JTMP_DIR/ob_isql4_nlt <<EOT
use night_log
go

SELECT
	"$D" as DATE,
	ob_id ,	
	"UT_START"+convert(varchar(5),start_time,108) as START ,
	"OB_GRADE"+convert( char(2), grade ) ,
	"COMMENT"+convert(varchar($LENGTH), public_comment ) 
FROM
	calchecker_metadata
WHERE
	(
	 convert(varchar(10),start_time,102) = "$DATEdot" and convert(varchar(5),start_time,108) >= "21:00"
OR
	 convert(varchar(10),start_time,102) = "$DATE1dot" and convert(varchar(5),start_time,108) < "21:00"
	)
AND
	name in ("$INSTRUMENT2")
ORDER by "START"
go
EOT

done

# ==========================================================================
# 1.3 switch: $NLT_APPLICABLE
# ==========================================================================
# we use the last date to decide for the old or the new scheme

LAST_DATE=`cat $JTMP_DIR/ob_list_dates | tail -1`
DATE_SHORT=`echo $LAST_DATE | sed "s/-//g"`

rm -f $JTMP_DIR/ob_isql_out*

# NLT scheme
if [ $DATE_SHORT -ge $NLT_APPLICABLE ]
then
	if [ $VERBOSE = YES ]
	then
		echo "[$TOOL_NAME] OB grades and comments taken from nightlog tool"
	else	
		echo "- Find OB grades and comments from nightlog tool ..."
	fi

	isql -S$ARCH_SERVER -Ucalchecker -Puser2qc -w 999 -i $JTMP_DIR/ob_isql1_nlt \
 | grep -v affected \
 | sed "1,2 d" \
 | sed "s/\\\//" \
 | sed "/^$/D" > $JTMP_DIR/ob_isql_out

	isql -S$ARCH_SERVER -Ucalchecker -Puser2qc -w 999 -i $JTMP_DIR/ob_isql2_nlt \
 | grep -v affected \
 | sed "1,2 d" \
 | sed "s/\\\//" \
 | sed "/^$/D" \
 | sed "s/COMMENT .*/NOCOMMENT/" \
 > $JTMP_DIR/ob_isql_out2

# sort before/after midnight
	grep "UT_START2" $JTMP_DIR/ob_isql_out2 2>/dev/null > $JTMP_DIR/ob_isql_out21
	grep "UT_START[0-1]" $JTMP_DIR/ob_isql_out2 2>/dev/null >> $JTMP_DIR/ob_isql_out21
	mv $JTMP_DIR/ob_isql_out21 $JTMP_DIR/ob_isql_out2

	isql -S$ARCH_SERVER -Ucalchecker -Puser2qc -w 999 -i $JTMP_DIR/ob_isql3_nlt \
 | grep -v affected \
 | sed "1,2 d" \
 | sed "s/\\\//" \
 | sed "/^$/D" \
 | sed "s/ICOMM .*/NOCOMMENT/" > $JTMP_DIR/ob_isql_out3

# sort before/after midnight
	grep "UT_START2" $JTMP_DIR/ob_isql_out3 2>/dev/null > $JTMP_DIR/ob_isql_out31
	grep "UT_START[0-1]" $JTMP_DIR/ob_isql_out3 2>/dev/null >> $JTMP_DIR/ob_isql_out31
	mv $JTMP_DIR/ob_isql_out31 $JTMP_DIR/ob_isql_out3

	isql -S$ARCH_SERVER -Ucalchecker -Puser2qc -w 999 -i $JTMP_DIR/ob_isql4_nlt \
 | grep -v affected \
 | sed "1,2 d" \
 | sed "s/\\\//" \
 | sed "/^$/D" \
 | sed "s/ICOMM .*/NOCOMMENT/" > $JTMP_DIR/ob_isql_out4

# sort before/after midnight
	grep "UT_START2" $JTMP_DIR/ob_isql_out4 2>/dev/null > $JTMP_DIR/ob_isql_out41
	grep "UT_START[0-1]" $JTMP_DIR/ob_isql_out4 2>/dev/null >> $JTMP_DIR/ob_isql_out41
	mv $JTMP_DIR/ob_isql_out41 $JTMP_DIR/ob_isql_out4

# old scheme
else
	if [ $VERBOSE = YES ]
	then
		echo "[$TOOL_NAME] OB grades and comments taken from associations database ..."
	else	
		echo "- Find OB grades and comments from associations database ..."
	fi

	isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $JTMP_DIR/ob_isql1 \
 | grep -v affected \
 | sed "1,2 d" \
 | sed "s/\\\//" \
 | sed "/^$/D" \
 | sed "s/${DFO_FILE_NAME}...........T/UT_START/" \
 | sed "s/UT_START...../&END/" \
 | sed "s/END.......//" \
 | sed "s/).*//" \
> $JTMP_DIR/ob_isql_out

	if [ -s  $JTMP_DIR/ob_isql_out ]
	then
		for OB in `cat $JTMP_DIR/ob_isql_out | awk '{print $2}' | sort -u`
		do
# we cannot find obs_start in any database, which would be the best choice;
# instead, we catch the first entry per OB always, this is at odds with the new NLT scheme but 
# there seems to be no other choice
			grep $OB $JTMP_DIR/ob_isql_out | head -1 >> $JTMP_DIR/ob_isql_out11
		done
		cat $JTMP_DIR/ob_isql_out11 | sort -k3,3 > $JTMP_DIR/ob_isql_out
	fi

	isql -S${QC1_SERVER} -Uqc -P`cat $QC1_PWD` -w999 -i $JTMP_DIR/ob_isql2 \
 | grep -v affected \
 | sed "1,2 d" \
 | sed "s/\\\//" \
 | sed "/^$/D" \
 | sed "s/COMMENT .*/NOCOMMENT/" \
 | sed "s/ COMMENT$/ NOCOMMENT/" \
 | grep -v "^                  Requested" \
 | grep -v "^Seeing:" \
 | grep -v "^Airmass:" \
 | grep -v "^FLI:" \
 | grep -v "^Moon Distance:" \
 | grep -v "^Sky Transparency:" \
 | grep -v "^Strehl Ratio:" \
 | sed "s/${DFO_FILE_NAME}...........T/UT_START/" \
 | sed "s/UT_START...../&END/" \
 | sed "s/END.......//" \
> $JTMP_DIR/ob_isql_out2

	if [ -s  $JTMP_DIR/ob_isql_out2 ]
	then
		for OB in `cat $JTMP_DIR/ob_isql_out2 | awk '{print $2}' | sort -u`
		do
			grep $OB $JTMP_DIR/ob_isql_out2 | head -1 >> $JTMP_DIR/ob_isql_out21
		done
		cat $JTMP_DIR/ob_isql_out21 | sort -k3,3 > $JTMP_DIR/ob_isql_out2
	fi
	
fi

if [ $VERBOSE = YES ]
then
	if [ $DATE_SHORT -ge $NLT_APPLICABLE ]
	then
		echo "[$TOOL_NAME] Queries in $JTMP_DIR/ob_isql1_nlt, $JTMP_DIR/ob_isql2_nlt, $JTMP_DIR/ob_isql3_nlt and $JTMP_DIR/ob_isql4_nlt."
		echo "[$TOOL_NAME] Output in $JTMP_DIR/ob_isql_out (grades), $JTMP_DIR/ob_isql_out2 (public comments), $JTMP_DIR/ob_isql_out3 (internal comments)"
		echo "[$TOOL_NAME] and $JTMP_DIR/ob_isql_out4 (grades+public comments)"
	else
		echo "[$TOOL_NAME] Queries in $JTMP_DIR/ob_isql1 and $JTMP_DIR/ob_isql2"
		echo "[$TOOL_NAME] Output in $JTMP_DIR/ob_isql_out (grades) and $JTMP_DIR/ob_isql_out2 (comments)"
	fi
fi

exit

