#!/bin/sh
# NAME:		calselManager	
# PURPOSE:      This is a tool for creation, maintenance and testing of calSelector OCA rules.
#		Components: 
#		 - migration for OCA rules from DFOS_OPS to CALSELECTOR syntax
#		 - ingestion of CALSELECTOR RLS file into database
#	    	 - testing with reference ABs
# AUTHOR:       Reinhard Hanuschik/QCG
# VERSION:      1.0 -- created (2011-11-15)
#		2.0 -- supports part 2 (RLS management) and time-match rules for part 1 (2011-12-12)
#		2.1 -- supports part 3 (testing); added: checks for unresolved macros (2012-01-31)
#		2.1.1- minor modifs for level2/direct; the tool version for testing is hard-coded (2012-02-06)
#		2.1.2- updated help (2012-02-07)
#		2.1.3- bug fix for dp_id which is not primary (2012-02-09)
#		2.2 -- debug option for dynamic tests; bug fix for multiple science files; option FILE for performance and regression tests; specify stop-date  (2012-02-10)
#		2.2.1- bug fix for dp_id which is primary (2012-02-16)
#		2.2.2- error mail if dynamic association failed; help for CHECK added (2012-02-23)
#		2.2.3- bug fix for mode -f (test), in case no static association exists (2012-03-15)
#		2.3 -- uses new calSelector v1.1; adapted: database interaction, sdiff; CALSELECTOR_TEST obsolete (2012-05-11)
#		2.3.1- replace observations by obs_metadata database (2012-08-24)
#		2.4 -- replace stargate1 with $DFO_WEB_SERVER (2013-10-30)
#		2.4.1- replace 'switch' by 'change' in awk commands (for OS upgrade) (2014-02-17)
#		3.0_beta1 -- partly rewritten, to cope with calSelector v2.0; verification part outsourced to verifyAB (2014-11-13)
#		3.0 -- includes changes to Raw2Raw flag; r2m_allowed dropped (2015-02-09)
#		3.0.1- uses dfs installation (2015-06-22)
#		3.0.2- rulesManager call modified; defaults for deletion and creation of VPRODUCTS: N (2015-08-13)
#		3.1 -- cope with calSelector v3.0; modified logging (2018-12-19)
#		3.2 -- added: VALIDATE mode (2019-02-16)
#
# PARAMETERS:	-r pathname of DFOS_OPS RLS file: optional, default is $DFO_CONFIG_DIR/OCA/<instr>.RLS
# OPTIONS:	-v|-h|-I[nformation]|-M[odify]
#		for more options type 'calselManager -h' or '-H'
# TOOLS CALLED: dfosRLSmassage.py
# CONFIG:	none; the tool reads from/writes into 
#		$DFO_CONFIG_DIR/CALSELECTOR/config.createCalibMap.calsel
#
# ToDo: 	align comments in database and on calselector web page and in $DFO_CONFIG_DIR/CALSELECTOR
# NOTE:		calSelector v3.0.3, as used here, is still buggy: it has no help, and it does not produce log files.
# =========================================================================
TOOL_VERSION="3.2"

TOOL_NAME="calselManager" 	
CMAP_TOOL_NAME="createCalibMap"

CALSELECTOR="CalSelector"
RULESMANAGER="RulesManager"
# there is also VirtualProductGenerator but this is not used by QC.

CALSELECTOR_CONFIG=$DFO_CONFIG_DIR/CALSELECTOR/CSConfiguration.properties # database configuration

GEN_VAL=10	# fixed extended validity, for calSelector performance and relaxed time matches

WEB_DIR="/home/qc/qc/ALL/OCA/CALSELECTOR"		# target dir for OCA rules and calibMap on $DFO_WEB_SERVER
OCA_URL="http://www.eso.org/qc/ALL/OCA/CALSELECTOR" 	# URL for collecting OCA rules
JQUERY_URL="http://www.eso.org/observing/dfo/quality/ALL/jscript"	# URL for jquery / tablesorter

RULES_DIR=$DFO_CONFIG_DIR/CALSELECTOR/DOWNLOADED

DBCM_MAIL="dbcm@eso.org"	# ops email to report issues with the IST

# =========================================================================
# 0.1 initialize
# =========================================================================

VERIFY=NO
INFO=NO
MODIFY=NO
BRINGBACK=NO
COMPILE=NO
UPLOAD=NO
RLSMNG=NO

CS_HELP=NO
MESSAGES=NORMAL
ALL_INS=NO

echo "USAGE: $TOOL_NAME  [-r <pathname of RLS file>] [-I|-C|-U]
	Tool to manage the OCA rules for calSelector.
	It is used for two main tasks: 
	- to align the DFOS_OPS and CALSELECTOR ${DFO_INSTRUMENT}.RLS rules,
	- to ingest them into the database.
                  
	-h: this help
	-v: display tool and calSelector versions and exit
	-H: display calSelector help and exit

Task 1a. Alignment of DFOS_OPS and current CALSELECTOR rules
[possible options: -I -r]
	-I: information about the alignment: automatic part, manual edits, compilation, upload 
	-r <pathname of DFOS_OPS RLS file> (default: $DFO_CONFIG_DIR/OCA/${DFO_INSTRUMENT}.RLS)
-----------------------------------------------------------------------
[Task 1a is needed for new instruments only.]

Task 1b. Modifications of an already existing CALSELECTOR RLS file
[possible options: -M -B -C -U]
	-M: information about how to modify CALSELECTOR RLS files
	-B: begin of modify workflow: copy existing RLS file to $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS
	-C: compile the edited RLS file (starts with $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS)
	-U: enter validity date, upload the RLS file to the proper local and web server address
-----------------------------------------------------------------------

Task 2. Manage CALSELECTOR RLS files in database:
[possible options: -m -d -D -R -A]
	-m <mode> (LIST|READ|VALIDATE|WRITE|REMOVE|CHECK, see below)

	  -d reference date
	  -R <pathname of CALSELECTOR RLS file> 
	  -D output in DEBUG mode
	  -A all instruments

	-m LIST      [-A]
	-m READ      [-d <ref_date> [-D]]
	-m CHECK     [-d <ref_date> [-D]]
	-m VALIDATE  -R <pathname>  
	-m WRITE     -R <pathname>  [-D]
	-m REMOVE                   [-D]

   modes:
	-m LIST:   	list all existing rules for $DFO_INSTRUMENT [option -A: all]
 	-m READ:   	download OCA rule (specify name or ref_date [option -d <ref_date>])
	-m CHECK:	check that all keywords as listed in the OCA rules are in the ISTs [-d <ref_date>, default: TODAY]
	-m VALIDATE:	syntax check of specified RLS file
 	-m WRITE:  	ingest (write) $DFO_INSTRUMENT RLS file to database [-R <pathname of RLS file>]; 
		   	dialog for stop-date and virtual products
		   	pathname must follow convention: \$DFO_CONFIG_DIR/CALSELECTOR/<date>/<instrument>_raw2master.<date>.RLS
 	-m REMOVE: 	remove $DFO_INSTRUMENT RLS file in database; dialog for selection;
		   	dialog for deletion of virtual products

	" > $TMP_DIR/usage

# =========================================================================
# 0.2 get options
# =========================================================================

while getopts hvd:m:r:ABCDHIMR:UV OPTION
do
	case "$OPTION" in
# general
	 h ) cat $TMP_DIR/usage | more ; exit ;;
	 v ) echo "$TOOL_NAME $TOOL_VERSION " | awk '{print $1" (dfos tool): "$2}'
 $CALSELECTOR  --version | awk '{print clslctr,"  (dfs tool): ",$1}' clslctr=$CALSELECTOR
	     exit ;;
	 H ) CS_HELP=YES ;;
	 D ) MESSAGES=DEBUG ;;

# unification and modification
	 r ) RLS_FILE=$OPTARG ;;
	 B ) BRINGBACK=YES ;;
	 C ) COMPILE=YES ;;
	 I ) INFO=YES ;;
	 M ) MODIFY=YES ;;
	 V ) VERIFY=YES ;;
	 U ) UPLOAD=YES ;;

# database upload
	 d ) REF_DATE=$OPTARG ; RLSMNG=YES ;;
	 m ) MODE=$OPTARG ; 	RLSMNG=YES ;;
	 R ) RLS_FILE=$OPTARG ; RLSMNG=YES ;;
	 A ) ALL_INS=YES ; 	RLSMNG=YES ;;

         * ) cat $TMP_DIR/usage ; exit ;;
	esac
done

if [ $CS_HELP = YES ]
then
	echo "calSelector help is available for the following components:
1 - RulesManager 
2 - CalSelector
Enter your choice (1-2 or E[xit]):"
	read HELP
	
	case $HELP in
	 1) $RULESMANAGER --help ;;
	 2) $CALSELECTOR  --help  ;;
	 E) exit ;;
	esac
	echo "ERROR: help doesn't work for the current calSelector version."
	exit
fi

if [ $MESSAGES = DEBUG ]
then
      	MESSAGES=--debug
else
	MESSAGES=""
fi

VALIDATE="--validate.rules=true"

if [ $RLSMNG = NO ] && [ $VERIFY = NO ] && [ $INFO = NO ] && [ $MODIFY = NO ] && [ $COMPILE = NO ] && [ $UPLOAD = NO ] && [ $BRINGBACK = NO ]
then
	if [ Q$RLS_FILE = Q ]
	then
		RLS_FILE=$DFO_CONFIG_DIR/OCA/$DFO_INSTRUMENT.RLS
		echo "No rules file specified, default chosen: \$DFO_CONFIG_DIR/OCA/${DFO_INSTRUMENT}.RLS. OK (Y/N) (Y)?"
		read input
		if [ "Q$input" != QN ]
		then
			RLS_FILE=$DFO_CONFIG_DIR/OCA/$DFO_INSTRUMENT.RLS
		else
			echo "Think about it."
			exit
		fi		
		ASSOC_FILE=$DFO_CONFIG_DIR/OCA/${DFO_INSTRUMENT}_association.h
	else
		if [ ! -s ${RLS_FILE} ]
		then
			echo "***ERROR: ${RLS_FILE} not existing. Exit."
			exit -1
		fi
		echo "Rules file chosen: ${RLS_FILE}. Confirm that this is in DFOS_OPS style: (Y/N)"
		read CONFIRM
		if [ Q$CONFIRM != QY ]
		then
			echo "Think about it."
			exit
		fi	
		
		echo "We also need the corresponding association file. For standard DFOS_OPS,
this file is $DFO_CONFIG_DIR/OCA/${DFO_INSTRUMENT}_association.h . Take either this one, or enter the full pathname:"
		read ASSOC_FILE
		if [ ! -s $ASSOC_FILE ]
		then
			echo "***ERROR: $ASSOC_FILE not existing. Exit."
			exit -1
		elif [ Q$ASSOC_FILE = Q ]
		then
			ASSOC_FILE=$DFO_CONFIG_DIR/OCA/${DFO_INSTRUMENT}_association.h
			echo "$ASSOC_FILE chosen."
			echo ""
		fi
	fi
fi

# =========================================================================
# 1. Options related to migration: info, verification, compilation
# 1.1 Verification of group-by clauses: likely obsolete
# =========================================================================

if [ $VERIFY = YES ]
then
	if [ ! -s $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS ]
	then
		echo "You must first run '$TOOL_NAME' before you can call this option. Exit."
		exit 
	fi

	echo "These are the 'group by' clauses in your \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS. They should contain "
	echo " TPL.START or ARCFILE only, unless you have multi-valued templates. Hit return:"
	read input
	echo "================="
	echo "lines:"
	grep -n "group by" $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS 
	echo "================="
	echo "
If there is a clause with more conditions, find it in $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS, analyze and probably fix it.
" 
	exit
fi

# =========================================================================
# 1.2.1 Info about OCA rules unification
# =========================================================================

if [ $INFO = YES ]
then
	echo "Information about the unification of DFOS_OPS and CALSELECTOR OCA rules
=======================================================================

Step 1 - the automatic part:
>> $TOOL_NAME -r <DFOS_OPS rules file> (default is $DFO_CONFIG_DIR/OCA/${DFO_INSTRUMENT}.RLS)
--> output in \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1

At that stage, 
- PACK_DIR entries have been removed,
- time match rules have been added:
  'and MJD-OBS tryBetween inputFile.MJD-OBS - \$VAL and inputFile.MJD-OBS + \$VAL and MJD-OBS between inputFile.MJD-OBS - \$GEN_VAL and inputFile.MJD-OBS + \$GEN_VAL'
  with \$VAL taken from the DELTAT_RULE and \$GEN_VAL hard-coded to be $GEN_CAL.
  Note the tryBetween (which follows the calibration plan) and the between (which is more relaxed).

Associated mcalibs *must* have a time match rule, 
and static mcalibs *should* have a non-between time match rule (likely a
PREVIOUS time match). Any exception to this is marked and needs to be edited in 
the next step.

If you plan to force the rule to go Raw2Raw, best is to add, in the very beginning, something like:

if INS.MODE==\"SOME.MODE\" then
{
  FORCE.RAW2RAW=T
}

Note that you can use any header keyword, but no \"synthetic\" keyword created by the OCA rule itself (like RAW.TYPE).

Step 2 - the review part: check \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS

- cp \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1 to \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS
  (NOT done by the tool in order to give you full control);
- work with \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS.

- remember this OCA RLS file can be used within CALSELECTOR and DFOS_OPS.
  Make sure to keep all data types that are needed for DFOS_OPS, no matter if they are used
  for CALSELECTOR or not. Make sure that every data type which is used for CALSELECTOR has
  a product defined, even if so far this was not needed.

Step 3 - compilation: Compile \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS:
>> $TOOL_NAME -C
This will expand the macros for RAW.TYPE, remove all comments etc.

Step 4 - ingestion: Enter a validity date and a comment, and store the rules file as ${DFO_INSTRUMENT}_raw2master.\${VAL_DATE}.RLS:
>> $TOOL_NAME -U
You define a validity date (that rule will be applied to data from that date on).
You also enter a comment to describe the rule (e.g. what is different to the other versions).
Now your file will be stored as ${DFO_INSTRUMENT}_raw2master.\${VAL_DATE}.RLS
in \$DFO_CONFIG_DIR/CALSELECTOR/\${VAL_DATE}. It is uploaded for reference to
${OCA_URL}/\${VAL_DATE}. 

It is now ready to be uploaded to the calSelector database.

The next step is to create or update the calib map: 
$CMAP_TOOL_NAME -r \$DFO_CONFIG_DIR/CALSELECTOR/\${VAL_DATE}/${DFO_INSTRUMENT}_raw2master.\${VAL_DATE}.RLS

Then, continue with uploading the RLS file to the calSelector database:
$TOOL_NAME -R \$DFO_CONFIG_DIR/CALSELECTOR/\${VAL_DATE}/${DFO_INSTRUMENT}_raw2master.\${VAL_DATE}.RLS .

You may find the following files useful:
- \$TMP_DIR/rm_all_products (list of all products)
- \$TMP_DIR/rm_gencalibs    (list of gencalibs)
" > $TMP_DIR/calsel_info
	cat $TMP_DIR/calsel_info | more
	exit
fi

# =========================================================================
# 1.2.2 Info about modifications
# =========================================================================

if [ $MODIFY = YES ]
then
	echo "
How to modify an existing OCA rule
=======================================================================
Step 1:
>> $TOOL_NAME -B
The dialog will ask you to specify the version date, to identify the rule you want to modify. 
The modification may end up in the same, modified RLS file, or in a new version.

Step 2: Interactive part: modify \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS

Step 3: Compile \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS:
>> $TOOL_NAME -C
This will expand the macros for RAW.TYPE, remove all comments etc.

Step 4: Enter a validity date and a comment, and store the rules file as ${DFO_INSTRUMENT}_raw2master.\${VAL_DATE}.RLS:
>> $TOOL_NAME -U
You define a validity date (that rule will be applied to data from that date on).
You also enter a comment to describe the rule (e.g. what is different to the other versions).
Now your file will be stored as ${DFO_INSTRUMENT}_raw2master.\${VAL_DATE}.RLS
in \$DFO_CONFIG_DIR/CALSELECTOR/\${VAL_DATE}. It is uploaded for reference to
${OCA_URL}/\${VAL_DATE}. 

It is now ready to be uploaded to the calSelector database.

The next step is to create or update the calib map: 
$CMAP_TOOL_NAME -r \$DFO_CONFIG_DIR/CALSELECTOR/\${VAL_DATE}/${DFO_INSTRUMENT}_raw2master.\${VAL_DATE}.RLS

Then, continue with uploading the RLS file to the calSelector database:
$TOOL_NAME -R \$DFO_CONFIG_DIR/CALSELECTOR/\${VAL_DATE}/${DFO_INSTRUMENT}_raw2master.\${VAL_DATE}.RLS ." | more

	exit
fi

# =========================================================================
# 1.3 Compilation of rules
#     brings $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS 
#         to $DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS
# =========================================================================

if [ $COMPILE = YES ]
then
	echo "We will now compile \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS. Hit return:"
	read OK_YN

	if [ ! -s $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS ]
	then
		echo "***ERROR: no file \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS found. Exit."
	fi

	CHECK_MODIF=`egrep "!!!CHECK|!!!FIX|FIX_BETWEEN" $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS | head -1`
	if [ "Q$CHECK_MODIF" != Q ]
	then
		echo "***ERROR: \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS still contains entries marked as '!!!CHECK' or similar: "
		egrep "!!!CHECK|!!!FIX|FIX_BETWEEN" $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS
		echo "Fix and then try again."
		exit
	fi

	mkdir $DFO_CONFIG_DIR/CALSELECTOR 2>/dev/null

	echo "Calling cpp -P $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS $DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS ..."
	cpp -P $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS $DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS

	CHECK_DEF=`grep " DEF_" $DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS | head -1`

	if [ $? != 0 ]
	then
		echo "An error occurred. Check out and try again."
	elif [ "Q$CHECK_DEF" != Q ]
	then
		echo "There seem to be issues with unresolved macros 'DEF_'. Please fix before continuing:"
		grep " DEF_" $DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS | head -1
	else
		echo "
If there are warnings, you may want to check and fix them in
\$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS and then compile again.

... the file was compiled successfuly. 
Check it out under \$DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS.
As the final step (upload the RLS file), call '$TOOL_NAME -U'."
	fi
	exit
fi

# =========================================================================
# 1.4 Validity date and upload to web site
#     brings $DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS
#     to     $DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/${DFO_INSTRUMENT}_raw2master.${VAL_DATE}.RLS
# =========================================================================

if [ $UPLOAD = YES ]
then
	if [ ! -s $DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS ]
	then
		echo "***ERROR: there is no file ${DFO_INSTRUMENT}_raw2master.RLS under \$DFO_CONFIG_DIR/CALSELECTOR ready for upload. "
		echo "          You may want to call '$TOOL_NAME -C' to compile a RLS file \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS "
		echo "          and then try the -U option again."
		exit
	fi

	echo "To define raw2master calSelector rules with versioning, we need a validity date 
(date from which on the rule set is valid). This date will become part of the name.
"

	CHECK_EXIST=`ls -d $DFO_CONFIG_DIR/CALSELECTOR/20* | head -1`
	if [ "Q$CHECK_EXIST" != Q ]
	then
		echo "You have already the following validity date(s):"
		ls -1 -d $DFO_CONFIG_DIR/CALSELECTOR/20*
	fi

	echo "
If you have (currently) only one rule, enter an early date (before start of operations).

Now enter the date for \$DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS (e.g. 2005-01-01):"

	read VAL_DATE
	if [ Q$VAL_DATE = Q ]
	then
		echo "Think about it."
		exit
	else
		if [ -d $DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE ]
		then
			CHECK_R2R=`ls $DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/* | grep ${DFO_INSTRUMENT}_raw2raw | head -1`
			CHECK_R2M=`ls $DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/* | grep ${DFO_INSTRUMENT}_raw2master | head -1`
			if [ Q$CHECK_R2M = Q ] && [ Q$CHECK_R2R != Q ]
			then
				echo "*** INFO: There is already a directory \$DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE"
				echo "          but with raw2raw content only. The raw2master rule will be added. Hit return to continue:"
				read input
			elif [ Q$CHECK_R2M != Q ]
			then
				echo "*** INFO: There is already a directory \$DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE"
				echo "          with raw2master content. "
          			echo "          Do you want to overwrite its content (Y/N) (N)?"
				read OVERWRITE_YN

				if [ Q$OVERWRITE_YN != QY ]
				then
					echo "OK, no action taken."
					exit
				fi
			fi
			NEW_DIR=N
		else
			mkdir $DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE
			NEW_DIR=Y
		fi

# check for entry in $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel
		echo "Now enter a comment to characterize this version 
(it will be stored in $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel)."
		if [ ! -s $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel ]
		then
			echo "***ERROR: please install ${CMAP_TOOL_NAME} tool and then come back to this step."
			exit -1
		fi

		echo "<your comment (one line only)>" > $TMP_DIR/calsel_comment
		CHECK_EXIST=`grep "$VAL_DATE" $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel`
		if [ "Q$CHECK_EXIST" = Q ]
		then
			echo "(e.g. \"applicable after CCD upgrade\" or \"includes new TFLATs introduced in P88\")
(one line only):"
			xterm -T "Enter your comment (one line only); use $DFO_EDITOR" -geom 100x20+1600+10 -e $DFO_EDITOR $TMP_DIR/calsel_comment
			LEAVE_YN=N
		else
			echo "
A comment is already existing:"
			echo $CHECK_EXIST
			echo "Leave it (Y) or replace it (N)? (Y)"
			read LEAVE_YN
			if [ Q$LEAVE_YN = QN ]
			then
				echo "(e.g. \"applicable after CCD upgrade\" or \"includes new TFLATs introduced in P88\")
(one line only):"
				xterm -T "Enter your comment (one line only); use $DFO_EDITOR" -geom 100x20+1600+10 -e $DFO_EDITOR $TMP_DIR/calsel_comment
			else
				LEAVE_YN=Y
			fi
		fi

		VERS_COMMENT=`cat $TMP_DIR/calsel_comment`
		if [ $NEW_DIR = Y ]
		then
			echo "START_DATE	$VAL_DATE	&&${VERS_COMMENT}&&" >> $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel
		elif [ Q$LEAVE_YN = QN ]
		then
			sed -i -e "s/^START_DATE.*${VAL_DATE}.*/START_DATE	$VAL_DATE	\&\&${VERS_COMMENT}\&\&/" $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel
		elif [ Q$LEAVE_YN = QY ]
		then
			echo "... comment left untouched."
		fi
	
		echo "
We will now move $DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS to $DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE
and also scp it to ${DFO_WEB_SERVER}:${WEB_DIR}/${DFO_INSTRUMENT}/${VAL_DATE}. 
Hit return to confirm and proceed:"
		read input

		mv $DFO_CONFIG_DIR/CALSELECTOR/${DFO_INSTRUMENT}_raw2master.RLS $DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/${DFO_INSTRUMENT}_raw2master.${VAL_DATE}.RLS

		ssh -o BatchMode=yes ${DFO_WEB_SERVER} "/home/qc/bin/qcDircheck $WEB_DIR/${DFO_INSTRUMENT}" 
		ssh -o BatchMode=yes ${DFO_WEB_SERVER} "/home/qc/bin/qcDircheck $WEB_DIR/${DFO_INSTRUMENT}/${VAL_DATE}" 
		chmod u+w $DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/${DFO_INSTRUMENT}_raw2master.${VAL_DATE}.RLS
		scp $DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/${DFO_INSTRUMENT}_raw2master.${VAL_DATE}.RLS ${DFO_WEB_SERVER}:${WEB_DIR}/${DFO_INSTRUMENT}/${VAL_DATE} 

		echo "
... done. Find the file under \$DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/${DFO_INSTRUMENT}_raw2master.${VAL_DATE}.RLS
and for reference under ${OCA_URL}/${DFO_INSTRUMENT} .

[If you have entered a wrong date, just delete \$DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE and call again '$TOOL_NAME -C'.
Then you should delete by hand the directory 
$OCA_URL/CALSELECTOR/$DFO_INSTRUMENT/${VAL_DATE} .]"

		echo "
It is recommended to create the calibmap for that file now:
	$CMAP_TOOL_NAME -r \$DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/${DFO_INSTRUMENT}_raw2master.${VAL_DATE}.RLS
  and	$CMAP_TOOL_NAME -r \$DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/${DFO_INSTRUMENT}_raw2master.${VAL_DATE}.RLS -A

Note that the new/updated CALSELECTOR rules file is NOT yet ingested to the associations database.
To do so, you need to call:
	$TOOL_NAME -m LIST	get an overview of existing rules files
	$TOOL_NAME -m REMOVE	remove the old version of the rules file
	$TOOL_NAME -m WRITE	ingest the new version of the rules file
"
	fi
	exit
fi

# =========================================================================
# 1.5 BRINGBACK=YES; bring an existing OCA rule back to $TMP_DIR for modifications
# =========================================================================

if [ $BRINGBACK = YES ]
then
	echo "Now we start the modification of an existing version, or the creation of a new version of a calselector RLS file."
	if [ ! -s $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel ]
	then
		echo "*** ERROR: no \$DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel found.
    It seems that no valid calselector RLS exist. Exit."

		exit -1
	fi
	
	CHECK_EXIST=`grep "^START_DATE" $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel | wc -l`
	if [ Q$CHECK_EXIST = Q1 ]
	then
		START_DATE=`grep "^START_DATE" $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel | awk '{print $2}'`
		echo "One version found: $START_DATE."
	elif [ Q$CHECK_EXIST = Q0 ]
	then
		echo "Nothing found. Exit."
		exit
	else
		echo "There is more than one version available. Select the one you want to use for modification or creation of a new file (enter the start date):"
		echo "START_DATE	COMMENT"
		echo "---------------------------------------------------"
		grep "^START_DATE" $DFO_CONFIG_DIR/CALSELECTOR/config.${CMAP_TOOL_NAME}.calsel | awk '{print $0}' | sed "s/^START_DATE[[:space:]]*//"
		read START_DATE
		if [ Q$START_DATE = Q ]
		then
			echo "You need to select a START_DATE. Try again."
			exit
		fi
	fi

	echo "
We copy \$DFO_CONFIG_DIR/CALSELECTOR/$START_DATE/${DFO_INSTRUMENT}_raw2master.${START_DATE}.RLS back to \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS. 
Hit return to continue:"
	read input
	cp $DFO_CONFIG_DIR/CALSELECTOR/${START_DATE}/${DFO_INSTRUMENT}_raw2master.${START_DATE}.RLS $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS
	echo "... done. Now you can start editing \$TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS and then continue with 
$TOOL_NAME -C
$TOOL_NAME -U
"
	exit
fi

# =========================================================================
# 2. OCA rule migration (DFOS_OPS to CALSELECTOR)
# 2.1 Preparation: suppress outdated content
# =========================================================================

if [ $RLSMNG = NO ] 
then
	echo "1.  Find matches pro.catg to raw.type for mapping ..."
	cat $RLS_FILE | dfosRLSmassage.py |\
 sed "s/^}/AA}/" |\
 awk '{
  if ( $1 == "action" ) { change=1 ; action=$2; recipe="none"; product="";  }
  if ( change == 1 && $1 == "recipe" )  { recipe=$2 }
  if ( change == 1 && $1 == "product" ) { product=$2; print action, recipe, product }
  if ( change == 1 && $1 == "AA}" ) { print action, recipe, product; change=0 }
 }' change=0 |\
 sort -u |\
 sed "s/;//" > $TMP_DIR/rm_dyn_products

	echo "    Check that all ACTIONs have a product defined:"
	CHECK_ISSUE=""
	for A in `cat $TMP_DIR/rm_dyn_products | awk '{print $1}' | sort -u`
	do
		CHECK_PROD=`grep "^$A " $TMP_DIR/rm_dyn_products | awk '{print $3}' | sort -u | head -1`
		if [ "Q$CHECK_PROD" = Q ]
		then
			CHECK_ISSUE=YES
			echo "*** WARNING: no product defined for $A"
		fi	
	done

	if [ Q$CHECK_ISSUE = QYES ]
	then
		echo "This should be fixed at least for all raw_types that are relevant for science reduction (either indirectly or directly). 
Also, all SCIENCE raw_types need a product defined (one is sufficient) in order to define the datasets (virtual products).
Technical or Health Check calibrations don't matter. 

There is no easy way for the tool to decide that these conditions are met. Therefore, please fix the OCA rules as
appropriate and then try again. If you are using an operational rules file, edit the association file
$ASSOC_FILE, and then call:
 'cd $DFO_CONFIG_DIR/OCA; gcc -E \"${DFO_INSTRUMENT}.h\" | grep -v \"^#\" | grep -v \"^$\" | dfosRLSmassage.py > $RLS_FILE' 
It might be necessary to add u+w permission to $RLS_FILE.
Then try again.

Please confirm that the above list does not contain any issues (Y/N) (N):"
		read CONFIRM
		if [ Q$CONFIRM != QY ]
		then
			echo "Think about it."
			exit
		fi
	fi

	cat $RLS_FILE | dfosRLSmassage.py |\
 sed "s/^}/AA}/" |\
 sed "s/select file/selectfile/"g |\
 awk '{
  if ( $1 == "action" ) { change=1 }
  if ( change == 1 && $1 == "selectfile" ) { product=$3; print product }
  if ( change == 1 && $1 == "AA}" ) { change=0 }
 }' change=0 |\
 sed "s/;//"g |\
 sed "s/and//" |\
 sort -u \
 > $TMP_DIR/rm_all_products

# find gencalibs
	rm -f $TMP_DIR/rm_gencalibs
	for A in `cat $TMP_DIR/rm_all_products | awk '{print $1}' | grep -v RASSOC | sed "s/_MASSOC//" | sort -u`
	do
		CHECK_GENCALIB=`grep " ${A}$" $TMP_DIR/rm_dyn_products | head -1 | awk '{print $1}'`
		if [ "Q$CHECK_GENCALIB" = Q ]
		then
			echo $A >> $TMP_DIR/rm_gencalibs
		fi
	done

# time match rules from $ASSOC_FILE
        grep DELTAT_RULE $ASSOC_FILE | grep "^//[[:space:]]ACTION"  > $TMP_DIR/rm_deltat

        sed -i -e "s/^.*ACTION_/ACTION_/" $TMP_DIR/rm_deltat
        sed -i -e "s/DELTAT_RULE=.*/and MJD-OBS tryBetween inputFile.MJD-OBS - & and inputFile.MJD-OBS + & and MJD-OBS between inputFile.MJD-OBS - $GEN_VAL and inputFile.MJD-OBS + $GEN_VAL/" $TMP_DIR/rm_deltat
        sed -i -e "s/DELTAT_RULE=//g" $TMP_DIR/rm_deltat
	
	cat $TMP_DIR/rm_deltat | sort -u > $TMP_DIR/rm_deltat1
	mv $TMP_DIR/rm_deltat1 $TMP_DIR/rm_deltat

# =========================================================================
# 2.2 classification
# =========================================================================

	cd $TMP_DIR
	echo "2.  We pre-edit the DFOS_OPS rules ...
2.1 Classification section (if...then): strip off PACK_DIR"

	cat $RLS_FILE | grep -v "PACK.DIR =" | dfosRLSmassage.py > $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1

	echo "2.2 Association section: check for well-formed RASSOC associations ..."
	CHECK_RASSOC=`grep "_RASSOC" $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1 | grep RAW.TYPE | head -1`
	if [ "Q$CHECK_RASSOC" != Q ]
	then
		echo "  There are RASSOC entries with RAW.TYPE being used. For the unification, they need to be replaced by the header keywords"
		echo "  as used in the ${DFO_INSTRUMENT}_classification.h file."
		echo "================="
		echo "lines:"
		grep -n "_RASSOC" $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1 | grep RAW.TYPE
		echo "================="
		echo "  You must replace RAW.TYPE by the proper values in the source file (it will still work fine in DFOS_OPS):"
		echo "  $ASSOC_FILE"
		echo "  After editing this file, call:
  'cd $DFO_CONFIG_DIR/OCA; gcc -E \"${DFO_INSTRUMENT}.h\" | grep -v \"^#\" | grep -v \"^$\" | dfosRLSmassage.py > $RLS_FILE' "
		echo "  Then try again."
		exit
	else
		echo "    ... ok, no issue found. Hit return:"
	fi
	read input

# =========================================================================
# 2.3 add between statements, only for dynamic mcalibs
#     for gencalibs, make sure to detect cases without MJD-OBS constraint!
# =========================================================================

	if [ -s $TMP_DIR/rm_deltat ]
	then
		echo "2.3 a) Mark existing MJD-OBS statements ..."
		sed -i -e "/MJD-OBS/s|^.*|& !!!CHECK DUPLICATE MJD_OBS!!!|" $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1 
# remove markings for gencalibs		
 		if [ -s $TMP_DIR/rm_gencalibs ]
		then
			for SC in `cat $TMP_DIR/rm_gencalibs`
			do
				sed -i -e "/$SC/s|!!!CHECK DUPLICATE MJD_OBS!!!||" $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1
			done
		fi
		
		echo "    b) Add between statements from $ASSOC_FILE ..."

		for ACTION in `cat $TMP_DIR/rm_deltat | awk '{print $1}' | sort -u`
		do
			for MCAL in `grep "^$ACTION" $TMP_DIR/rm_deltat | sort -u | awk '{print $2}'`
			do
				MCAL1=`echo $MCAL | sed "s/_RASSOC//" | sed "s/_MASSOC//"`
				TIME_MATCH=`grep "^$ACTION $MCAL1 " $TMP_DIR/rm_deltat | sort -u | sed "s/^$ACTION $MCAL1 //"`
				cat $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1 |\
 awk '{ 
	if ( $1 == "action" && $2 == action && change == 0) {change=1; print $0} 
	else if ( change == 0) {print $0}
	else if ( change == 1 && $1 == "select" && ($4 == mcal"_RASSOC" || $4 == mcal"_MASSOC" || $4 == mcal )) {change=0; print $0,timematch} 
	else {print $0}
      }' change=0 action=$ACTION mcal=$MCAL1 timematch="$TIME_MATCH" > $TMP_DIR/raw2master.RLS1
				mv $TMP_DIR/raw2master.RLS1 $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1
			done
		done
		echo "    ... done. Hit return:"
	else
		echo "    b) No time-match rules found in $ASSOC. Hit return:"
	fi
	read input

	echo "    c) Check for possible DUPLICATE entries:"
	rm -f $TMP_DIR/ca_duplicate
	cat $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1 | grep DUPLICATE > $TMP_DIR/ca_duplicate
	if [ -s $TMP_DIR/ca_duplicate ]
	then
		echo "Detected:"
		echo "================="
		cat $TMP_DIR/ca_duplicate
		echo "================="
		echo " Please check these cases in $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1 (grep for 'CHECK DUPLICATE MJD_OBS') 
and make sure that the time constraints become unique. Hit return:"
		read input
	fi	

	echo "    d) Check for missing MJD-OBS statements for static calibrations:"

	if [ -s $TMP_DIR/rm_gencalibs ]
	then
		MJD_CONSTRAINT=NO
		for GC in `cat $TMP_DIR/rm_gencalibs`
		do
			CHECK_MJD=`grep $GC $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1 | grep -v MJD-OBS | tail -1`
			if [ "Q$CHECK_MJD" != Q ]
			then
				sed -i -e "/$GC/s|^.*|& !!!CHECK MISSING MJD_CONSTRAINT!!!|"  $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1
				echo "  $GC has time constraint missing, marked as 'CHECK MISSING MJD_CONSTRAINT'!"
				MJD_CONSTRAINT=YES
			fi
		done

		if [ Q$MJD_CONSTRAINT = QNO ]
		then
			echo "    ... ok, no issue found. Hit return:"
		else
			echo "    ... to be fixed in the original files. Hit return:"	
		fi
	else
		echo "   ... no gencalibs found. Hit return:"
	fi
	read input

# =========================================================================
# 2.4 end
# =========================================================================

	echo "   ... done.

The output file is $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS1. It probably needs more edits, see log.
Save it as $TMP_DIR/${DFO_INSTRUMENT}_raw2master.RLS. Next, call '$TOOL_NAME -C'.

Get more information how to edit this file with '$TOOL_NAME -I' 
Hit return:"
	read input
	exit
fi

# =========================================================================
# 3. rulesManager
# 3.1 find MODE
# =========================================================================

if [ $RLSMNG = YES ]
then
	case $MODE in
 	 READ|WRITE|VALIDATE|CHECK|REMOVE|LIST ) : ;;
 	 * ) echo "You must enter a valid mode (one of READ|WRITE|VALIDATE|CHECK|REMOVE|LIST)." ; exit ;;
	esac

# =========================================================================
# 3.2 mode=LIST
# =========================================================================

	if [ $MODE = LIST ]
	then
		case $ALL_INS in
		 "YES" ) echo "Existing CALSELECTOR rule file(s) for all instruments (identified by \$DFO_FILE_NAME):
" ;;
		 "NO"  ) echo "Existing CALSELECTOR rule file(s) for ${DFO_INSTRUMENT}: 
" ;;
		esac

               	cat > $TMP_DIR/cs_read <<EOT
select
	convert( char(6),dp_prefix) as "DFO_FILE_NAME",
        convert(varchar(10),rule_start,102) as "rule_start",
        convert(varchar(10),rule_stop,102) as "rule_stop",
	convert(varchar(35),filename) as "name",
	--convert(varchar(2),raw2master_allowed) as "r2m_allowed?",
	--convert( char(10), version) as "version",
	convert( char(200), description) as "description"
from
        bookkeeping..oca_rules
	order by rule_start
go
EOT
		rm -f $TMP_DIR/cs_rules_ALL
       		#$RULESMANAGER --conf-file=$CALSELECTOR_CONFIG $MESSAGES --mode=LIST --dp-prefix=${DFO_FILE_NAME} | sed "s/^.*filename/&			/" | sed "s/^.*description/&		/"
# better:
		isql -U`grep "^spring.datasource.username" $CALSELECTOR_CONFIG | sed "s/^.*=//"` -P`grep "^spring.datasource.password" $CALSELECTOR_CONFIG | sed "s/^.*=//"` -S$ARCH_SERVER -w999 -i $TMP_DIR/cs_read |\
 sed "s/name     /rule_name/" |\
 sed "s/description                                                                                                              /description/" |\
 sed "s/\./-/" | sed "s/\./-/" | sed "s/\./-/" | sed "s/\./-/" | sed "s/------------------------------------------------------------------------------------------------------//" \
> $TMP_DIR/cs_rules_ALL

		if [ $ALL_INS = YES ]
		then
       	        	cat $TMP_DIR/cs_rules_ALL | head -2
       	        	cat $TMP_DIR/cs_rules_ALL | sed "1,2 d" | sort -k1,1 | more
			echo "Find this list in $TMP_DIR/cs_rules_ALL."
			exit
		fi
		cat $TMP_DIR/cs_rules_ALL | head -2
		cat $TMP_DIR/cs_rules_ALL | grep $DFO_FILE_NAME
		exit
        fi

# =========================================================================
# 3.3 mode=READ: download the OCA rule (specified by date)
# =========================================================================

	if [ $MODE = READ ]
	then
		mkdir $RULES_DIR 2>/dev/null
		cd $TMP_DIR
	        if [ Q$REF_DATE = Q ]
	        then
			rm -f $TMP_DIR/rulesManager.txt

			$TOOL_NAME -m LIST | tee -a $TMP_DIR/rulesManager.txt
	                echo "Select the $DFO_FILE_NAME OCA rule file (column #4) you want to download:"
			read RLS_NAME
			if [ Q$RLS_NAME = Q ]
			then
				echo "No rule specified. Exit."
				exit
			else
				REF_DATE=`grep " $RLS_NAME " $TMP_DIR/rulesManager.txt | awk '{print $2}'`
			fi
		else
			echo "Looking for OCA rule applicable to $REF_DATE ..."
       			RLS_NAME=`$RULESMANAGER --conf-file=$CALSELECTOR_CONFIG $MESSAGES --dp-prefix=${DFO_FILE_NAME} --mode=READ --date=$REF_DATE | grep "Rules written to" | awk '{print $4}'`
			if [ Q$RLS_NAME = Q ]
			then
				echo "... nothing found. Check availability with '$TOOL_NAME -m LIST'."
				exit
			else	
				echo "... found: $RLS_NAME."
				echo ""
			fi
		fi
	 	echo "Download OCA rule $RLS_NAME ... (hit return):"
		read input

       		$RULESMANAGER --conf-file=$CALSELECTOR_CONFIG $MESSAGES --dp-prefix=${DFO_FILE_NAME} --mode=READ --date=$REF_DATE | grep "Rules written to" 
		CHECK_EMPTY=`grep "^null" $TMP_DIR/${RLS_NAME} | head -1 | awk '{print $1}'`
		if [ Q$CHECK_EMPTY = Qnull ]
		then
			echo "None found."
			exit
		else
			rm -f $RULES_DIR/$RLS_NAME
			mv $TMP_DIR/${RLS_NAME} $RULES_DIR/
        		echo "Find output as $RULES_DIR/$RLS_NAME ."
		fi
		exit
	fi

# =========================================================================
# 3.4 mode=CHECK
# =========================================================================

	if [ $MODE = CHECK ]
	then
		echo "This mode can be used to check that all keywords in the $DFO_FILE_NAME rules are in the ISTs.

Since the tool has no way to tell which keywords should go to the raw table 
and which to the product table, it is perfectly fine if the following keywords 
are reported to be missing:

PRO.CATG from obs_metadata..<instr>_public
DPR.CATG|TYPE|TECH from qc_metadata..m_<instr>_public2

If other keywords are claimed to be missing, this is significant and could either mean
a typo in your OCA rules (like e.g. SEP.ARM instead of SEQ.ARM) or an issue in the ISTs
which you would then report to $DBCM_MAIL. 

Hit return:"
		read input
 
	        if [ Q$REF_DATE != Q ]
	        then
	                echo "Check that all keywords in the $DFO_FILE_NAME rules applicable to date $REF_DATE are in the IST ..."
	        else
	                REF_DATE=`date -u +%Y-%m-%d`
	                echo "Check that all keywords in the $DFO_FILE_NAME rules applicable to current date $REF_DATE are in the IST ..."
	        fi

       		$RULESMANAGER --conf-file=$CALSELECTOR_CONFIG $MESSAGES --dp-prefix=${DFO_FILE_NAME} --mode=CHECK --date=$REF_DATE | grep "Missing keyword"

		echo "
If you are in doubt about the currently existing rules, call '$TOOL_NAME -m LIST'."
        	exit
	fi

# =========================================================================
# 3.5 mode=VALIDATE (WRITE with $VALIDATE only)
# =========================================================================

	if [ $MODE = VALIDATE ]
	then
		if [ Q$RLS_FILE = Q ]
		then
			echo "***ERROR: no \$RLS_FILE defined. The call is:
$TOOL_NAME -m WRITE -R <pathname to RLS file>"
			exit -1
		fi

		if [ ! -s ${RLS_FILE} ]
		then
			echo "***ERROR: ${RLS_FILE} not existing. "
			exit -1
		fi

		RLS_FILE1=`basename $RLS_FILE`
		RLS_DIRNAME=`dirname $RLS_FILE`
		VAL_DATE=`basename $RLS_DIRNAME`
		STOP_DATE=2999-12-31

		echo "
This is the workflow to check the syntax of an OCA rules file ("validate"), without a real ingestion.
The tool will check for syntax errors of the specified calSelector OCA rules file, to prepare for ingestion.
The syntax check is passed successfully if you find the log entry \"finished validateAndInitProperties\".
Ignore when the tool asks you \"Do you really want to save ...\" (will be auto-answered 'N').

Hit return to start the syntax check of $RLS_FILE:"
		read input

 		rm -f rulesmanager.log
		$RULESMANAGER --conf-file=$CALSELECTOR_CONFIG $MESSAGES --dp-prefix=${DFO_FILE_NAME} --mode=WRITE --start-date=$VAL_DATE --stop-date=$STOP_DATE --oca-rules-file=$RLS_FILE $VALIDATE <<EOT
N

EOT
		echo "
N
Syntax check finished."
		exit
	fi

# =========================================================================
# 3.6 mode=WRITE
# =========================================================================

	if [ $MODE = WRITE ]
	then
		if [ Q$RLS_FILE = Q ]
		then
			echo "***ERROR: no \$RLS_FILE defined. The call is:
$TOOL_NAME -m WRITE -R <pathname to RLS file>"
			exit -1
		fi

		if [ ! -s ${RLS_FILE} ]
		then
			echo "***ERROR: ${RLS_FILE} not existing. "
			exit -1
		fi

		RLS_FILE1=`basename $RLS_FILE`
		RLS_DIRNAME=`dirname $RLS_FILE`
		VAL_DATE=`basename $RLS_DIRNAME`

		echo "
We upload the rules file into the calSelector database. This will be done in 5 steps. 
If you want to check the syntax only ("validate"), without a real ingestion, call the tool with '-m VALIDATE'.
Before real ingestion the tool will ask you \"Do you really want to save ...\".

1. Confirm that this rules file is a valid CALSELECTOR RLS file 
(conditions: it precompiled fine, and you have checked its calib map) (Y/N) (N):"
		read CONFIRM
		
		if [ Q$CONFIRM != QY ]
		then
			echo "Think about it. "
			exit
		fi

		echo "2. Start-date and stop-date of rule validity:
- the start-date will be derived from the pathname;
- enter the stop-date (in case of multiple versions, make sure to have no overlap and no gaps; 
  the last of a previous version is the first day of the new version;
  default: 2999-12-31):"
		read STOP_DATE

		if [ Q$STOP_DATE = Q ]
		then
			STOP_DATE=2999-12-31
		fi

		CHECK_FORMAT=`echo $STOP_DATE | wc -c`
		if [ $CHECK_FORMAT != 11 ]
		then
			echo "***ERROR: stop-date $STOP_DATE doesn't seem to have the proper format. Exit."
			exit -1
		fi
			
		echo "3. You can optionally enter a short description for this rule file
   (e.g. \"applicable after CCD upgrade\" or \"includes new TFLATs introduced in P88\").
   It will appear on the time coverage overview. Do you want a comment (Y/N) (N)?"
		read COMM_YN

		rm -f $TMP_DIR/calsel_comment
		if [ Q$COMM_YN != QY ]
		then
			echo "No comment provided."
			echo ""
			echo "start-date: $VAL_DATE; stop-date $STOP_DATE"
		else
			echo "   Now enter a comment (one line only):"
			echo "<your comment (one line only)>" > $TMP_DIR/calsel_comment
			xterm -T "Enter your comment (one line only); use $DFO_EDITOR" -geom 100x20+1600+10 -e $DFO_EDITOR $TMP_DIR/calsel_comment
			echo ""
			echo "start-date: $VAL_DATE; stop-date: $STOP_DATE; comment: \"`cat $TMP_DIR/calsel_comment`\""
		fi

		if [ -s $TMP_DIR/calsel_comment ]
		then
			OCA_RULES_DESCRIPTION="`cat $TMP_DIR/calsel_comment`"
		else
			OCA_RULES_DESCRIPTION=" "
		fi

		echo "Hit return to continue:"
		read input

		echo "4. Decision about virtual products: 
With the successful ingestion of the rules file, a process can be triggered to
create all virtual products (datasets) for the corresponding validity period. These will
always be needed but it could be that the virtual products already exist from a previous 
ingestion and can still be applied. You would then want to keep them (their creation might
take minutes or hours, depending on the number of datasets affected). If you have modified 
the grouping part of the OCA rules, or their validity period, you likely need to create new
virtual products. If you ingest for the first time, *always create virtual products*.

Do you want to create new virtual products (Y/N) (N):"
		read CREATE_VPRODUCTS
		if [ Q$CREATE_VPRODUCTS != QY ]
		then
			CREATE_VPRODUCTS=N
			echo "   No new virtual products will be created."
		else
			echo "   New virtual products will be created."
		fi

		echo "
5. Now we start the ingestion.
Note:
- there will be a syntax check which is more stringent than the test done 
  by the precompiler within the migration. 
- there will also be a test if the new ruleset overlaps with any pre-existing rule.

Once successfully passed, the tool will ask you \"Do you really want to save ...\",
with \"start-date\" derived from the pathname, and \"stop-date\" just entered."

		if [ $CREATE_VPRODUCTS = Y ]
		then
			echo "
With the successful ingestion of the rules file, a process is triggered to
create all virtual products (datasets) for the corresponding validity period. Depending
on the number of datasets affected, this may take minutes or hours. Please 
DO NOT INTERRUPT this process, and DO NOT attempt to continue the workflow before 
the virtual products are created. The tool will
come back with a message saying \" Added ... files\"."
		fi

		echo "
The log file is available under `pwd`/rulesmanager.log.
You can watch it calling 'tail -f `pwd`/rulesmanager.log'.

Hit return:"
		read input

		if [ $CREATE_VPRODUCTS = Y ]
		then
			VPRODUCTS_CREATE="--update-virtual-products=true"
		else
			VPRODUCTS_CREATE=""
		fi

 		rm -f rulesmanager.log
		$RULESMANAGER --conf-file=$CALSELECTOR_CONFIG $MESSAGES --dp-prefix=${DFO_FILE_NAME} --mode=WRITE --start-date=$VAL_DATE --stop-date=$STOP_DATE --oca-rules-file=$RLS_FILE --oca-rules-description="$OCA_RULES_DESCRIPTION" $VPRODUCTS_CREATE $VALIDATE

        	if [ $? = 0 ]
	        then
			mv rulesmanager.log $TMP_DIR/
			grep -v "Could not find enough" $TMP_DIR/rulesmanager.log > $TMP_DIR/calselector.log1
			mv $TMP_DIR/calselector.log1 $TMP_DIR/rulesmanager.log
			echo "
Was the ingestion successful (Y/N) or did you abort (A)?"
			read SUCCESS_YN

			if [ Q$SUCCESS_YN = QY ]
			then
                		echo "Ingestion of $RLS_FILE1 successful.
The log file is in $TMP_DIR/calselector.log.
The following rules now exist for $DFO_FILE_NAME:
"
                		$TOOL_NAME -m LIST
			else
				echo "OK, think about it."
			fi
        	else
               		echo "An error occurred."
        	fi
		exit
	fi

# =========================================================================
# 3.7 mode=REMOVE
# =========================================================================

	if [ $MODE = REMOVE ]
	then
		echo "You want to remove a rules file from the calSelector database. 
"

               	$TOOL_NAME -m LIST 

		echo "
Enter the rule_name to be removed:"

		read RLS_NAME
		echo ""
		if [ Q$RLS_NAME = Q ]
		then
			echo "You have to enter a rule_name."
			exit
		fi

               	cat > $TMP_DIR/cs_read <<EOT
select
        convert(varchar(10),rule_start,102) as "rule_start",
        convert(varchar(10),rule_stop,102) as "rule_stop",
	convert(varchar(35),filename) as "name",
	convert(char(60), description) as "description"
from
        bookkeeping..oca_rules
where
	filename like "${RLS_NAME}%"
go
EOT
		rm -f $TMP_DIR/cs_rules_ALL
		isql -U`grep "^spring.datasource.username" $CALSELECTOR_CONFIG | sed "s/^.*=//"` -P`grep "^spring.datasource.password" $CALSELECTOR_CONFIG | sed "s/^.*=//"` -S$ARCH_SERVER -w999 -i $TMP_DIR/cs_read |\
 grep $RLS_NAME |\
 sed "s/\./-/" | sed "s/\./-/" | sed "s/\./-/" | sed "s/\./-/" \
 > $TMP_DIR/cs_rls_info

		if [ ! -s $TMP_DIR/cs_rls_info ]
		then
			echo "***ERROR: something went wrong with the rule_name $RLS_NAME. Exit."
			exit
		fi
		echo "rule_start rule_stop  rule_name"
		echo "======================================================"
		cat $TMP_DIR/cs_rls_info | awk '{print $1,$2,$3}'

		VAL_DATE=`cat $TMP_DIR/cs_rls_info | awk '{print $1}'`
		END_DATE=`cat $TMP_DIR/cs_rls_info | awk '{print $2}'`

		echo "
For safety, we download the rule before deletion ..."

		PREV_PWD=$PWD
		cd $TMP_DIR
       		$RULESMANAGER --conf-file=$CALSELECTOR_CONFIG $MESSAGES --dp-prefix=${DFO_FILE_NAME} --mode=READ --date=$VAL_DATE | grep -v "Rules written to"

		mv $TMP_DIR/${RLS_NAME} $RULES_DIR/
		cd $PREV_PWD

       		echo "   ... find output as $RULES_DIR/$RLS_NAME ."

		echo "
Decision about virtual products: 
Virtual products (datasets) have probably been created before, for the validity period of 
the OCA rules. If you ingest a new set of OCA rules, the old virtual products might still 
be applicable, and you may want to decide to not delete them. This is the case when no 
changes are applied to the grouping part of the OCA rules. If you delete the virtual 
products, you can recreate them later with the ingestion of the new OCA rules.

Do you want to delete the existing virtual products for the ruleset $RLS_NAME (Y/N) (N):"
		read DELETE_VPRODUCTS

		if [ Q$DELETE_VPRODUCTS != QY ]
		then
			DELETE_VPRODUCTS=N
		fi

# the flag is called --update-virtual-products but means --delete-virtual-products!
		if [ $DELETE_VPRODUCTS = Y ]
		then
			VPRODUCTS_DELETE="--update-virtual-products=true"
		else
			VPRODUCTS_DELETE=""
		fi

		case $DELETE_VPRODUCTS in
	  	 "Y" ) echo "Now we continue with REMOVE, virtual products will be deleted." ;;
		 "N" ) echo "Now we continue with REMOVE, virtual products will not be deleted." ;;
		esac
		echo "Hit return:"
		read input

		cat > $TMP_DIR/cs_call <<EOT
$RULESMANAGER --conf-file=$CALSELECTOR_CONFIG $MESSAGES --dp-prefix=${DFO_FILE_NAME} --mode=REMOVE --start-date=$VAL_DATE --stop-date=$END_DATE $VPRODUCTS_DELETE
echo "Hit return:"
read input
EOT
		chmod u+x $TMP_DIR/cs_call
		xterm -T "CALSELMANAGER log watcher (REMOVE)" -geom 200x20+1600+10 -e $TMP_DIR/cs_call

	        if [ $? = 0 ]
       		then
                	echo "The following rules now exist for $DFO_FILE_NAME:
"
                	$TOOL_NAME -m LIST
        	else
                	echo "An error occurred."
        	fi

		echo "
If you actually wanted to replace a rule by a modified one, you can now call 
$TOOL_NAME -m WRITE -R \$DFO_CONFIG_DIR/CALSELECTOR/$VAL_DATE/${DFO_INSTRUMENT}_raw2master.${VAL_DATE}.RLS
(this name is guessed)."
        	exit
	fi
fi

exit

