#!/usr/bin/perl
#
# Lisa Germany
# 16th June 2002
# Updated: 
# 23rd June 2003      Lisa Germany      Expanded to a general program to do
#                                       BADPIX masks for WFI/SUSI/EFOSC/EMMI
#
############################################################################
#
# INPUT:
# Needs 3 files containing - list of bias images
#                          - list of flats with high counts
#                          - list of flats with low counts
#
# DESCRIPTION:
# - Processes raw images to make 8 badpix files - one for each chip
# - Gzips images ready for saving on tape
# - Tars and Gzips badpix files ready to go onto the web
#
############################################################################


if ( $#ARGV != 1 ) { die "\n\nUsage: mkBADPIX.pl instrument date\n\n  Instrument is one of: WFI\n                        EMMI\n                        EFOSC\n                        SUSI\n\n"}

$instrument = $ARGV[0];
$date = $ARGV[1];



##############################################################################
####                                                                      ####
####  (1) Make backup of image in directory Raw/                          ####
####                                                                      ####
##############################################################################

if ( ! -e "Raw"){
    system ( "mkdir Raw");
    system ( "cp *.fits Raw/");
}


##############################################################################
####                                                                      ####
####  (2) IF WFI:  Run esowfi on all images                               ####
####                                                                      ####
##############################################################################

if ($instrument eq "WFI"){

    unlink "login.cl" if -e "login.cl";
    system( "cp \/data\/BADPIX\/source\/login.cl \.\/login.cl" );
    open ( F1, ">>login.cl" ) || die "Could not open login.cl\n";
    print( F1 "esowfi\n");
    print( F1 "unlearn esohdr\n");
    print( F1 "esohdr WFI*.fits queryty=no \n");
    print( F1 "logout\n" ); 
    close ( F1 );
    system( "cl" );
}


##############################################################################
####                                                                      ####
####  (3) Determine biases, flatshigh and flatslow                        ####
####      Use mscstat on chip 2 to determine mean                         ####
####      Write out to appropriate files flatshigh flatslow               ####
####                                                                      ####
##############################################################################

unlink "tmpflatcounts";  #delete file tmpflatcounts if program has been run before
unlink "tmpfiletype";  #delete file tmpfiletype if program has been run before
unlink "flatshigh" if -e "flatshigh";  #delete file flatshigh if program has been run before
unlink "flatslow" if -e "flatslow";    #delete file flatslow if program has been run before
unlink "darks" if -e "darks";  #delete file darks if program has been run before
unlink "flats" if -e "flats";  #delete file flats if program has been run before

####### WFI #######  WORKS!!



if ($instrument eq "WFI"){

    
    system( "ls -1 WFI_Dark*.fits > darks" );
    system( "ls -1 WFI_FlatDomeIma*.fits > flats" );
    @allflats = `ls -1 WFI_FlatDomeIma*.fits`;
    chomp @allflats;
    
    unlink "login.cl" if -e "login.cl";
    system( "cp \/data\/BADPIX\/source\/login.cl \.\/login.cl" );
    open ( F1, ">>login.cl" ) || die "Could not open login.cl\n";
    
    foreach $flat (@allflats){
	print( F1 "mscstat ${flat}[2] fields=image,mean format=no >> tmpflatcounts\n");
    }
    
    
    print( F1 "logout\n" ); 
    close ( F1 );
    system( "cl" );
    
    open (F1, "<tmpflatcounts");
    
    open (FHIGH, ">>flatshigh");
    open (FLOW, ">>flatslow");
    
    while (<F1>){
	($file2, $mean) = split(/\s+/, $_);
	
	$_ = $file2;                 # get rid of the [2][im2] extension needed 
	if (/(\S+)\[2\]\[im2\]/){    # to do the mscstat
	    $file=$1;
	}
	
	if ($mean > 15000){          # write flats with > 15000 cts to flatshigh
	    print( FHIGH "$file\n");
	}else{
	    print( FLOW "$file\n");  # write flats with < 15000 cts to flatslow
	}
    }
    
    close (F1);
    close (FHIGH);
    close (FLOW);

}



####### EFOSC #######    WORKS!!

if ($instrument eq "EFOSC"){


    system( "ls -1 EFOSC_Dark*.fits > darks" );
    system( "ls -1 EFOSC_FlatIma*.fits > flats" );
    @allflats = `ls -1 EFOSC_FlatIma*.fits`;
    chomp @allflats;

    unlink "login.cl" if -e "login.cl";
    system( "cp \/data\/BADPIX\/source\/login.cl \.\/login.cl" );
    open ( F1, ">>login.cl" ) || die "Could not open login.cl\n";

    foreach $flat (@allflats){
	print( F1 "imstat $flat fields=image,mean format=no >> tmpflatcounts\n");
    }
    
    print( F1 "logout\n" ); 
    close ( F1 );
    system( "cl" );

    open (F1, "<tmpflatcounts");
    open (FHIGH, ">>flatshigh");
    open (FLOW, ">>flatslow");
    
    while (<F1>){
	($file, $mean) = split(/\s+/, $_);
	
	if ($mean > 15000){          # write flats with > 15000 cts to flatshigh
	    print( FHIGH "$file\n");
	}else{
	    print( FLOW "$file\n");  # write flats with < 15000 cts to flatslow
	}
    }
    
    close (F1);
    close (FHIGH);
    close (FLOW);

}


####### SUSI #######   WORKS!!!

if ($instrument eq "SUSI"){

    
    system( "cp \/data\/BADPIX\/source\/table.conv \.\/table.conv" );


    @allimages = `ls -1 ONTT*.fits`;
    chomp @allimages;
    

    system( "cp \/data\/BADPIX\/source\/login.cl \.\/login.cl" );
    open ( F1, ">>login.cl" ) || die "Could not open login.cl\n";
    
    foreach $image (@allimages){
	
	system( "hierarch28 $image" );  # Replace ESO keyword for image type
	print( F1 "hselect $image field=ARCFILE,DPRTYPE expr=yes >> tmpfiletype\n");
    }
    print( F1 "logout\n" ); 
    close ( F1 );
    system( "cl" );
    
    
    
    open (F1, "<tmpfiletype");	# Sort out what files are biases and what are flats
    open (FLAT, ">>flats");
    open (BIAS, ">>darks");
    
    while (<F1>){
	($image, $type) = split(/\s+/, $_);
	
	if ($type eq "LAMPFLAT"){
	    print( FLAT "$image\n");
	}elsif ($type eq "BIAS"){
	    print( BIAS "$image\n");
	}else{
	    die "$image is neither a flat or a bias\n";
	}
    }
    close (F1);
    close (FLAT);
    close (BIAS);
    
#### Now determine the counts in each flat image
    
    system( "cp \/data\/BADPIX\/source\/login.cl \.\/login.cl" );
    open ( F1, ">>login.cl" ) || die "Could not open login.cl\n";
    print( F1 "imstat \@flats fields=image,mean format=no >> tmpflatcounts\n");
    print( F1 "logout\n" ); 
    close ( F1 );
    system( "cl" );
    
    
    
    open (F1, "<tmpflatcounts");
    open (FHIGH, ">>flatshigh");
    open (FLOW, ">>flatslow");
    
    
    while (<F1>){
	($file, $mean) = split(/\s+/, $_);
	
	if ($mean > 15000){          # write flats with > 15000 cts to flatshigh
	    print( FHIGH "$file\n");
	}else{
	    print( FLOW "$file\n");  # write flats with < 15000 cts to flatslow
	}
    }
    
    close (F1);
    close (FHIGH);
    close (FLOW);
}




####### EMMI #######  WORKS!!!

if ($instrument eq "EMMI"){


    system( "cp \/data\/BADPIX\/source\/table.conv \.\/table.conv" );
    
    @allimages = `ls -1 ONTT*.fits`;
    chomp @allimages;


    system( "cp \/data\/BADPIX\/source\/login.cl \.\/login.cl" );
    open ( F1, ">>login.cl" ) || die "Could not open login.cl\n";
    
    foreach $image (@allimages){

	print "Converting multi-extension fits file to a normal fits file with inemmi.prg.  Please wait...\n";

	`inmidas -p 66 -j "@@ inemmi $image emmi.bdf ; bye"`;  #combine all chips into one image
	`inmidas -p 66 -j "outdisk\/fits emmi.bdf $image ; bye"`;  #rename output image
	
	system( "hierarch28 $image" );  # Replace ESO keyword for image type
	print( F1 "hselect ${image} field=ARCFILE,DPRTYPE expr=yes >> tmpfiletype\n");
    }

    print( F1 "logout\n" ); 
    close ( F1 );
    system( "cl" );
    
    
    open (F1, "<tmpfiletype");	# Sort out what files are biases and what are flats
    open (FLAT, ">>flats");
    open (BIAS, ">>darks");
    
    while (<F1>){
	($image, $type) = split(/\s+/, $_);

	if ($type eq "LAMPFLAT"){
	    print( FLAT "$image\n");
	}elsif ($type eq "BIAS"){
	    print( BIAS "$image\n");
	}else{
	    die "$image is neither a flat or a bias\n";
	}
    }
    close (F1);
    close (FLAT);
    close (BIAS);

#### Now determine the counts in each flat image

    system( "cp \/data\/BADPIX\/source\/login.cl \.\/login.cl" );
    open ( F1, ">>login.cl" ) || die "Could not open login.cl\n";
    open ( F2, "<flats") || die "Could not open flats\n";
    
    while (<F2>){
	$flat = $_;
	chomp $flat;	   
	print( F1 "imstat ${flat} fields=image,mean format=no >> tmpflatcounts\n");
    }
    
    print( F1 "logout\n" ); 
    close ( F2 );
    close ( F1 );
    system( "cl" );


    open (F1, "<tmpflatcounts");
    open (FHIGH, ">>flatshigh");
    open (FLOW, ">>flatslow");
    
    
    while (<F1>){
	($file, $mean) = split(/\s+/, $_);
	
	if ($mean > 15000){          # write flats with > 15000 cts to flatshigh
	    print( FHIGH "$file\n");
	}else{
	    print( FLOW "$file\n");  # write flats with < 15000 cts to flatslow
	}
    }
    
    close (F1);
    close (FHIGH);
    close (FLOW);
}

system ("rm *.bdf");


##NOW TO MAKE THIS BIT WORK!!###

##############################################################################
####                                                                      ####
####  (4) Overscan and Trim all images                                    ####
####      Make the master BIAS frame                                      ####
####      Bias subtract FLAT images                                       ####
####      Make FLATHIGH and FLATLOW combined images                       ####
####      Divide FLATLOW/FLATHIGH to make master frame                    ####
####                                                                      ####
##############################################################################

unlink "login.cl" if -e "login.cl";
system( "cp \/data\/BADPIX\/source\/login.cl \.\/login.cl" );
open ( F1, ">>login.cl" ) || die "Could not open login.cl\n";


if ($instrument eq "WFI"){    # if WFI overscan subtract and trim the images
    print( F1 "ccdproc *.fits output=\"\" ccdtype=\"\" noproc=no oversca=yes trim=yes fixpix=no zerocor=no darkcor=no flatcor=no sflatcor=no biassec=\!biassec trimsec=\!trimsec fixfile=badpix zero=BIAS flat=SKYFLAT minrepl=1. interac=no functio=legendre order=1 sample=* naverag=1 niterat=1 low_rej=3. high_re=3. grow=0. \n");
}


print( F1 "zerocombine \@darks output=BIAS.fits ccdtype=\"\" process=no\n");

print( F1 "ccdproc \@flats output=\"\" ccdtype=\"\" noproc=no oversca=no trim=no fixpix=no zerocor=yes darkcor=no flatcor=no sflatcor=no biassec=\"\" trimsec=\"\" merge=yes fixfile=badpix zero=BIAS flat=SKYFLAT minrepl=1. interac=no functio=legendre order=1 sample=* naverag=1 niterat=1 low_rej=3. high_re=3. grow=0. \n");

print( F1 "flatcombine \@flatshigh output=FLATHIGH ccdtype=\"\" process=no subsets=no \n");
print( F1 "flatcombine \@flatslow output=FLATLOW ccdtype=\"\" process=no subsets=no \n");

print( F1 "mscarith FLATLOW.fits / FLATHIGH.fits FLATLHRATIO.fits \n");

if ($instrument eq "WFI"){
    print( F1 "mscsplit FLATLHRATIO.fits output=FLATLHRATIO \n");
}

print( F1 "unlearn ccdmask \n");


if ($instrument eq "WFI"){
    for ($i=1;$i<=8;$i++) {
	print( F1 "ccdmask FLATLHRATIO\_$i.fits BADPIX5$i.pl lsigma=20 hsigma=20 \n");
    }
}else{
    print( F1 "ccdmask FLATLHRATIO.fits BADPIX\_${instrument}_${date}.pl lsigma=20 hsigma=20 \n");
}

print( F1 "logout\n" );
close ( F1 );
system( "cl" );


##############################################################################
####                                                                      ####
####  (3) Backup and Cleanup                                              ####
####                                                                      ####
##############################################################################

#print "Tarring up the BADPIX files...\n";

if ($instrument eq "WFI"){
    system "tar cvf BADPIX\_WFI\_${date}.tar BADPIX*.pl\n";
    print "Gzipping BADPIX\_WFI\_${date}.tar...\n";
    system "gzip BADPIX\_WFI\_${date}.tar \n";
}

#unlink "login.cl" if -e "login.cl";
#unlink "logfile" if -e "logfile";
#unlink "WFI*.fits";

#print "\n Gzipping original images to save to tape.\n";
#print "This is going to take a while!!\n";
#system "gzip Raw/*.fits\n";
#system "gzip BIAS.fits\n";
#system "gzip FLATHIGH.fits\n";
#system "gzip FLATLOW.fits\n";