#!/usr/bin/perl
## monitor_dust.pl
###
use Class::Struct;
use CGI qw(param);
use Time::Local;
use strict;


my @mint = ("16","17","18","19","20","21", "22");

my $remote_dir = "/data/ftp/pub/DUST";
my $gp5_dir = "/data/web/mirror/www.ssd.noaa.gov/htdocs/PS/FIRE/DUST/PNGFILES_DUST";
my $Dist1='oper\@10.144.3.38';
my $gp5= 'sabweb\@10.144.3.87';
my $ftpnon='oper\@10.144.3.43';

my $julday=0;
my $year = 0;
my $leapday = 0;
my @date = ();
my $i =0;
my $mm = 0;
my $dd = 0;
my $getmint = ' ';
my $begin_time = 16;
my $end_time = 22;
my $outputFile = "Dust.log";
my $outlog_dir = "/data/Dust/logs";
my $output_dir = "/data/Dust/output/";
my $ProdId_name = ' ';
my $ProdId_name_hr = ' ';
my $Id_name = ' ';
my $remote_dir = '/data/ftp/pub/DUST/';
my $remote_dir_satepsnon = '/data/ftp/pub/DUST/output/';
my $output_number = 0;
my $emailList = ' ';
my $cclist = ' ';
my $num_satepsnon_grib = 0;
my $num_satepsnon_nc = 0;
my $num_dist1_grib = 0;
my $num_dist1_nc = 0;
my $num_gp5_png = 0;
my $output_table_dir = "/data/Dust/table";
my $output_TableFile = "Dust_Table.txt";

$year=$ARGV[0];
$julday=$ARGV[1];
$mm=$ARGV[2];
$dd=$ARGV[3];

my %output_prefix = qw (
  grd MYDdust.aod_conc.v6.3.4.
  nc  MODIS_DUST.
  jpg MYDdustconc.v6.3.4.
  gp5jpg MYDdust.
);
my %output_suffix = qw (
 grd .grib
 nc  .nc.gz
 jpg .jpg
);

my %output_dir = qw (
  grd  gggg  
);

if (length($year) eq 0 ) {

$leapday = 1;

 print " before $leapday  \n";

($year, $mm, $dd, $julday) = getDate($leapday);

}
print " time=$year, $mm, $dd, $julday  \n";
open(MFILE, ">> $outlog_dir/$outputFile") || die "Can not open  $outputFile for writing: $!";

system("cd $output_dir/$year$mm$dd/AOD/");

my $file_number = `grep  "Fatal error" $outlog_dir/run_Dust_Modis_$year$mm$dd.log  | wc -l`;

print "number $file_number \n";

if ($file_number > 1 ) {
print MFILE "$year$mm$dd  DUST processing is  Failed !!!!!  \n";
print MFILE "-----------------------------------------------------------------------------\n";
} else {
print MFILE "$year$mm$dd  DUST processing is  Completed  !  \n";
print MFILE "-----------------------------------------------------------------------------\n";

$Id_name = "$year$mm$dd.";

open(DUSTFILE, ">> $output_table_dir/$output_TableFile") || die "Can not open  $output_TableFile for writing: $!";
writehead();


#### check data on Dist1  NetCDF #####
$output_number = 0;

for ($i=$begin_time; $i<=$end_time; $i++) {
$ProdId_name = $Id_name.$i;
$ProdId_name_hr ="$Id_name.hr$i";
#### check local output data  ########
my $local_dir = "$output_dir$year$mm$dd/GRIB";
my $file_name = "$local_dir/MYDdust.aod_conc.v6.3.4.$year$mm$dd.hr$i.grib";
print "file_name = $file_name =\n";
if( -e $file_name) {
    $output_number = $output_number + 1;
my $getLocal_Grib = getProduc($year,$mm,$dd,$local_dir,$file_name,$ProdId_name);
print " getLocal_Grib = $getLocal_Grib\n";




#### check data on Dist1  NetCDF #####
my $getDist1Net = ' ';
 $getDist1Net = getDist1($year,$mm,$dd,$Dist1,$remote_dir,$output_prefix{nc},$output_suffix{nc},$ProdId_name);
 print " getDist1Net = $getDist1Net\n";
 if ( $getDist1Net == 'Failure' ) {
####   print MFILE "|In Dist1     |MODIS_DUST.$ProdId_name.nc.gz|NO DUST Data in This Time|\n";
 }
 else {
   print MFILE "|In Dist1 | MODIS_DUST.$ProdId_name.nc.gz | $getDist1Net |\n";
#   $output_number = $output_number + 1;
   $num_dist1_nc = $num_dist1_nc + 1;
 }
#### check NETCDF on satepsnon   ####
my $getNETCDF_FTP = ' ';
 $getNETCDF_FTP = getSateps($year,$mm,$dd,$ftpnon,$remote_dir_satepsnon,$output_prefix{nc},$output_suffix{nc},$ProdId_name);
 print "getNETCDF_FTP = $getNETCDF_FTP\n";
if ( $getNETCDF_FTP == 'Failure' ) {
####   print MFILE "|In Dist1     |MYDdust.aod_conc.v6.3.4.$ProdId_name.grib|NO DUST Data in This Time|\n"
;
 }
 else {
   $num_satepsnon_nc = $num_satepsnon_nc + 1;
}
#### check data on Dist1 grib   ####
#for ($i=$begin_time; $i<=$end_time; $i++) {
$ProdId_name = $Id_name . "hr";
$ProdId_name = $ProdId_name . $i;
print " IN dist1 grib $ProdId_name \n";

my $getDist1Grib = ' ';
 $getDist1Grib = getDist1($year,$mm,$dd,$Dist1,$remote_dir,$output_prefix{grd},$output_suffix{grd},$ProdId_name);
 print " getDist1Grib = $getDist1Grib\n";
 if ( $getDist1Grib == 'Failure' ) {
####   print MFILE "|In Dist1     |MYDdust.aod_conc.v6.3.4.$ProdId_name.grib|NO DUST Data in This Time|\n";
 }
 else {
   $num_dist1_grib = $num_dist1_grib + 1;
   print MFILE "|In Dist1 | MYDdust.aod_conc.v6.3.4.$ProdId_name.grib | $getDist1Grib |\n";
 }
##### check  data on satepsnon   ####
my $getGrib_FTP = ' ';
 $getGrib_FTP = getSateps($year,$mm,$dd,$ftpnon,$remote_dir_satepsnon,$output_prefix{grd},$output_suffix{grd},$ProdId_name);
 print "getSateps = $getGrib_FTP\n";
if ( $getGrib_FTP == 'Failure' ) {
####   print MFILE "|In Dist1     |MYDdust.aod_conc.v6.3.4.$ProdId_name.grib|NO DUST Data in This Time|\n";
 }
 else {
   $num_satepsnon_grib = $num_satepsnon_grib + 1;

}
####### check gp5   png   ####
my $getPNG_gp5 = ' ';
print " IIII = $i\n";
 $getPNG_gp5 = getGP5($i);
print "PNG_gp5 = $getPNG_gp5\n";

 if ($getPNG_gp5 == 'Failure' ) {
####   print MFILE "|In Dist1     |MYDdust.aod_conc.v6.3.4.$ProdId_name.grib|NO DUST Data in This Time|\n";
 } 
  else { 
   $num_gp5_png = $num_gp5_png + 1;
###  print the return time to table #####
   print DUSTFILE "$ProdId_name", "     $getLocal_Grib","  $getDist1Net ", "  $getDist1Grib "," $getGrib_FTP","  $getNETCDF_FTP ","  $getPNG_gp5"." \n"  ;

} 



##### 
}#end if

} # end loop
print " Number of data = $output_number   $num_satepsnon_grib   $num_satepsnon_nc   $num_dist1_grib  $num_dist1_nc  $num_gp5_png \n";

if (( $output_number < 3) || ( $num_satepsnon_grib ne $output_number) || ($num_dist1_grib ne $output_number) || ($num_gp5_png ne $output_number)) {
 my ($year_now, $mm_now, $dd_now, $julday_now) = getDate(0);
 open(EMILFILE, ">$output_table_dir/mailDUST_ERROR.txt ") || die "Can not open mailDUST_ERROR.txt for writing: $!";
 print EMILFILE "On $year_now$mm_now$dd_now the MODIS DUST product ended abnormally. \n";
 print EMILFILE "Please open a help ticket and notify the point of contact  \n";
 print EMILFILE "Liang Chen ( Liang.chen\@noaa.gov, 301-683-3286) \n";
 print EMILFILE "Additional details may be obtained from the product monitoring logs at \n";
 print EMILFILE "http://www.ssd.noaa.gov/PS/FIRE/DUST/PNGFILES_DUST/Dust.log   \n";
close(EMILFILE);
#$cclist = "-c  'ESPCoperations\@noaa.gov' ";
$emailList = "'Liang.chen\@noaa.gov' ";
system("/bin/mail -s 'MODIS DUST  error ' Liang.chen\@noaa.gov < /data/Dust/table/mailDUST_ERROR.txt ");

}
}
print "nuimber of grd on Satepsanon = $num_satepsnon_grib $num_satepsnon_nc $num_dist1_grib $num_dist1_nc   \n";


close(MFILE); 
close(DUSTFILE);

print "end of monitor  \n";



sub getDate
{
 my $In_leapday =  shift(@_);
 print " in getDate leap    @_ \n";

 my $yday_in = time()-86400*($In_leapday);
 my ($sec_in,$min_in,$hour_in,$D_in,$M_in,$Y_in,$W_in,$J_in,$I_in) = localtime($yday_in);
print "test2  $sec_in,$min_in,$hour_in,$D_in,$M_in,$Y_in,$W_in,$J_in,$I_in   \n";

 my $year_in = $Y_in + 1900;
 my $mm_in =  $M_in + 1;
 if ($mm_in < 10 ) {
  $mm_in = "0$mm_in";
 }
  my $dd_in = $D_in;
 if ($dd_in < 10) {
  $dd_in = "0$dd_in";
 }
 my $julday_in = $J_in + 1;
 if ($julday_in < 10)
 {
    $julday_in = "00" . $julday_in;
 }
 elsif (($julday_in < 100 )  && ($julday_in > 9))
 {
    $julday_in = "0" . $julday_in;
 }

 if ($hour_in < 10) {
   $hour_in = "0$hour_in";
 }
 if ($min_in< 10) {
  $min_in = "0$min_in";
 }
 print "year_in  $year_in  $mm_in $dd_in $julday_in  $min_in  $hour_in  \n";

# print "year_in  $year_in  $mm_in $dd_in $julday_in  \n";

 return ($year_in,$mm_in,$dd_in,$julday_in);

}

sub getSateps
{
 my ($year,$mm, $dd,$remoteSite,$remoteDustDataDir,$output_prefix,$output_suffix,$ProdId_name) = @_;
  my @inFileList = ' ';
 my $inFile = ' ';
 my @splitMsg = ' ';
 my  $mon = 0;
 my $digMon = 0;
 my $hhmm = ' ';
 my $hhmm_1 = ' ';
 my $hhmm_2 = ' ';
 my $retur_data = 'Failure     ';
 my $filesize = 0;
 my $day = ' ';

 my %months = qw(NONE 0 JAN 01 FEB 02 MAR 03 APR 04 MAY 05 JUN 06
                   JUL 07 AUG 08 SEP 09 OCT 10 NOV 11 DEC 12);

 my ($year_today, $mm_today, $dd_today, $julday_today) = getDate(0);

 my $inPattern = "$output_prefix$ProdId_name$output_suffix";
 my $sftpfile1 = '/data/Dust/logs/getRemoteDust_File.txt';
print "IIII inPattern  $inPattern   \n";
print "WWW  $remoteSite   $remoteDustDataDir  \n";

open  ( LUOUT,  ">$sftpfile1" )  ||  die " Can't seem to open $sftpfile1: $!";
  print  LUOUT   "sftp $remoteSite << EOF\n";
  print  LUOUT   "    cd  $remoteDustDataDir \n";
  print  LUOUT   "    ls -l $inPattern \n";
  print  LUOUT   "    bye\n";
   print  LUOUT   "EOF\n";
close ( LUOUT );
chmod  (0777,  $sftpfile1) ;

#-----------------------------------------------------------------------------
open ( SFTPOUT,  "$sftpfile1 |" );

 @inFileList = <SFTPOUT>;
 print "in remote sateps @inFileList \n";


 if (@inFileList ne ' ' ) {
   foreach $inFile (@inFileList) {
     chomp($inFile);
     @splitMsg = split(/\s+/, $inFile);
     $mon = $splitMsg[-4];
     $mon =~ tr/a-z/A-Z/;
     $digMon = $months{$mon};
     $day = $splitMsg[-3];
     if ($day < 10) {
       $day = "0$day";
     }
     $hhmm = $splitMsg[-2];
     $hhmm =~ s/\://g;
     $filesize = $splitMsg[-5];
     if ($filesize > 70 ) {
       $retur_data = $year_today.$digMon.$day.$hhmm;
     }
   }
 }
 close(SFTPOUT);
print "in remote sateps : $retur_data \n";
 return $retur_data;

}

sub getProduc
{
 my ($year,$mm, $dd,$output_dir,$inPattern,$ProdId_name) = @_;
 my @inFileList = ' ';
 my $inFile = ' ';
 my @splitMsg = ' ';
 my  $mon = 0;
 my $digMon = 0;
 my $hhmm = 0;
 my $retur_data = 'Failure     ';
 my $filesize = 0;
 my $day = 0;

 my %months = qw(NONE 0 JAN 01 FEB 02 MAR 03 APR 04 MAY 05 JUN 06
                   JUL 07 AUG 08 SEP 09 OCT 10 NOV 11 DEC 12);
 my ($year_today, $mm_today, $dd_today, $julday_today) = getDate(0);

# my $inPattern = "$output_prefix$ProdId_name$output_suffix";

print " file name :$inPattern \n";
 chdir("$output_dir");
  @inFileList = `ls -l $inPattern`;
 print "ingetProduc:  @inFileList \n";

 if (@inFileList ne ' ' ) {
  foreach $inFile (@inFileList) {
     chomp($inFile);
     print " inFile  $inFile  \n";
     @splitMsg = split(/\s+/, $inFile);
     $mon = $splitMsg[5];
     $mon =~ tr/a-z/A-Z/;
     $digMon = $months{$mon};
     $day = $splitMsg[6];
     if ($day < 10 ) {
      $day = "0$day";
     }
     $hhmm = $splitMsg[7];
     $hhmm =~ s/\://g;
#    print " in InputHYSPLIT HH $mon $day $hhmm  \n";
     $filesize = $splitMsg[4];
     if ($filesize > 70 ) {
       $retur_data = $year_today.$digMon.$day.$hhmm;
     }
    }
   }

print "getProduc retur_data: $retur_data\n";
return $retur_data;
}

sub getDist1
{
 my ($year,$mm, $dd,$remoteSite,$remoteDustDataDir,$output_prefix,$output_suffix,$ProdId_name) = @_;
  my @inFileList = ' ';
 my $inFile = ' ';
 my @splitMsg = ' ';
 my  $mon = 0;
 my $digMon = 0;
 my $hhmm = ' ';
 my $hhmm_1 = ' ';
 my $hhmm_2 = ' ';
 my $retur_data = 'Failure     ';
 my $filesize = 0;
 my $day = ' ';

 my %months = qw(NONE 0 JAN 01 FEB 02 MAR 03 APR 04 MAY 05 JUN 06
                   JUL 07 AUG 08 SEP 09 OCT 10 NOV 11 DEC 12);

 my ($year_today, $mm_today, $dd_today, $julday_today) = getDate(0);

 my $inPattern = "$output_prefix$ProdId_name$output_suffix";
 my $sftpfile1 = '/data/Dust/logs/getRemoteDust_File.txt';
print "RRR inPattern  $inPattern   \n";
print "UUUU  $remoteSite   $remoteDustDataDir  \n";

open  ( LUOUT,  ">$sftpfile1" )  ||  die " Can't seem to open $sftpfile1: $!";
  print  LUOUT   "sftp $remoteSite << EOF\n";
  print  LUOUT   "    cd  $remoteDustDataDir \n";
  print  LUOUT   "    ls -l $inPattern \n";
  print  LUOUT   "    bye\n";
   print  LUOUT   "EOF\n";
close ( LUOUT );
chmod  (0777,  $sftpfile1) ;

#-----------------------------------------------------------------------------
open ( SFTPOUT,  "$sftpfile1 |" );

 @inFileList = <SFTPOUT>;
 print "in remote sateps @inFileList \n";


 if (@inFileList ne ' ' ) {
   foreach $inFile (@inFileList) {
     chomp($inFile);
     @splitMsg = split(/\s+/, $inFile);
     $mon = $splitMsg[-4];
     $mon =~ tr/a-z/A-Z/;
     $digMon = $months{$mon};
     $day = $splitMsg[-3];
     if ($day < 10) {
       $day = "0$day";
     }
     $hhmm = $splitMsg[-2];
     $hhmm =~ s/\://g;
     $filesize = $splitMsg[-5];
     if ($filesize > 70 ) {
       $retur_data = $year_today.$digMon.$day.$hhmm;
     }
   }
 }
 close(SFTPOUT);
print "in remote sateps : $retur_data \n";
 return $retur_data;

}

sub getGp5
{
 my ($year,$mm, $dd,$remoteSite,$remoteDustDataDir,$output_prefix,$output_suffix,$ProdId_name) = @_;
  my @inFileList = ' ';
 my $inFile = ' '; 
 my @splitMsg = ' ';
 my  $mon = 0;
 my $digMon = 0;
 my $hhmm = ' ';
 my $hhmm_1 = ' ';
 my $hhmm_2 = ' ';
 my $retur_data = 'Failure     ';
 my $filesize = 0;
 my $day = ' '; 
 
 my %months = qw(NONE 0 JAN 01 FEB 02 MAR 03 APR 04 MAY 05 JUN 06
                   JUL 07 AUG 08 SEP 09 OCT 10 NOV 11 DEC 12); 

 my ($year_today, $mm_today, $dd_today, $julday_today) = getDate(0);
 
 my $inPattern = "$output_prefix$ProdId_name$output_suffix";
 my $sftpfile1 = '/data/Dust/logs/getRemoteDust_File.txt';
print "RRR inPattern  $inPattern   \n";
print "UUUU  $remoteSite   $remoteDustDataDir  \n";

open  ( LUOUT,  ">$sftpfile1" )  ||  die " Can't seem to open $sftpfile1: $!";
  print  LUOUT   "sftp $remoteSite << EOF\n";
  print  LUOUT   "    cd  $remoteDustDataDir \n";
  print  LUOUT   "    ls -l $inPattern \n";
  print  LUOUT   "    bye\n";
   print  LUOUT   "EOF\n";
close ( LUOUT );
chmod  (0777,  $sftpfile1) ;

#-----------------------------------------------------------------------------
open ( SFTPOUT,  "$sftpfile1 |" );

 @inFileList = <SFTPOUT>;
 print "in remote sateps @inFileList \n";



 if (@inFileList ne ' ' ) {
   foreach $inFile (@inFileList) {
     chomp($inFile);
     @splitMsg = split(/\s+/, $inFile);
     $mon = $splitMsg[-4];
     $mon =~ tr/a-z/A-Z/;
     $digMon = $months{$mon};
     $day = $splitMsg[-3];
     if ($day < 10) {
       $day = "0$day";
     }
     $hhmm = $splitMsg[-2];
     $hhmm =~ s/\://g;
     $filesize = $splitMsg[-5];
     if ($filesize > 70 ) {
       $retur_data = $year_today.$digMon.$day.$hhmm;
     }
   }
 }
 close(SFTPOUT);
print "in remote sateps : $retur_data \n";
 return $retur_data;

}

sub writehead
{
print DUSTFILE "DUST_output_time  LOCAL_GRIB    NETCDF_Dist1   GRIB_Dist1    NETCDF_FTP    GRIB_FTP       PNG_GP5\n";
}

sub getGP5
{
  my ($ProdId_name) = @_;
  my @inFileList = ' ';
 my $inFile = ' ';
 my @splitMsg = ' ';
 my  $mon = 0;
 my $digMon = 0;
 my $hhmm = 0;
 my $retur_data = 'Failure     ';
 my $filesize = 0;
 my $day = 0;

 my %months = qw(NONE 0 JAN 01 FEB 02 MAR 03 APR 04 MAY 05 JUN 06
                   JUL 07 AUG 08 SEP 09 OCT 10 NOV 11 DEC 12);
 my $output_pre = 'MYDdust.';
 my $output_suffix = '.jpg';

 my ($year_today, $mm_today, $dd_today, $julday_today) = getDate(0);

# $ProdId_name = substr($ProdId_name,8,4);
print " in gp5 date: $ProdId_name \n";

 my $inPattern = "$output_pre$ProdId_name$output_suffix";
 my $sftpfile1 = '/data/Dust/logs/getRemote_GP5_File.txt';
 my $remoteSite = 'sabweb\@10.144.3.87';
 my $remoteFimmaDataDir = '/data/web/mirror/www.ssd.noaa.gov/htdocs/PS/FIRE/DUST/PNGFILES_DUST';

 open  ( LUOUT,  ">$sftpfile1" )  ||  die " Can't seem to open $sftpfile1: $!";
  print  LUOUT   "sftp $remoteSite << EOF\n";
  print  LUOUT   "    cd  $remoteFimmaDataDir \n";
  print  LUOUT   "    ls -l $inPattern \n";
  print  LUOUT   "    bye\n";
   print  LUOUT   "EOF\n";
close ( LUOUT );
chmod  (0777,  $sftpfile1) ;

#-----------------------------------------------------------------------------
open ( SFTPOUT,  "$sftpfile1 |" );

 @inFileList = <SFTPOUT>;
 print "in remote sateps @inFileList \n";

 if (@inFileList ne ' ' ) {
   foreach $inFile (@inFileList) {
     chomp($inFile);
     print " inFile  $inFile  \n";
     @splitMsg = split(/\s+/, $inFile);
     $mon = $splitMsg[-4];
     $mon =~ tr/a-z/A-Z/;
     $digMon = $months{$mon};
     $day = $splitMsg[-3];
     if ($day < 10) {
      $day = "0$day";
     }
     $hhmm = $splitMsg[-2];
     $hhmm =~ s/\://g;
#    print " in InputHYSPLIT HH $mon $day $hhmm  \n";
     $filesize = $splitMsg[-5];
     if ($filesize > 70 ) {
       $retur_data = $year_today.$digMon.$day.$hhmm;
     }
   }
 }
 close(SFTPOUT);
print "in remote sateps : $retur_data \n";
 return $retur_data;

}