#!/usr/bin/env perl
# ./require_obsys 19990801 000000 240000 5 ssmi_wentz_tpw
# Script to check the existence of observation files for a given
#   date range and observation file class from a resource file
# See usage() of this script for more information.
#
# !REVISION HISTORY:
#
# 2001.08.21 Eueng-nan Yeh   Specs and man page.
# 2001.08.22 Eueng-nan Yeh   Initial code.
# 2001.10.02 Eueng-nan Yeh   Accept obclass "all"
# 2001.10.17 Eueng-nan Yeh   Accept multiple obclasses
# 2001.12.18 Eueng-nan Yeh   set @valid, @valid1 as private in get_rcd
# 2002.02.13 Eueng-nan Yeh   Removed number index from obsys class list
# 2002.03.18 Eueng-nan Yeh   Allowed the line "BEGIN" to have more than 2 fields 
# 2002.09.16 Eueng-nam Yeh   Accept records in obsys.rc with no machine name defined
#=========================================================================
      use warnings;
      use Getopt::Long;             # command line options

      @options = qw(drc=s class_desc hard_check la=s rid=s h v q);
      $ENV{PATH} = "./:" . $ENV{PATH};
      $tname = "obsys.rc";          # fileName for dataBase table

# Initialization
# --------------
      ($bymd,$bhms,$ihms,$nstep,$oclass,@class)=init($tname, @options);

# Process observation class as "all" condition
# --------------------------------------------
      if(lc($oclass) eq "all") { do_All(@class); }

# Generate YYYYMMDDhh array @ymd1
# -------------------------------
      @ymd1 = mk_date($bymd,$bhms,$ihms,$nstep,$opt_la);
      print("ymd1=@ymd1\n") if($opt_v);

# Get records from $opt_drc and then validate file existence
# ----------------------------------------------------------
      print(">> Processing require...\n") if(!$opt_q);
      @class = split(/,/, $oclass);
      for($i=0; $i<scalar(@class); $i++) {
        print("> obclass: $class[$i]\n") if(!$opt_q);
        get_rcd($opt_drc, $class[$i], @ymd1);
      }  

# All done
# --------
      close(LUN) || die "Couldn't close file: $opt_drc $!\n";
      print("\nJob finished.\n") if(!$opt_q);
      exit(0);   #0 as a success exit

#=========================================================================
#
# init: sets optional and required command line arguments and check
#       database resource file.
#
      sub init {
        my ($tname, @opt) = @_;
        my ($op1,$op2,$x,$rc,$bymd,$bhms,$ihms,$nstep,$oclass);
        my ($i, $j, $k, $n, @mday, $y4, $m2, $d2, @class);

        @mday=(31,28,31,30,31,30,31,31,30,31,30,31);
        $rc = GetOptions(@opt);
        ($opt_q = $opt_q) or ($opt_q = 0);
        if($opt_q) { $opt_v = 0; }  #opt_q overrules opt_v
        ($opt_h = $opt_h) or ($opt_h = 0);
        ($opt_la = $opt_la) or ($opt_la = 0);
        ($opt_hard_check = $opt_hard_check) or ($opt_hard_check = 0);
        ($opt_drc = $opt_drc) or ($opt_drc = $tname);
        ($opt_class_desc = $opt_class_desc) or ($opt_class_desc = 0);
        print("Database resource file = $opt_drc\n") if ($opt_v);
        open(LUN,"$opt_drc") || die "Fail to open file: $opt_drc $!\n";
# check obclass first
        @class = ck_obclass();
        if($opt_class_desc) { classDesc(@class); }
        usage() if ($opt_h || !$rc || ($#ARGV != 4));
        ($bymd, $bhms, $ihms, $nstep, $oclass) =
          ($ARGV[0], $ARGV[1], $ARGV[2], $ARGV[3], $ARGV[4]);
# validate $bymd
        if(length($bymd) != 8) {msg("E","Beginning YYYYMMDD '$bymd' wrong");}
        $y4=substr($bymd,0,4);  $m2=substr($bymd,4,2);  $d2=substr($bymd,6,2);
        if($m2>12 || $m2<1) {msg("E","Beginning YYYYMMDD '$bymd' wrong");}
        if(($y4%4 == 0 && $y4%100 != 0) || $y4%400==0 ) {$mday[1] = 29;}
        if($d2>$mday[$m2-1] || $d2<1) {
          msg("E","Beginning YYYYMMDD '$bymd' wrong");}
#validate $bhms
        if(length($bhms) != 6 || $bhms >= 240000) {
          msg("E","Beginning HHMMSS '$bhms' wrong");
        }
#validate time increment $ihms and time step $nstep
        if(length($ihms) != 6 || $ihms <=0) {
          msg("E","time step increment HHMMSS '$ihms' wrong");
        }
        if($nstep <= 0) {msg("E","Number of timesteps '$nstep' wrong");}
        return($bymd, $bhms, $ihms, $nstep, $oclass, @class);
      }

#=========================================================================
#
# do_All: looping through database resource file to hard check the existence
#         of each file.
#
      sub do_All {
        my(@class) = @_;
        my($m,$n,$host,$oclass,$nrcd,$rcd,@arr,@field,$sdate,$edate,$hinc,$ymdh);
        $m = scalar(@class);
        $cdate = `date '+%Y%m%d'`;   chomp($cdate);  #current date YYYYMMDD
        $host = `hostname`;  chomp($host);           #local hostName
# check resource file for matched conditions
        for($n=0; $n<$m; $n++) {       #through all classes
          $oclass = $class[$n];
          print(">>class = $oclass\n") if(!$opt_q);
          $nrcd = 0;
LOOP1:    while( defined($rcd = <LUN>) ) {
            chomp($rcd);
            $rcd =~ tr/\t/ /s;   $rcd =~ tr/  / /s;   $rcd =~ s/^ | $//g;
            @arr = split(/ /, $rcd);
            if(length($rcd) <= 2  || $rcd =~ /#/ || scalar(@arr)<2) { next; };
            if($rcd =~ /^BEGIN/i && ($arr[1] eq $oclass)) {#found
              while( defined($rcd = <LUN>) ) {
                chomp($rcd);
                $rcd =~ tr/\t/ /s;   $rcd =~ tr/  / /s;   $rcd =~ s/^ | $//g;
                if(length($rcd) <= 2  || ($rcd =~ /#/)) { next; };
                if($rcd =~ /END/i) {  #end of block
                  if ($nrcd <= 0) {   #no records
                    msg("E", 
                    "No valid record found for class '$oclass' on $opt_drc.");
                  } else {
                    last LOOP1;
                  }
                }
                @field = split(/ /, $rcd);
                if(scalar(@field) == 3 ) {   #for every 3 fields line: Date Increment File
                  $nrcd++;
                  ($sdate, $edate) = ck_date($field[0]);      #return yyyymmddhh
                  $hinc = substr($field[1], 0, 2);            #increment in HH format
                  $ymdh = DateList($sdate, $edate, $hinc);
                  get_ls($host, $field[2], $ymdh);
                }  #End of if(scalar(@field) == 3 )
              }  #End of while( defined($rcd = <LUN>) )
            }   #End of if ($rcd =~ /BEGIN/i && $arr[1] =~ /^$oclass$/)
          }  #LOOP1: End while( defined($rcd = <LUN>) )
        }  #for($n=0; $n<$m; $n++)
        close(LUN) || die "Couldn't close file: $opt_drc $!\n";
        print("\nJob finished.\n") if(!$opt_q);
        exit(0);      #exit with a success
      }
#=========================================================================
#
#classDesc: To get the data description for each obclass defined and quit.
#
      sub classDesc {
        my(@class) = @_;
        my($nclass, $i, @desc, $rcd, $x);

        $nclass = scalar(@class);
        print("\n");
LOOP_A: for($i=0; $i<$nclass; $i++) {
          $desc[$i] = "";
          while( defined($rcd = <LUN>) ) {  # <== to read a rcd
            chomp($rcd);
            $rcd =~ tr/\t/ /s;   $rcd =~ tr/  / /s;   $rcd =~ s/^ | $//g;
            if( $rcd =~  /^#\+/ && length($rcd) > 2 && $rcd =~ /BEGIN/) { #found BEGIN
              while( defined($rcd = <LUN>) ) {  # <== to read a rcd
                chomp($rcd);
                $rcd =~ tr/\t/ /s;   $rcd =~ tr/  / /s;   $rcd =~ s/^ | $//g;
                if( length($rcd) <= 2 ) { next; }
                if($rcd =~ /$class[$i]/ && $rcd =~ /:/) {
                  ($x, $desc[$i]) = split(":", $rcd);
                  seek(LUN, 0, 0);    next LOOP_A;
                }
                if( $rcd =~  /^#\+/ && length($rcd) > 2 && $rcd =~ /END/) { #found END
                  seek(LUN, 0, 0);    next LOOP_A;
                }
              }
            }  #found BEGIN
          }  #End of while( defined($rcd = <LUN>) )
        }
        print("     obclass               Data Description\n");
        print("     ----------------      -------------------------\n");
        for($i=0; $i<$nclass; $i++) {
          $x = $i + 1;
          printf("     %-20s %-45s\n", $class[$i], $desc[$i]);
        }
        print("\n");    exit(0);  #0 as a success exit
      }
#=========================================================================
#
# mk_date: Returns with array of unique records of YYYYMMDDhh.
#
      sub mk_date {
        my($bymd,$bhms,$ihms,$nstep,$opt_la) = @_;
        my(@mday,@ymd,@ymd1,$y4,$m2,$d2,$i,$n,$m,$k);
        @mday=(31,28,31,30,31,30,31,31,30,31,30,31);
        $m = 0;   $k = $bhms/10000;  $ymd[$m]=$bymd . sprintf("%02d",$k);
        $y4=substr($ymd[$m],0,4);  $m2=substr($ymd[$m],4,2);  $d2=substr($ymd[$m],6,2);
        if(($y4%4 == 0 && $y4%100 != 0) || $y4%400==0 ) { $mday[1] = 29; }
        $i = $nstep;
        if($opt_la) { $i += $opt_la; }  #look ahead option
        for($n=1; $n<$i; $n++) {
          $bhms += $ihms;
          $m++;
          while($bhms >= 240000 ) {
            $bhms -= 240000;
            $d2++;
            if($d2>$mday[$m2-1]) {$d2 -= $mday[$m2-1];  $m2++;}
            if($m2>12) {
              $m2 -= 12;
              $y4++;
              if(($y4%4 == 0 && $y4%100 != 0) || $y4%400==0 ) {$mday[1] = 29; }
              else { $mday[1] = 28; }
            }
          }  #End of while($bhms >= 240000 )
          $k = $bhms/10000;
          $ymd[$m]=$y4 . sprintf("%02d",$m2) . sprintf("%02d",$d2) . sprintf("%02d",$k);
        } #End of for($n=1; $n<$i; $n++)
        return(@ymd);
      }

#=========================================================================
#
# get_rcd: get valid records @data1 from database template file $opt_drc,
#          list files if $opt_hard_check used to validate the existence of files
#
      sub get_rcd {
        my($tname, $oclass, @ymd1) = @_;  # ymd in YYYYMMDDHH format
        my($i,$j, @arr, $nrcd, @field, @data, $m, $n, @data1, @ymd);
        my($k,$k1,$found,@name,@y4,$sdate,$edate,$hinc,$ymdh,$bymd,$eymd);
        my($l1, $l2, @valid, @valid1);
        $yfound = $notfound = 0;
        $m = scalar(@ymd1);
        $nrcd = 0;
        $l1 = length($oclass);
        $cdate = `date '+%Y%m%d'`;   chomp($cdate);  #current date YYYYMMDD
        $host = `hostname`;  chomp($host);           #local hostName
        $bymd = substr($ymd1[0], 0, 8);
        $eymd = substr($ymd1[$m-1], 0, 8);
# check resource file for matched conditions
        $n = 0;
LOOP1:  while( defined($rcd = <LUN>) ) {
          chomp($rcd);
          $rcd =~ tr/\t/ /s;   $rcd =~ tr/  / /s;   $rcd =~ s/^ | $//g;
          @arr = split(/ /, $rcd);
          if(length($rcd) <= 2  || $rcd =~ /#/ || scalar(@arr)<2) { next; };
          $l2 = length($arr[1]);
          if($rcd =~ /^BEGIN/i && $arr[1] =~ /^$oclass/ && $arr[1] =~ /$oclass$/ && $l1 == $l2) {#found
            while( defined($rcd = <LUN>) ) {
              chomp($rcd);
              $rcd =~ tr/\t/ /s;   $rcd =~ tr/  / /s;   $rcd =~ s/^ | $//g;
              if(length($rcd) <= 2  || ($rcd =~ /#/)) { next; };
              if($rcd =~ /END/i) {  #end of block
                if ($nrcd <= 0) {   #no records
                  msg("E", 
                 "No valid record found for class '$oclass' from $bymd to $eymd on $tname.");
                } else {
                  last LOOP1;
                }
              }
              @field = split(/ /, $rcd);
              if(scalar(@field) == 3 ) {   #for every 3 fields line: Date Increment File
                ($sdate, $edate) = ck_date($field[0]);      #return yyyymmddhh
                $hinc = substr($field[1], 0, 2);            #increment in HH format
                $ymdh = DateList($sdate, $edate, $hinc);   
                $found = 0;
                for($i=0; $i<$m; $i++) {   #looping through all input ymd1
                  if($ymdh =~ /$ymd1[$i]/ && length($field[2])>10) {
                    $data[$nrcd] = $field[2];
                    push(@valid, $ymd1[$i]);
                    if($found == 0 ) { $y4[$n] = $ymd1[$i]; }  #y4 is valid date list
                    else { $y4[$n] .= "," . $ymd1[$i]; }
                    $name[$n] = $field[2];       #field number 3 is the fullname
                    $nrcd++;
                    $found = 1;
                  }
                }  #End of for($i=0; $i<$m; $i++)
              }  #End of if(scalar(@field) == 3 )
              if($found == 1) { $n++; }
            }  #End of while( defined($rcd = <LUN>) )
          }   #End of if ($rcd =~ /BEGIN/i && $arr[1] =~ /^$oclass$/)
        }  #LOOP1: End while( defined($rcd = <LUN>) )
        if($opt_v) {for($i=0; $i<scalar(@name); $i++) {print("$name[$i]: $y4[$i]\n");}}
        if($nrcd <= 0) {msg("E", "ClassName '$oclass' not found."); }

# Unique valid date @valid1 and invalid date list @ymd
        @valid = sort(@valid);
        $n = 0;    $valid1[$n] = $valid[0];
        for($i=1; $i<scalar(@valid); $i++) {
          if($valid[$i] gt $valid1[$n]) {
            push(@valid1, $valid[$i]);
            $n++;
          }
        }
        print("\nDate found <<@valid1>>\n") if ($opt_v); #YYYYMMDDHH
        $n = scalar(@valid1);
        if(($m-$n) > 0 ) {         #invalid date found
          $k = $k1 = 0;
LOOP2:    for($j=0; $j<$m; $j++) {
            for($i=$k; $i<$n; $i++) {
              if($ymd1[$j] == $valid1[$i]) {
                $k++;
                next LOOP2;
              }
            }
            $ymd[$k1] = $ymd1[$j];
            $k1++;
          }  #End of for($j=0; $j<$m; $j++)
          print("\nFile name template not found for the following date ");
          print("for obclass '$oclass' from $tname:\n");
          msg("E", "  >>@ymd<<"); 
        }
# Unique data @data1
        @data = sort(@data);
        $m = 0;    $data1[$m] = $data[0];
        for($i=1; $i<scalar(@data); $i++) {
          if($data[$i] gt $data1[$m]) {
            push(@data1, $data[$i]);
            $m++;
          }
        }
        $j = 0;
        if(!$opt_q) {
          print("\nFile name template found for obclass '$oclass' from $tname:\n");}
        for($i = 0; $i<scalar(@data1); $i++) {
          ($j++) if( length($data1[$i]) > 10 );
          print("$data1[$i]\n") if(!$opt_q);
        }
        if($j <= 0) {msg("E","No matched records found for obclass '$oclass' from $bymd to $eymd.");}

# valid the existence of file (hard check)
        if($opt_hard_check) {   #for file list
          print("\n>> Checking the existence of files...\n") if(!$opt_q);
          print("local host = $host\n") if($opt_v);
          $i = scalar(@name);
          for($j=0; $j<$i; $j++) {
            get_ls($host, $name[$j], $y4[$j]);
          }
          if(!$opt_q) {
            print("\nA total of $yfound files valid.\n");
            print("A total of $notfound files not valid.\n");
          } 
        }   # if($opt_hard_check)
        seek(LUN, 0, 0);
        return();
      }

#=========================================================================
#
#ch_obclass: 1. BEGIN/END pair; 2. BEGIN/END all caps; 3. duplicate obClassNames.
#
      sub ck_obclass {
        my($nclass, $rcd, @arr, @class, @slass, $i, $j, $k, $mark);
        $nclass = $mark = $k = 0;
        while( defined($rcd = <LUN>) ) {  # <== to read a rcd
          $k++;
          chomp($rcd);
          $rcd =~ tr/\t/ /s;   $rcd =~ tr/  / /s;   $rcd =~ s/^ | $//g;
          if(length($rcd) <= 2 || ($rcd =~ /#/)) { next; };  #skip lines with #
          if($rcd =~ /BEGIN/i ) {
            $mark++;   #after BEGIN
            if( !($rcd =~ /BEGIN/)) {
              msg("W", "All caps 'BEGIN': $rcd.");
            }
            if($mark != 1) {
              msg("E", "BEGIN\/END not in pair: $rcd.");
            }
            @arr = split(/ /, $rcd);
            if(scalar(@arr) < 2 ) { msg("E", "Format is BEGIN obclassName."); }
            $class[$nclass] = $arr[1];
            $nclass++;
          } elsif($rcd =~ /END/i ) {
            $mark--;   #after END (a pair)
            if( !($rcd =~ /END/)) {
              msg("W", "All caps 'END': $rcd.");
            }
            if($mark != 0) {
              msg("E", "BEGIN\/END not in pair: $rcd.");
            }
          } elsif($mark == 0) {  # Illegal lines
             msg("W", "Illegal line_$k: $rcd.");
          } #End of if($rcd =~ /BEGIN/i )
        }  #End of while( defined($rcd = <LUN>) )
        @slass = sort(@class);   #sorted class
# checking for duplicate obclasses block:
        for ($i = 0; $i < scalar(@slass)-1; $i++) {
          for ($j = $i+1; $j < scalar(@slass); $j++) {
            if($slass[$j] =~ /^$slass[$i]/i && $slass[$j] =~ /$slass[$i]$/i ) {
              if($slass[$j] =~ /$slass[$i]/) {
                msg("E", "Identical obclass '$slass[$j]' found.");  #identical
              } else { msg("W", "obclass $slass[$j] == $slass[$i]?"); }
            }
          }
        }
        if($opt_v) {
          print("\n>> Class list:\n");
          for ($i = 0; $i < scalar(@class); $i++) {
            printf("%03d: %-35s\n", $i+1, $class[$i]);
          }
        }
        seek(LUN, 0, 0);
        return(@class);
      }

#=========================================================================
#
# DateList: return a string of YYYYMMDDHH,... from beginning YYYYMMDDHH
#           to ending YYYYMMDDHH with HH as increment
#
     sub DateList {
       my($bymdh, $eymdh, $hinc) = @_;
       my(@mday,$ymdh,$ymdh1,$bhms,$y4,$m2,$d2);

       @mday=(31,28,31,30,31,30,31,31,30,31,30,31);
       $ymdh1=$bymdh;
       $y4=substr($ymdh1,0,4);  $m2=substr($ymdh1,4,2);  $d2=substr($ymdh1,6,2);
       $bhms = substr($ymdh1,8,2);
       if(($y4%4 == 0 && $y4%100 != 0) || $y4%400==0 ) { $mday[1] = 29; }
       do {
         $bhms += $hinc;   #in HH format
         while($bhms >= 24 ) {
           $bhms -= 24;
           $d2++;
           if($d2>$mday[$m2-1]) {$d2 -= $mday[$m2-1];  $m2++;}
           if($m2>12) {
             $m2 -= 12;
             $y4++;
             if(($y4%4 == 0 && $y4%100 != 0) || $y4%400==0 ) {$mday[1] = 29; }
             else { $mday[1] = 28; }
           }
         }  #End of while($bhms >= 24 )
         $ymdh = $y4 . sprintf("%02d",$m2) . sprintf("%02d",$d2). sprintf("%02d",$bhms);
         if($ymdh > $eymdh) { return($ymdh1); }   #beyond eymdh
         $ymdh1 .= ",".$ymdh;
       } until ($eymdh <= $ymdh);
       return($ymdh1);
     }

#=========================================================================
#
# ck_date: subroutine to check and convert YYYYMMDD_HHz-YYYYMMDD_HHz(present)
#          and return with start, end in YYYYMMDDHH format
#
      sub ck_date {
        my($date_date) = @_;   #YYYYmmDD_HHz-YYYYmmDD_HHz(present)  HHz<24z
        my(@mday) = (31, 28, 31, 30,31,30,31,31,30,31,30,31);
        my($yyyy, $mm, $dd, $leap, $date, @arr, $m, $n, @d1, @t1, @x);

        if(length($date_date) <= 15) {msg("E", "WrongFormat: $date_date");}
        @arr = split(/-/, $date_date);
        $m = scalar(@arr);
        if($m != 2) {msg("E", "WrongFormat: $date_date");}
        for($n=0; $n<$m; $n++) {
          if($arr[$n] =~ /prese/i) {
            $d1[$n] = 'present';    $t1[$n] = '18';
          } else {
            if(length($arr[$n]) != 12) {msg("E","WrongFormat: $date_date");}
            @x = split(/_/, $arr[$n]);  #YYYYmmDD_HHz
            $d1[$n] = $x[0];          $t1[$n] = substr($x[1], 0, 2);
            if($t1[$n]>=24 || $t1[$n]<0){msg("E","WrongFormat: $arr[$n]");}
          }
          $date = $d1[$n];
          $leap = 0;      #regiular year
          if(length($date) <= 7 && $date =~ /pre/i ) { $date = $cdate; }
          if(length($date) != 8) {msg("E", "WrongDate:'$date' $rcd."); }
          $yyyy = substr($date, 0, 4);
          $mm   = substr($date, 4, 2);
          $dd   = substr($date, 6, 2);
          if($mm>12 || $mm < 1) {msg("E", "WrongDate:'$date' $rcd."); }
          if(($yyyy%4 == 0 && $yyyy%100 != 0) || $yyyy%400==0 )
          {$leap = 1; }
          if($mm == 2 ) {
            if($dd > ($mday[1]+$leap) || ($dd < 1)) {
              msg("E", "WrongDate:'$date' $rcd."); }
          } else {
            if($dd>$mday[$mm-1] || ($dd<1)) {
              msg("E", "WrongDate:'$date' $rcd."); }
          }
          $d1[$n] = $date*100 + $t1[$n];  #yyyymmddhh
        }
        return($d1[0], $d1[1]);
      }

#=========================================================================
#
# get_ls: to validate (hard check) if the given file do exist
#
      sub get_ls {
        my($host, $fullname, $valid) = @_;
        my($i, $j, $k, $m, $n, @ymd, $name, $new, $y4,$y2,$m2,$d2,$h2);
        my($mass, @arr, $local, $opt, $rsh);
        $rsh = "ssh";
        @ymd = split(/,/, $valid);   # $valid in the form of YYYYMMDDHH
        $n = scalar(@ymd);
        $mass = $ENV{MHOST};
        $opt = "-lL";
        for($i=0; $i<$n; $i++) {
          $new = $fullname; 
          $y4 = substr($ymd[$i], 0, 4);
          $y2 = substr($ymd[$i], 2, 2);
          $m2 = substr($ymd[$i], 4, 2);
          $d2 = substr($ymd[$i], 6, 2);
          $h2 = substr($ymd[$i], 8, 2);
          $new =~ s/%y4/$y4/g;
          $new =~ s/%y2/$y2/g;
          $new =~ s/%m2/$m2/g;
          $new =~ s/%d2/$d2/g;
          $new =~ s/%h2/$h2/g;
  #        $new =~ s/%h2/[0-9][0-9]/g;
          $new =~ s/%c/?/g;       #Warning: this may generate more than one files
          $new =~ s/%n/[0-9]/g;   #Warning: this may generate more than one files
          if( ($new =~ /MHOST/) && !$mass ) {
             msg("E", "MHOST not defined: $new.");
             $notfound++;
             next;        #skip this record
          }
          $new =~ s/\${MHOST}/$mass/g;
          @arr = split(/:/, $new);         
          $m2 = scalar(@arr);
          if( $host =~ /$arr[0]/ || $arr[0]=~ /$host/ || $m2 == 1) {
            $local = 0;        #local server
            if( $m2 == 1 ) { @list = `ls $opt $arr[0]`; #no hostName
            } else {
              @list = `ls $opt $arr[1]`;
            }
          } else { 
            if($opt_rid) {
              $arr[0] = $opt_rid ."\@" .$arr[0];
            }
            @list = `$rsh $arr[0] ls $opt $arr[1]`;
          }
          $j = scalar(@list);
          if ($j<1) {    # exit after first Error encountered
            $notfound++;
            msg("E", "Server $arr[0] not accessible or no files found $arr[1].");
            next;
          } else {        #file exists
            if($list[0] =~ /password/) { msg("E", "$arr[0] need new password."); }
            for($k = 0; $k<$j; $k++) {
              $list[$k] =~ tr/  / /s;        $list[$k] =~ s/^ //;
              @arr = split(/ /, $list[$k]);   
              for($m = 0; $m<=3; $m++) { shift(@arr); }
              print("Valid:@arr") if(!$opt_q);
              $yfound++;
            } # for($m = 0; $m<$j; $m++)
          }
        }   #for($i=0; $i<$n; $i++)
      }
#=========================================================================
#
# msg: subroutine to print out error and warning messages.  Exit the
#      software if the first input argument indicates an error "E".
#
       sub msg {
         my ($a, $b) = @_;
         $a =~ tr/a-z/A-Z/;
         print("-$a: $b\n");
         if($a =~ /^E/) { exit(1); };  # Exit for error
       }

#=========================================================================
sub usage {

   print <<"EOF";

NAME
     require_obsys - To check the existence of observation files for a given
                     date range and observation file class from a 
                     resource file

SYNOPSIS

     require_obsys [...options...]  bymd bhms ihms nstep obclass

DESCRIPTION

     Require_obsys is a utility for checking the file existence from specified
     date range and observation file class from mass storage.  The full path
     names of the files to be checked are specified in a database resource
     file (see RESOURCE FILE below) by means of GrADS-like templates which
     will be used by require_obsys.  For example, a resource file
     could specify a file name of the form:

     gatun:/silo3/dao_ops/conv+tovs/ods/Y%y4/M%m2/r4ods.t%y4%m2%d2_%h2z

     The following parameters are required on input:

     bymd     beginning year-month-day, e.g., 19990809
     bhms     beginning hour-min-sec, e.g., 000000
     ihms     time step increment in hour-min-sec format, e.g., 240000
     nstep    number of timesteps for time looping, e.g., 6.
     obclass  file class, e.g., conv_tovs, ers2, qscat, ssmi_wentz_tpw,
              ssmi_wentz_wind, sonde_hirs, and tovs_l1b etc.
              Note: You can enter multiple obclasses such as conv_tovs,ers2
              (no blank spaces and comma is the separator)

     For example,

         require_obsys -hard_check 19990809 000000 240000 6 conv_tovs

     will generate the followings date (YYYYMMDDHH): 
     1999080900,1999081000,1999081100,1999081200,1999081300,1999081400

    For each one of these dates, require_obsys will expand the
    GrADS-like template and list the corresponding files from
    mass storage. For the examples above, the files to be listed
    are:

    gatun:/silo3/dao_ops/conv+tovs/ods/Y1999/M08/r4ods.t19990809_00z
    gatun:/silo3/dao_ops/conv+tovs/ods/Y1999/M08/r4ods.t19990810_00z
    gatun:/silo3/dao_ops/conv+tovs/ods/Y1999/M08/r4ods.t19990811_00z
    gatun:/silo3/dao_ops/conv+tovs/ods/Y1999/M08/r4ods.t19990812_00z
    gatun:/silo3/dao_ops/conv+tovs/ods/Y1999/M08/r4ods.t19990813_00z
    gatun:/silo3/dao_ops/conv+tovs/ods/Y1999/M08/r4ods.t19990814_00z

OPTIONS

     -h              prints this page
     -class_desc     to get the description for each obclass and quit
     -drc fullName   user specified database resource file with file path
                      (default is obsys.rc)
     -hard_check     actually check if the specified file do exist in the mass storage
                      (default is soft_check that is the input conditions will be
                      verified from the observation database resource file)
     -rid user       user ID used in remote host machine
                      (default is ENV{USER} from local machine)
     -la lstep       Look ahead lstep time steps, that is, checking the existence of
                      future data files (default: lstep=0)
     -v              verbose mode (default is quiet)
     -q              very quiet (if success, nothing will be printed; if failure, 

RESOURCE FILES

     The resource file consists of information blocks for each obclass.
     Records defined by the obclass block show the remote file name template
     and the time range (start/end date and increment) which the remote file name
     template can be applied to.  For example:

     BEGIN obclass
       startYYYYMMDD_HHz-endYYYYMMDD_HHz incrementHHMMSS remote_file_name_template
     END

     The remote file name template is following:

           rhost:path/fname

     This is better explained with an example:

     helios1:/u/joiner/tovs/l1b/%y4%m2/l1c_new/hirs.nj.hdf.t%y4%m2%d2

     In this example, the remote host (rhost) is "helios1", the path
     is /u/joiner/tovs/l1b/%y4%m2/l1c_new, and the fname is
     hirs.nj.hdf.t%y4%m2%d2.

     This file name "template" is a GrADS like pattern for matching
     remote file names (that is, files to be retrieved).
     Supported tokens are:

          %y4       year,  e.g., 1997
          %y2       year,  e.g., 97
          %m2       month, e.g., 12
          %d2       day,   e.g., 31
          %h2       hour,  e.g., 18
          %c        matches any character [?]
          %n        matches any single digit [0-9]

    In addition to GrADS-like tokens, any defined environment variable
    can be used in the remote file name template. For example,

     \${MHOST}:/u/joiner/tovs/l1b/%y4%m2/l1c_new/hirs.nj.hdf.t%y4%m2%d2

   would take the remote host name from the environment variable \$MHOST.

NOTE

    The observation file class "all" is a special case.  Under this
    circumstance, each record defined by the database resource file
    will be hard checked for the date/time range as stated in the
    database file (command line input date/time information will be
    accepted but ignored).

SEE ALSO

    acquire_obsys - Retrives files from mass storage with look-ahead capability

AUTHORS

     Eueng-nan Yeh (enyeh\@dao.gsfc.nasa.gov)

EOF

  exit(1)
}
