#!/usr/local/bin/perl
#################################################################
#
# + FtpLocate - FTP search engine
#
# by Distributed System Lab E.E. NCKU Taiwan 2000
# http://ftp.ee.ncku.edu.tw/ftplocate/
#
#################################################################

if ( defined($ENV{'REMOTE_ADDR'}) ) { exit; }	# cgi not allowed

use Socket;
use IO::Socket;
use FileHandle;
require ("ftp.pl");
unshift(@INC, substr($0, 0, rindex($0,'/')) );	# add prog path to INC
require ("config");

############################# sub ##############################

sub timed_ftpdirread {
  my $site=$_[0];
  my $ret=0;

  eval {
    local $SIG{ALRM} = sub { die "alarm\n" }; # NB: \n required
    alarm $FTPTIMEOUT;
    if ( $_=<ftp::NS> ) {
      $ret=1;
    } else {
      $ret=0;
    }
    alarm 0;
  };
  if ($@) {	# eval error, it means timeout
    print timestr()." flcollect - ls -lR $site timeout\n";
    print {$LOG} timestr()." flcollect - ls -lR $site timeout\n";
    return 0;		
  }
  return $ret;
}

sub dirstr {
  my $dir=$_[0];

  $dir=~s!^\./!!;	# remove ./ at start
  $dir=~s!/\./!/!g;	# replace /./ with /
  $dir=~s!/+!/!g;	# replace multiple / with one /
  $dir=~s!^/!!;		# remove / at start
  $dir=~s!/$!!;		# remove / at end

  if ( $dir eq "" ) {	
    return $dir;	# since $dir will be used as $dir/$fname, we use "" for / case
  } else {
    return "/$dir";
  }
}

sub ftp_ls {
  my ($site, $dir, $user, $pwd)=@_;

  print timestr()." flcollect - ls -lR $site\n";
  print {$LOG} timestr()." flcollect - ls -lR $site\n";

  # ls -lR from ftp server and convert to records
  if ( ftp::open("$site",21,1,2) !=1 ) {
    print timestr()." flcollect - $site connect failed\n";
    print {$LOG} timestr()." flcollect - $site connect failed\n";
    return 0;
  }
  if ( ftp::login("$user", "$pwd") !=1 ) {
    print timestr()." flcollect - $site login($user/$pwd) failed\n";
    print {$LOG} timestr()." flcollect - $site login($user/$pwd) failed\n";
    ftp::close();
    return 0;
  }
  ftp::cwd("/");
  ftp::dir_open("-lRa $dir");

  open(T, ">$TMPDIR/$site.ls.$$");
  while ( timed_ftpdirread($site) ) {
    print T $_;
  }
  close(T);

  ftp::dir_close();
  ftp::close();

  return 1;
}


sub ftp_lsfile {
  my ($site, $user, $pwd, $source)=@_;
  my $lsfile, $localfile, $ret;
  
  $_=$source;
  ($lsfile)=m!^file://(.+)\s*$!i;  $lsfile="/$lsfile";
  if ( $lsfile =~ m!\.gz$!i 
    || $lsfile =~ m!\.Z$!i ) {
    $localfile="$TMPDIR/$site.lsgz.$$";
  } elsif ( $lsfile =~ m!\.zip$!i ) {
    $localfile="$TMPDIR/$site.lszip.$$";
  } else {
    $localfile="$TMPDIR/$site.ls.$$";
  }

  print timestr()." flcollect - ftp $site ".filename($lsfile)." file\n";
  print {$LOG} timestr()." flcollect - ftp $site ".filename($lsfile)." file\n";

  $ret=ftpget($site, $user, $pwd, $lsfile, $localfile);
  if ( $ret==1 ) {
    return 1;
  } elsif ( $ret==-1 ) {
    print timestr()." flcollect - $site connect failed\n";
    print {$LOG} timestr()." flcollect - $site connect failed\n";
    return 0;
  } elsif ( $ret==-2 ) {
    print timestr()." flcollect - $site login($user/$pwd) failed\n";
    print {$LOG} timestr()." flcollect - $site login($user/$pwd) failed\n";
    return 0;
  } else {
    print timestr()." flcollect - $site $lsfile not found\n";
    print {$LOG} timestr()." flcollect - $site $lsfile not found\n";
    return 0;
  }
}


sub ls_2_filelist {
  my ($site, $dir, $xfertime)=@_;
  my ($currentdir, $ftype);
  my ($sitedir, $sitefile, $sitesize, $sitexfer, $siterate);
  my (@l, $mode, $size, $month, $mday, $time, $fname);

  $sitedir=0;
  $sitefile=0;
  $sitesize=0;
  $sitexfer=0;

  if ( -f "$TMPDIR/$site.ls.$$" ) {
    @l=stat("$TMPDIR/$site.ls.$$"); $sitexfer=$l[7];	# file size
    open(S, "$TMPDIR/$site.ls.$$");
    print timestr()." flcollect - convert ls into $site.gz\n";
    print {$LOG} timestr()." flcollect - convert ls into $site.gz\n";

  } elsif ( -f "$TMPDIR/$site.lsgz.$$" ) {
    @l=stat("$TMPDIR/$site.lsgz.$$"); $sitexfer=$l[7];	# file size
    open(S, "$CMD_GZIP -dc $TMPDIR/$site.lsgz.$$|");
    print timestr()." flcollect - convert lsgz into $site.gz\n";
    print {$LOG} timestr()." flcollect - convert lsgz into $site.gz\n";

  } elsif ( -f "$TMPDIR/$site.lszip.$$" ) {
    @l=stat("$TMPDIR/$site.lszip.$$"); $sitexfer=$l[7];	# file size
    open(S, "$CMD_UNZIP -p $TMPDIR/$site.lszip.$$|");
    print timestr()." flcollect - convert lszip into $site.gz\n";
    print {$LOG} timestr()." flcollect - convert lszip into $site.gz\n";

  } else {
    return 0;
  }

  open(T, ">$TMPDIR/$site.tmp.$$");
  $currentdir=dirstr($dir);
  while ( <S> ) {
    s/\s+$//;
    if (/(.+):\s*$/) {
      $currentdir=dirstr($1);
      next;
    } elsif ( /^total\s\d+/ ) {
      next;
    } else {
      @l=split(/\s+/);
      ($mode, $size, $month, $mday, $time, $fname)=
      	(@l[0], @l[$#l-4], @l[$#l-3], @l[$#l-2], @l[$#l-1], @l[$#l]);
      if ( $fname eq "" || $fname eq ".." || $fname eq "." ) { next; }

      $ftype=substr($mode,0,1);
      print T "$currentdir/$fname"."!!".$ftype."!!".$size."!!"."$month $mday $time\n";

      if ( $ftype eq "-" ) {
        $sitefile++;
        $sitesize=$sitesize+$size;
      } elsif ( $ftype eq "d" || $ftype eq "l" ) {
        $sitedir++;
      }
    }
  }
  close(T);
  close(S);

  unlink "$TMPDIR/$site.ls.$$";
  unlink "$TMPDIR/$site.lsgz.$$";
  unlink "$TMPDIR/$site.lszip.$$";

  if ( $sitefile < 5 ) {
    print timestr()." flcollect - $site data error\n";
    print {$LOG} timestr()." flcollect - $site data error\n";
    unlink "$TMPDIR/$site.tmp.$$";
    return 0;
  }
  `$CMD_SORT $TMPDIR/$site.tmp.$$|$CMD_GZIP >$FILELISTDIR/collect/$site.gz`;
  unlink "$TMPDIR/$site.tmp.$$";

  # summary
  open(S, ">$FILELISTDIR/collect/$site.summary");
  if ( $xfertime==0 ) { $xfertime=1; }
  $siterate=int($sitexfer/$xfertime);
  print S "$site!!$sitedir!!$sitefile!!$sitesize!!$siterate!!".timestr()."\n";
  close(S);

  return 1;
}


sub xfer_filelist {
  my ($site, $source)=@_;
  my ($url, $remote_sock, $type, $filename);

  print timestr()." flcollect - xfer $site filelist from $source\n";
  print {$LOG} timestr()." flcollect - xfer $site filelist from $source\n";

  $url="$source?cmd=zget_filelist&fsite=$site";
  $remote_sock=http_open($url);
  if ( ! $remote_sock) {
    print timestr()." flcollect - connect $source failed\n";
    print {$LOG} timestr()." flcollect - connect $source failed\n";
    return 0;
  }
  print $remote_sock "GET $url HTTP/1.0\n\n";

  while (<$remote_sock>) {
    s/\s+$//;	# remove \n or \r or both at line end
    if ( m!^Content-Type: (.*)! ) {
      $type=$1;
    } elsif ( m!^Content-Disposition:\s*attachment;\s*filename=(.*\.gz)!i ) {
      $filename=$1;
    } elsif ( $_ eq "" ) {
      last;
    }
  }

  if ( $filename eq "$site.gz" && lc($type) =~ "application/x-unknown" ) {
    open(F, ">$TMPDIR/$site.gz.tmp.$$");
    while (<$remote_sock>) { print F $_; }
    close(F);
    close($remote_sock);
    `$CMD_CP $TMPDIR/$site.gz.tmp.$$ $FILELISTDIR/collect/$site.gz`;
    unlink("$TMPDIR/$site.gz.tmp.$$");
    return 1;
  } else {
    print timestr()." flcollect - xfer $site filelist from $source format error\n";
    print {$LOG} timestr()." flcollect - xfer $site filelist from $source format error\n";
    close($remote_sock);
    return 0;
  }
}

sub xfer_summary {
  my ($site, $source)=@_;
  my ($url, $remote_sock, $type);

  print timestr()." flcollect - xfer $site summary from $source\n";
  print {$LOG} timestr()." flcollect - xfer $site summary from $source\n";

  $url="$source?cmd=get_summary&fsite=$site";
  $remote_sock=http_open($url);
  if ( ! $remote_sock) {
    print timestr()." flcollect - connect $source failed\n";
    print {$LOG} timestr()." flcollect - connect $source failed\n";
    return 0;
  }
  print $remote_sock "GET $url HTTP/1.0\n\n";

  while (<$remote_sock>) {
    s/\s+$//;	# remove \n or \r or both at line end
    if ( m!^Content-Type: (.*)! ) {
      $type=$1;
    } elsif ( $_ eq "" ) {
      last;
    }
  }

  if ( lc($type) =~ "text/plain" ) {
    while (<$remote_sock>) {
      chop;
      if ( $_ =~ "^$site!!" ) { 
        open(S, ">$FILELISTDIR/collect/$site.summary");
        print S "$_\n";
        close(S);
        close($remote_sock);
        return 1;
      }
    }
  }
  print timestr()." flcollect - xfer $site summary from $source format error\n";
  print {$LOG} timestr()." flcollect - xfer $site summary from $source format error\n";
  close($remote_sock);
  return 0;      
}


############################# main ##############################

$SIG{INT} = sub { 
  unlink("$FTPLOCATEDIR/$myname.RUNNING"); 
  print timestr()." user break!\n";
  exit 1;
};

$myname=filename($0);
if ( -f "$FTPLOCATEDIR/$myname.RUNNING") {
  print "Another $myname running! abort!\n";
  exit(1);
} else {
  open(T,">$FTPLOCATEDIR/$myname.RUNNING"); print T $$; close(T); # create tag
}

if ( $ARGV[0] ne "" ) {
  @ftpsite=@ARGV;
} else {
  @ftpsite=sitelist();
}

$LOG= FileHandle->new();
open ($LOG, ">>$SYSTEMLOG");
select($LOG); $| = 1; 	# unbuffer mode
select(stdout); $|=1;	# unbuffer mode

$success=0;

if ( ! -d "$FILELISTDIR" ) {
  mkdir("$FILELISTDIR", 0755);
}
if ( ! -d "$FILELISTDIR/collect" ) {
  mkdir("$FILELISTDIR/collect", 0755);
}

foreach $site (@ftpsite) {

  if ( -f "$FILELISTDIR/collect/$site.gz" &&
       -f "$FILELISTDIR/collect/$site.summary" ) {
    print timestr()." flcollect - $site.gz found, skip to next\n";
    print {$LOG} timestr()." flcollect - $site.gz found, skip to next\n";
    $success++;
    next;
  }

  ($dir, $user, $pwd, $source)=sitecfg($site);

  if ( $source eq "direct" ) {
    $t=time();
    if ( ftp_ls($site, $dir, $user, $pwd)==0 ) { next; }
    $xfertime=time()-$t;
    if ( ls_2_filelist($site, $dir, $xfertime)==0 ) { next; }
    $success++;

  } elsif ( $source =~ m!^file://! ) {
    $t=time();
    if ( ftp_lsfile($site, $user, $pwd, $source)==0 ) { next; }
    $xfertime=time()-$t;
    if ( ls_2_filelist($site, $dir, $xfertime)==0 ) { next; }
    $success++;

  } elsif ( $source =~ m!^http://! ) {
    if ( xfer_filelist($site, $source) 
      && xfer_summary($site, $source) ) {
      $success++;
    }

  } else {
    print timestr()." flcollect - invalid filelist source for $site\n";
    print {$LOG} timestr()." flcollect - invalid filelist source for $site\n";
  }
}

print timestr()." flcollect - done\n";
print {$LOG} timestr()." flcollect - done\n";
close($LOG);

unlink("$FTPLOCATEDIR/$myname.RUNNING");	# remove tag
if ( $success>0) {
  exit 0;
} else {
  exit 1;
}
