#!/usr/local/bin/perl
#################################################################
#
# + FtpLocate - FTP search engine
#
# by Distributed System Lab E.E. NCKU Taiwan 2000
# http://ftp.ee.ncku.edu.tw/ftplocate/
#
#################################################################

if ( defined($ENV{'REMOTE_ADDR'}) ) { exit; }	# cgi not allowed

use Socket;
require ("ftp.pl");
unshift(@INC, substr($0, 0, rindex($0,'/')) );	# add prog path to INC
require ("config");

############################# subs ##############################

sub listfile {
  my ($pattern,$opt,$output)=@_;
  my ($host, $path, $type, $size, $date, $key, $i);
  my (@ftpsite, %siteid, $siteid);

  @ftpsite=sitelist();
  for ( $i=0; $i<=$#ftpsite; $i++ ) {
    $siteid{$ftpsite[$i]}=sprintf("%03d", $i);
  }

  open(G, "$CMD_GLIMPSE $opt -y -H $FILELISTDIR -e '$pattern' |");
  open(T, ">>$output");

  $i=0;
  while ( <G>) {
    chop;
    ($host,$path,$type,$size,$date)=m#.*/(.*)/\d+:\s*(.*)!!(.)!!(\d+)!!(.*)#;

    # size too small, contains no useful description
    if ( $size < 8 ) { next; }
    # only file is wanted
    if ( $type eq "d" ) { next; }

    $key=prefixname($path)."/".filename($path);
    $siteid=$siteid{$host};
    print T "$key:$siteid:$size:$host:$path\n"; 
    $i++;
  }

  close(T);
  close(G);
}   

############################# main ##############################

$SIG{INT} = sub { 
  unlink("$FTPLOCATEDIR/$myname.RUNNING"); 
  print timestr()." user break!\n";
  exit 1;
};

if ( -f "$FTPLOCATEDIR/flindex.pl.RUNNING") {
  print "flindex.pl is running, description file list is not available, abort!\n";
  exit(1);
}

$myname=filename($0);
if ( -f "$FTPLOCATEDIR/$myname.RUNNING") {
  print "Another $myname running! abort!\n";
  exit(1);
} else {
  open(T,">$FTPLOCATEDIR/$myname.RUNNING"); print T $$; close(T); # create tag
}

open (LOG, ">>$SYSTEMLOG");
select(LOG); $| = 1; 	# unbuffer mode
select(stdout); $|=1;	# unbuffer mode

# find files containing descrptions information

print timestr()." dcollect - get 00index.txt list\n";
print LOG timestr()." dcollect - get 00index.txt list\n";
listfile('/0(0|0-|0_)index.txt\!\!', '-i',    "$TMPDIR/dcollect.tmp.$$");

print timestr()." dcollect - get 00index list\n";
print LOG timestr()." dcollect - get 00index list\n";
listfile('/0(0|0-|0_)index\!\!',     '-i',    "$TMPDIR/dcollect.tmp.$$");

print timestr()." dcollect - get INDEX list\n";
print LOG timestr()." dcollect - get INDEX list\n";
listfile('/INDEX\!\!',              '-w',    "$TMPDIR/dcollect.tmp.$$");

print timestr()." dcollect - get *.lsm list\n";
print LOG timestr()." dcollect - get *.lsm list\n";
listfile('\.lsm\!\!',              '-i -w', "$TMPDIR/dcollect.tmp.$$");

print timestr()." dcollect - get rfc-index.txt list\n";
print LOG timestr()." dcollect - get rfc-index.txt list\n";
listfile('/rfc-index\.txt\!\!',         '', "$TMPDIR/dcollect.tmp.$$");

print timestr()." dcollect - get internet draft index.txt list\n";
print LOG timestr()." dcollect - get internet draft index.txt list\n";
listfile('/1id-index\.txt\!\!',         '', "$TMPDIR/dcollect.tmp.$$");


print timestr()." dcollect - sort description file list\n";
print LOG timestr()." dcollect - sort description file list\n";

`$CMD_SORT -f -t: +0 +1 $TMPDIR/dcollect.tmp.$$|$CMD_UNIQ -i > $DESCDIR/collect.lst`;
unlink("$TMPDIR/dcollect.tmp.$$");


print timestr()." dcollect - rename old description files\n";
print LOG timestr()." dcollect - rename old description files\n";

if ( ! -d "$DESCDIR" ) {
  mkdir("$DESCDIR", 0755);
}
if ( ! -d "$DESCDIR/collect" ) {
  mkdir("$DESCDIR/collect", 0755);
}
if ( ! -d "$DESCDIR/collect/old" ) {
  mkdir("$DESCDIR/collect/old",0755);
}

# mv collect/* to collect/old/*
opendir(D, "$DESCDIR/collect"); @f=readdir(D); closedir(D);
foreach $f (@f) {
  if ( $f ne "."  && $f ne ".." && $f ne "old" ) {
    rename("$DESCDIR/collect/$f", "$DESCDIR/collect/old/$f");
  }
}

# get id/pwd for each site
foreach (sitecfglist()) {
  ($site, $dir, $user, $pwd, $source)=split(/,/, $_);
  $user{$site}=$user;
  $pwd{$site}=$pwd;
}

print timestr()." dcollect - collect description files\n";
print LOG timestr()." dcollect - collect description files\n";

open(F, "$DESCDIR/collect.lst");

$line=0;
$currentkey="";
$local=0; $skip=0; $success=0; $fail=0;

while ( <F> ) {
  chop;
  $line++;
  ($key,$siteid,$size,$host,$path)=split(/:/, $_,5);

  if ( lc($key) eq lc($currentkey) ) {
    $skip++;
    next; 
  }
    
  $f=$key; $f=~s!/!#!g;

  # get filesize at local
  @l=(); @l=stat("$DESCDIR/collect/old/$f"); $localsize=$l[7]; 
  if ( $localsize == $size ) {
    rename("$DESCDIR/collect/old/$f", "$DESCDIR/collect/$f");
    print timestr()." dcollect - local get $f $host:$path $size\n";
    print LOG timestr()." dcollect - local get $f $host:$path $size\n";
    $currentkey=$key; 	# set newkey after get success
    $local++;

  } elsif ( ftpget($host, $user{$site}, $pwd{$site}, $path, "$DESCDIR/collect/$f") ==1 ) {
    print timestr()." dcollect - ftp get $f $host:$path $size\n";
    print LOG timestr()." dcollect - ftp get $f $host:$path $size\n";
    $currentkey=$key; 	# set newkey after get success
    $success++

  } else {
    print timestr()." dcollect - fail get $f $host:$path $size\n";
    print LOG timestr()." dcollect - fail get $f $host:$path $size\n";
    $fail++;
  }
}
close(F);

print timestr()." dcollect - clear old description files\n";
print LOG timestr()." dcollect - clear old description files\n";

opendir(D, "$DESCDIR/collect/old"); @f=readdir(D); closedir(D);
chdir("$DESCDIR/collect/old"); unlink(@f);

print timestr()." dcollect - done. total $line, local $local, skipped $skip, ftpok $success, ftperr $fail\n";
print LOG timestr()." dcollect - done. total $line, local $local, skipped $skip, ftpok $success, ftperr $fail\n";
close(LOG);

unlink("$FTPLOCATEDIR/$myname.RUNNING");	# remove tag
