#!/usr/local/bin/perl
#################################################################
#
# + FtpLocate - FTP search engine
#
# by Distributed System Lab E.E. NCKU Taiwan 2000
# http://ftp.ee.ncku.edu.tw/ftplocate/
#
#################################################################

if ( defined($ENV{'REMOTE_ADDR'}) ) { exit; }	# cgi not allowed

unshift(@INC, substr($0, 0, rindex($0,'/')) );	# add prog path to INC
require ("config");

############################# subs ##############################

%badword=(
'file', 1,
'note', 1,
'some', 1,
'Here', 1,
'To', 1,
'You', 1,
'If', 1,
'the', 1,
'Readme', 1,
'File', 1,
'These', 1,
'NOTE', 1,
'From', 1,
'NetWare', 1,
'Sample', 1,
'The', 1,
'Filename', 1,
'This', 1);

sub parse_00index {
  my ($infofile,$path,$output)=@_;
  my (@p, $me, $prefix, $fname, $desc);

  $me=filename($path); 
  $prefix=prefixname($path);

  open(I, $infofile);
  open(T, ">>$output");
  while ( <I> ) {
    chop;
    ($fname,$desc)=split(/[:\s]+/, $_, 2);
    if ( $fname =~ /^drwx/ ) {
      last;			# ls -lR file
    }

    $fname=~s!/$!!;
    if ( $fname !~ /^[\w\d]/ || length($fname)>40 || 
         $fname eq $me || $badword{$fname}==1) {
      next;			# fname to strange
    }

    $desc=~s!\d\d+/\d\d/\d\d\s*!!;	# remove date
    $desc=~s!DIR\s*!!;			# remove DIR
    $desc=~s!^EXE\s*!!;			# remove EXE at desc start
    $desc=~s!^[\d\.]+[Kk]\s*!!;		# remove size str at desc start

    $desc=~s/([\W])[Bb]y[\W].*/$1/;
    $desc=~s/=>.*/$1/;
    $desc=~s!,!!g;
    $desc=~s!-+!-!g;
    $desc=~s!\s\d\d\d+\s! !g; $desc=~s!^\d\d\d+\s!!;
    $desc=~s!\s[A-Z]\s! !; $desc=~s!^[A-Z]\s!!;
    $desc=~s/^[\-\s]*//;
    $desc=clean_str($desc);

    @p=split(/\s+/, $desc);
    if ( $#p<1 || $desc!~/\w/ ) {	# desc too short
      next;
    }

    $fname=lc($fname);			# lowercase
    if ( length($desc) >80 ) {
      $desc=substr($desc,0,75)."...";
    }
#    print "$fname!!$prefix!!$desc\n";
    print T "$fname!!$prefix!!$desc\n";
  }
  close(T);
  close(I);
}

sub parse_bsdindex {
  my ($infofile,$path,$output)=@_;
  my ($fname, $desc, @s);

  open(I, $infofile);
  open(T, ">>$output");
  while ( <I> ) {
    chop;
    @s=split(/\|/, $_);
    if ( $#s < 4 ) { last; }
    $fname=$s[0]; 
    $desc=$s[3];

    $fname=lc($fname);			# lowercase
    $desc=clean_str($desc);
    if ( length($desc) >80 ) {
      $desc=substr($desc,0,75)."...";
    }
#    print "$fname!!!!$desc\n";
    print T "$fname!!!!$desc\n";
  }
  close(T);
  close(I);
}

sub parse_lsm {
  my ($infofile,$path,$output)=@_;
  my ($prefix, $fname, $desc);

  open(I, $infofile);
  open(T, ">>$output");
  while ( <I> ) {
    chop;
    if ( /^Desc.*[:=]\s*(.*)/i || /^Disc.*[:=]\s*(.*)/i ) {
      $prefix=prefixname($path);
      $fname=filename($path);

      $desc=clean_str($1);
      if ( length($desc)<3 ) {
        $desc=<I>; chop($desc);
      }

      $fname=lc($fname);			# lowercase
      $desc=clean_str($desc);
      if ( length($desc) >80 ) {
        $desc=substr($desc,0,75)."...";
      }
#      print "$fname!!$prefix!!$desc\n";
      print T "$fname!!$prefix!!$desc\n";
      last;
    }
  }
  close(T);
  close(I);
}

sub parse_rfcindex {
  my ($infofile,$path,$output)=@_;
  my ($fname, $desc, $i);

  open(I, $infofile);
  open(T, ">>$output");
  while ( <I> ) {
    chop;
    if ( /^(\d\d\d\d)\s(.*)/ ) {
      $i=$1; $desc=$2; 
      while ( <I> ) {
        chop; if ( /^$/ ) { last; }
	$desc=$desc.$_;
      }

      $i=~s/^0+//; $fname="rfc$i.txt";
      $desc=clean_str($desc);
      $_=$desc; 
      while ( s/^(.*[a-z0-9])\. [A-Z\(].*$/$1/ ) {
        $desc=$_;
      }
      $desc=clean_str($desc);
      if ( length($desc) >80 ) {
        $desc=substr($desc,0,75)."...";
      }
#      print "$fname!!!!$desc\n";
      print T "$fname!!!!$desc\n";
    }
  }
  close(T);
  close(I);
}

sub parse_draftindex {
  my ($infofile,$path,$output)=@_;
  my ($line, $fname, $desc, $date);

  open(I, $infofile);
  open(T, ">>$output");
  while ( <I> ) {
    chop; if ( /\./ ) { last; }
  }

  while ( <I> ) {
    chop;
    $line=$_;

    while ( <I> ) {
      chop;
      if ( /^$/ || /^\s$/ || /^-*$/ ) {
        last;
      } else {
        $line=$line.$_;
      }
    }
    if ( $line =~ m#"(.*)".*(\d\d/\d\d/\d\d\d\d).*<(.*)># ) {
      $fname=$3;
      $date=$2; $date=~s/\s//;
      $desc=$1." $date";
      $desc=clean_str($desc);
      if ( length($desc) >80 ) {
        $desc=substr($desc,0,75)."...";
      }
#      print "$fname!!!!$desc\n";
      print T "$fname!!!!$desc\n";
    }
  }
  close(T);
  close(I);
}

############################# main ##############################

$SIG{INT} = sub { 
  unlink("$FTPLOCATEDIR/$myname.RUNNING"); 
  print timestr()." user break!\n";
  exit 1;
};

$myname=filename($0);
if ( -f "$FTPLOCATEDIR/$myname.RUNNING") {
  print "Another $myname running! abort!\n";
  exit(1);
} else {
  open(T,">$FTPLOCATEDIR/$myname.RUNNING"); print T $$; close(T); # create tag
}

open (LOG, ">>$SYSTEMLOG");
select(LOG); $| = 1; 	# unbuffer mode
select(stdout); $|=1;	# unbuffer mode

print timestr()." dindex - parsing collections to desc records\n";
print LOG timestr()." dindex - parsing collections to desc records\n";

open(L, "$DESCDIR/collect.lst");

$currentkey="";
while ( <L> ) {
  chop;
  ($key,$siteid,$size,$host,$path)=split(/:/, $_,5);

  if ( $key eq $currentkey ) { next; }
  
  $f=$key; $f=~s!/!#!g;
  if ( -f "$DESCDIR/collect/$f" ) {

    if ( $path=~ /\.lsm$/i ) {
      print timestr()." dindex - parse_lsm $f\n";
      print LOG timestr()." dindex - parse_lsm $f\n";
      parse_lsm("$DESCDIR/collect/$f", $path, "$TMPDIR/dindex.tmp.$$");

    } elsif ( $path=~m!bsd.*/packages[^/]*/INDEX$!i || 
              $path=~m!bsd.*/ports[^/]*/INDEX$!i ) {
      print timestr()." dindex - parse_bsdindex $f\n";
      print LOG timestr()." dindex - parse_bsdindex $f\n";
      parse_bsdindex("$DESCDIR/collect/$f", $path, "$TMPDIR/dindex.tmp.$$");

    } elsif ( $path=~m!/rfc-index\.txt$!i ) {
      print timestr()." dindex - parse_rfcindex $f\n";
      print LOG timestr()." dindex - parse_rfcindex $f\n";
      parse_rfcindex("$DESCDIR/collect/$f", $path, "$TMPDIR/dindex.tmp.$$");

    } elsif ( $path=~m!/1id-index\.txt$!i ) {
      print timestr()." dindex - parse_draftindex $f\n";
      print LOG timestr()." dindex - parse_draftindex $f\n";
      parse_draftindex("$DESCDIR/collect/$f", $path, "$TMPDIR/dindex.tmp.$$");

#    } elsif ( $path=~m!/0+[-_]?index.txt$!i || 
#              $path=~m!/0+[-_]?index$!i ) {
     } else {
      print timestr()." dindex - parse_00index $f\n";
      print LOG timestr()." dindex - parse_00index $f\n";
      parse_00index("$DESCDIR/collect/$f", $path, "$TMPDIR/dindex.tmp.$$");
    }

  $currentkey=$key;
  }
}

print timestr()." dindex - sort desc records\n";
print LOG timestr()." dindex - sort desc records\n";

`$CMD_SORT -f -t! +4 +0 +2 $TMPDIR/dindex.tmp.$$|$CMD_UNIQ -i>$DESCDIR/desc.txt`;
unlink("$TMPDIR/dindex.tmp.$$");

print timestr()." dindex - make desc database\n";
print LOG timestr()." dindex - make desc database\n";

unlink("$DESCDIR/desc.db");
open(D, "$DESCDIR/desc.txt");
dbmopen(%desc, "$DESCDIR/desc", 0644);
while (<D>) {
  ($fname,$prefix,$desc)=split(/!!/,$_,3);
  if ( $prefix ne "" ) {
    $desc{"$prefix/$fname"}=$desc;
  } else {
    $desc{"$fname"}=$desc;
  }
}
dbmclose(%desc);
close(D);

print timestr()." dindex - clear cache\n";
print LOG timestr()." dindex - clear cache\n";

if ( -d "$CACHEDIR" ) {
  opendir(D, "$CACHEDIR"); @f=readdir(D); closedir(D);
  chdir($CACHEDIR); unlink(@f);
} else {
  mkdir ($CACHEDIR, 0777);
}

print timestr()." dindex - done\n";
print LOG timestr()." dindex - done\n";
close(LOG);

unlink("$FTPLOCATEDIR/$myname.RUNNING");	# remove tag
