#################################################################
#
# + FtpLocate - FTP search engine
#
# by Distributed System Lab E.E. NCKU Taiwan 2000
# http://ftp.ee.ncku.edu.tw/ftplocate/
#
#################################################################

#############################################################################
####################### global depend on script name ########################
#############################################################################

$MASTERURL="http://ftp.ee.ncku.edu.tw/cgi-bin/ftplocate/flserv.pl";

$MYNAME=$ENV{'SCRIPT_NAME'};
$SCRIPT_flsummary=dirname($ENV{'SCRIPT_NAME'})."/flsummary.pl"; 
$SCRIPT_flsearch=dirname($ENV{'SCRIPT_NAME'})."/flsearch.pl"; 
$SCRIPT_dsearch=dirname($ENV{'SCRIPT_NAME'})."/dsearch.pl"; 
$SCRIPT_flmap=dirname($ENV{'SCRIPT_NAME'})."/flmap.pl"; 
$SCRIPT_fltop=dirname($ENV{'SCRIPT_NAME'})."/fltop.pl"; 
$SCRIPT_flhistory=dirname($ENV{'SCRIPT_NAME'})."/flhistory.pl"; 

$STR_dslab=qq|
<font size=-1>
<a href="$URL_ftplocate" target=_blank>
FtpLocate 2.01
</a>
&nbsp;
<i>
by
<a href="http://turtle.ee.ncku.edu.tw" target=_blank>Distributed System Lab</a>
<a href="http://www.ee.ncku.edu.tw" target=_blank>E.E.</a>
<a href="http://www.ncku.edu.tw" target=_blank>NCKU</a>
2000
</i>
|;



#############################################################################
########################### normal misc function ############################
#############################################################################

require 'sys/syscall.ph';

sub gettimeofday {
  my ($t, @t);

  $t = pack("LL", ()); 
  syscall( &SYS_gettimeofday, $t, 0);
  @t = unpack("LL", $t);
  return ($t[0]+$t[1]/1000000);
}

sub dirname {
  my $i;
  $i=rindex($_[0],'/');
  if ( $i > 0 ) {
    return substr($_[0], 0, $i);
  } else {
    return ".";
  }
}

sub filename {
  my $i;
  $i=rindex($_[0],'/');
  if ( $i >= 0 ) {
    substr($_[0], $i+1);
  } else {
    return($_[0]);
  }
}

sub timestr {
  my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst,$t);
  ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =localtime;
  $t=sprintf ("%4d/%02d/%02d %02d:%02d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
  $t;
}  

sub sizestr {
  my $size=$_[0];
  my ($s, $n);

  if ( $size>=1024*1024*1024 ) {
    $n=int($size/1024/1024/1024*100+0.5)/100;
    $s=$n."GB";
  } elsif ( $size>=1024*1024 ) {
    $n=int($size/1024/1024*10+0.5)/10;
    $s=$n."MB";
  } elsif ( $size>=1024 ) {
    $n=int($size/1024*10+0.5)/10;
    $s=$n."k";
  } else {
    $s=int($size)."byte";
  }
  $s;
}

sub clean_str {
  my $s=$_[0];
  $s =~ s!\s+! !g; $s =~ s!^\s*!!; $s =~ s!\s*$!!; 
  return $s;
}

sub ftpget {
  my ($site,$user,$pwd,$path,$localfname)=@_;
  my ($dname, $fname);

  if ( ftp::open("$site",21,1,2) !=1 ) {
#    print timestr()." - $site connect failed\n";
    return -1;
  }
  if ( ftp::login($user, $pwd) !=1 ) {
#    print timestr()." - $site login failed\n";
    ftp::close();
    return -2;
  }

  $dname=dirname($path); if ($dname eq "") { $dname="/"; }
  $fname=filename($path);
  if ( ftp::cwd($dname) !=1 ) {
#    print timestr()." - $site cwd($dname) failed\n";
    ftp::close();
    return -3;
  }

  ftp::type('I');
  if ( ftp::get($fname, $localfname, 1) !=1 ) {
#    print timestr()." - $site get($fname) failed\n";
    ftp::close();
    return -4;
  }

  ftp::close();
  return 1;
}

# http_open(url)
# return: a socket to web server or proxy
sub http_open {
  my ($server, $script, $proxyserver, $proxyport, $remote_sock);  

  $_=$_[0]; 
  ($server, $script)=m!http://([^/]+)(/.*)!i;
  if ( $server eq "" ) {  return 0; }	# not http

  $_=$ENV{'http_proxy'};
  ($proxyserver, $proxyport)=m!http://([^/]+):(\d+)/.*!i;

  if ($proxyserver ne "" ) {	# if defined proxy, try proxy first!
    $remote_sock=new IO::Socket::INET(	Proto=>'tcp',
					PeerAddr=>$proxyserver,
					PeerPort=>$proxyport,);
    if ( $remote_sock ) {
      $remote_sock->autoflush(1);
      return $remote_sock;
    }
  }

  $remote_sock=new IO::Socket::INET(	Proto=>'tcp',
					PeerAddr=>$server,
					PeerPort=>80,);
  $remote_sock->autoflush(1) if ( $remote_sock );
  return $remote_sock;
}

use Sys::Hostname;
sub default_domain {
  my ($hostname, $fqdn);

  open (R, "/etc/resolv.conf");		# from resolv.conf
  while (<R>) {
    if ( /domain\s+\.?(.*)\s?/i ) { close(R); return($1); }
  }
  close(R);

  $hostname=hostname();			# from hostname
  if ($hostname =~ /\./ ) {
    return( substr($hostname,index($hostname,'.')+1) );
  }

  open (R, "/etc/hosts");		# from hosts
  while (<R>) {
    if ( /$hostname\./i ) {
      foreach ( split(/\s+/) ) {
        if ( /$hostname\./i && (length($_)>length($fqdn)) ) { $fqdn=$_; }
      }
      close(R);
      return( substr($fqdn,index($fqdn,'.')+1) );
    }
  }
  close(R);
  return("");
}


sub ip2fqdn {
  my $ip=$_[0];
  my (@ip, $fqdn);

  $ip =~ s/\s//g;
  @ip=split(/\s*,\s*/, $ip);
  foreach (@ip) { 

    eval {
      local $SIG{ALRM} = sub { die "alarm\n" }; # NB: \n required
      alarm $DNSTIMEOUT;
      $fqdn=gethostbyaddr(inet_aton($_),AF_INET);
      alarm 0;
    };
    if ($@) {	# eval error, it means timeout
      next;
    }

    if ( $fqdn=~/\w/ && $fqdn ne "localhost" ) { # found a reasonable fqdn
      if ( $fqdn =~ /\./ ) {
        return($fqdn);
      } else {
        return ($fqdn.'.'.default_domain());
      }
    }
  }
  return($ip);
}

sub fqdn2ip {
  my $fqdn=$_[0];
  my $ip;

  $ip=inet_ntoa(scalar(gethostbyname($fqdn)));
  if ( $ip =~ /^\d+\.\d+\.\d+\.\d+$/ ) {
    return($ip);
  } else {
    return("127.0.0.1");
  }
}

sub my_ip {
  return fqdn2ip(hostname());
}

sub my_fqdn {
  my ($hostname, $domain);

  $hostname=hostname();
  if ( $hostname =~ /\./ ) {
    return $hostname;
  } else {
    return $hostname.'.'.default_domain();
  }
}

sub peer_ip {
  my ($hersockaddr, $family, $port, $heraddr, @a);

  $hersockaddr=getpeername(stdin);
  ($family, $port, $heraddr)=unpack('S n a4 x8',$hersockaddr);
  @a=unpack('C4',$heraddr);
  return "$a[0].$a[1].$a[2].$a[3]";
}

sub client_ip {
  my $ip;

  if ( defined($ENV{'REMOTE_ADDR'}) ) {			# web client
    if (defined $ENV{'HTTP_X_FORWARDED_FOR'} &&
        $ENV{'HTTP_X_FORWARDED_FOR'} !~ /^10\./ &&
        $ENV{'HTTP_X_FORWARDED_FOR'} !~ /^172\.[1-2][6-9]\./ &&
        $ENV{'HTTP_X_FORWARDED_FOR'} !~ /^192\.168\./ &&
        $ENV{'HTTP_X_FORWARDED_FOR'} !~ /^127\.0\./ ) {
      $ip=$ENV{'HTTP_X_FORWARDED_FOR'};
    } else {
      $ip=$ENV{'REMOTE_ADDR'};
    }
  } elsif ( defined ($ENV{'inetd_dummy'}) ) {		# socket client
    $ip=peer_ip();
  } else {						# local client
    $name=my_ip();
  }
}

$LEFT="&lt;";
$RIGHT="&gt;";
$_pagemin=0; $_pagemax=0; $_page=0;
sub init_pageline {
  my ($list_total, $n_item, $n_page, $page, $url)=@_;
  my ($listmin, $listmax);

  $urlstr=$url;
  $_page=$page;

  $page_total=int(($list_total-1)/$n_item)+1;	#start from 1..total
  if ( $_page<1 ) { $_page=1; }
  if ( $_page>$page_total) { $_page=$page_total; }

  $_pagemin=int(($_page-1)/$n_page)*$n_page+1;
  $_pagemax=$_pagemin+$n_page-1;
  if ($_pagemax>$page_total) { $_pagemax=$page_total; }

  $listmin=($_page-1)*$n_item;
  $listmax=$listmin+$n_item-1;
  if ( $listmax>$list_total-1 ) { $listmax=$list_total-1; }

  return ($listmin, $listmax);
}

sub print_pageline {
  my ($n,$i);

  if ( $_pagemin>1 ) {
    $n=$_pagemin-1;
    print qq|<a href="$urlstr&page=$n" style="text-decoration: none">$LEFT</a>&nbsp;\n|;
  }
  for ($i=$_pagemin; $i<=$_pagemax; $i++) {
    if ( $i eq $_page ) {
      print qq|<a href="$urlstr&page=$i" style="text-decoration: none">[$i]</a>&nbsp;\n|;
    } else {
      print qq|<a href="$urlstr&page=$i" style="text-decoration: none">$i</a>&nbsp;\n|;
    }
  }
  if ( $_pagemax<$page_total ) {
    $n=$_pagemax+1;
    print qq|<a href="$urlstr&page=$n" style="text-decoration: none">$RIGHT</a>\n|;
  }
}

#############################################################################
####################### ftplocate specific misc function ####################
#############################################################################

sub prefixname {
  my $path=$_[0];
  my ($prefix, @p);

  # use parent+grandpa as prefix
  @p=split(/\//, $path);
  pop(@p); $prefix=pop(@p); $prefix=pop(@p)."/$prefix"; 

  return $prefix;
}

sub descstr {
  my $path=$_[0];
  my ($fname, $desc);

  $fname=lc(filename($path));
  $desc=$desc{prefixname($path)."/$fname"};
  if ( $desc eq "" ) { 
    $fname=~s/\.gz$//;
    $fname=~s/\.tar$//;
    $fname=~s/\.tgz$//;
    $desc=$desc{$fname};
  }
  return $desc;
}

sub safe_query_str {
  my $query=$_[0];

  $query =~ s/'/"/g; 
  # glimpse/agrep only accept pattern <32(including 1st and end ')
  $l=length($query);
  if ( $l >28 ) { $query=substr($query,$l-28,$l); }

  return $query;
}

sub glimpse_query_str {
  my $query=$_[0];

  # replace path in query with ^, cause we only index filename in glimpse
  $query=~s#[\w\d\.\-+=_\\]*/#\^#g;
  $query=~s#\^+#\^#g;
#  $query=~s#/#;\^#g; $query=~s/^;//; 
  return $query;
}
  
sub agrep_query_str {
  my $query=$_[0];

  # chnage ^ to / since we assume query are for filename
  $query=~s#^\^#/#;
  $query=~s#([,;\s])\^#$1/#g;
  # chnage $ to !! cause !! is at end of filename in index
  $query=~s#\$#\\!\\!#g;
  # if not a compounded query, limit it to filename
  if ($query !~ m#[;,~/]# ) {
    $query=$query.'[^/]*$';
  }
  return $query;
}

# call by reference! match array from 1st element
sub array_match {
  my ($ra_a, $ra_b)=@_;
  my $i;

  for ($i=0; $i<$#$ra_a; $i++) {
    if ( $$ra_a[$i] ne $$ra_b[$i] ) { return $i; }
  }
  return $i;
}

sub server_sort {
  my ($client, @server)=@_;
  my $i;

  @c=reverse(split(/\./,lc($client)));
  for ( $i=0; $i<=$#server; $i++) {
    @s=reverse(split(/\./,lc($server[$i])));
    $match{$server[$i]}=array_match(\@c, \@s);
    $num{$server[$i]}=$i;
  }
  return ( sort { $match{$b}<=>$match{$a}||$num{$a}<=>$num{$b} } @server);
}

sub sitecfglist {
  my @cfg;

  open(C, $SITECFG);
  while ( <C> ) {
    s/\s//g;
#    s/#.*$//;
    if ( /^#/ || /^$/ ) { next; }
    push (@cfg, $_);
  }
  close(C);
  return @cfg;
}

sub sitelist {
  my (@ftpsite, $site, $parm);

  foreach (sitecfglist()) {
    ($site, $parm)=split(/,/,$_,2);
    push (@ftpsite, $site);
  }
  return @ftpsite;
}

# this func is sloooow because each call will read whole cfg file.
# since it is only used in flcollect, speed doesn't matter
sub sitecfg {
  my $ftp=$_[0];
  my ($site, $dir, $id, $pwd, $source);

  foreach ( sitecfglist() ) {
    ($site, $dir, $id, $pwd, $source)=split(/,/, $_);
    if ( $site eq $ftp ) {
      last;
    }
  }
  if ($dir eq "" ) { $dir="/"; }
  if ($id eq "") { $id="anonymous"; }
  if ($pwd eq "") { $pwd="ftplocate@"; }
  if ($source eq "") { $source="direct"; }

  return ($dir, $id, $pwd, $source);
}

($LOCK_SH, $LOCK_EX, $LOCK_NB, $LOCK_UN) = (1,2,4,8);
sub loguser {
  my ($starttimestr,$usedtime,$client,$fsite,$founditem,$query,$cache)=@_;
  my $script;

  $usedtime=int($usedtime*100+0.5)/100;

  if ( $input{'page'} ne "" ) { return 1; }

  if ( $cache=~/cache/ ) {
    $cache="+";
  } else {
    $cache="-";
  }

  $script=filename($0); $script=~s/\.pl$//;

  open(ULOG, ">>$USERLOG");
  flock(ULOG, $LOCK_EX); seek(ULOG, 0, 2);
  print ULOG "$starttimestr $usedtime $client $script($fsite) $cache $founditem $query\n";
  flock(ULOG, $LOCK_UN);
  close(ULOG);
}

sub html_header_str {
  my $title=$_[0];

  if ( $title eq "" ) { 
    $title=$STR_title;
  }
  return qq|Content-type: text/html\n\n
<html>
<head>
   <meta http-equiv="Content-Type" content="text/html; charset=big5">
   <meta http-equiv="Pragma" content="no-cache">
   <title>$title</title>
</head>
<body bgcolor="#FFFFFF" background="$BACKGROUND">
<center>
|;
}

#############################################################################
################################ core function ##############################
#############################################################################

# do_flsearch(client,query,fsite,resultfname)
# return: in file $resultfname
sub do_flsearch {
  my ($client, $query, $fsite, $resultfname)=@_;
  my ($optf, $optw);
  my ($i, $q, @s);
  my ($setlist_str, $setall, $setfound, $setnum, $setused);
  my (%setcount,%setlist, %moreset);
  my ($site, @ftpsite);

  $fsite=~s/'//ig;   
  if ( $fsite eq "" ) {
    $optf="";
  } else {
    $optf="-F '$fsite'";		# for glimpse
    $MAXSET=$MAXSET*5;			# if query on 1 site, more detail
  }

  $q=glimpse_query_str(safe_query_str(clean_str($query)));
  if ( $q=~/\^[\w\d\.\-+=_\\]+\$/ ) {		# user specify word search
    $optw='-w';
  }
  open(G, "$CMD_GLIMPSE -i -Ny $optf $optw -H $FILELISTDIR -e '$q' 2>/$TMPDIR/flsearch.tmp.$$|");
  while(<G>) {
    chop; @s=split(/\//);
    $setnum=pop(@s); $site=pop(@s);
    if ($setcount{$site} < $MAXSET) {
      $setlist{$site}=$setlist{$site}."$site/$setnum ";
      $setcount{$site}++;
    } else {
      $moreset{$site}=1;
    }
  }
  close(G);

  $setfound=0; $setall=0;
  open(F, "$TMPDIR/flsearch.tmp.$$");
  unlink("$TMPDIR/flsearch.tmp.$$");
  while ( <F> ) {
    ($setfound, $setall)=/match\D+(\d+)\D+(\d+)/;
  }
  close(F);

  @ftpsite=server_sort($client,sitelist());
  $setused=0;
  foreach $site ( @ftpsite ) {
    if ($setused+$setcount{$site} < $MAXSET) {
      $setlist_str=$setlist_str.$setlist{$site};
      $setused=$setused+$setcount{$site};
    } else {
      @s=split(/ /, $setlist{$site});
      for ( $i=0; $i<=$#s; $i++) {
        $setlist_str=$setlist_str."$s[$i] ";
        $setused++;
        if ( $setused>$MAXSET ) { last; }
      }
      if ($i<$#s) { $moreset{$site}=1; };
    }
  }

  open (M, ">$CACHEDIR/$resultfname.more");
  print M "$setused/$setfound/$setall\n";
  foreach $site ( @ftpsite ) {
    if ( $moreset{$site}==1 ) { print M "$site\n"; }
  }
  close(M);

  chdir($FILELISTDIR);
  $q=agrep_query_str(safe_query_str(clean_str($query)));
  `$CMD_AGREP -i -e '$q' $setlist_str /dev/null>'$CACHEDIR/$resultfname'`;

  if ( $?!=0 && $q=~/ / ) {	# replace space with ; in pattern and try agign
    $q=safe_query_str(clean_str($query));
    $q=~s/\s/;/g;
    `$CMD_AGREP -i -e '$q' $setlist_str /dev/null>'$CACHEDIR/$resultfname'`;
  }

  return(1);
}

# do_search(query,resultfname)
# return: in file $resultfname
sub do_dsearch {
  my ($query, $resultfname)=@_;
  my $q;

  $q=safe_query_str(clean_str($query));
  # if not a compounded query, limit it to desc
  if ($q !~ m#[;,~/]# ) {
    $q='.*\!\!.*\!\!.*'.$q;
  }
  `$CMD_AGREP -i -e '$q' $DESCDIR/desc.txt>'$CACHEDIR/$resultfname'`;

  if ( $?!=0 && $q=~/ / ) {	# replace space with ; in pattern and try agign
    $q=safe_query_str(clean_str($query));
    $q=~s/\s/;/g;
    `$CMD_AGREP -i -e '$q' $DESCDIR/desc.txt>'$CACHEDIR/$resultfname'`;
  }

  return 1;
}

# summarylist(client)
# return(@summary), $summary[i]="$host!!$dcount!!$count!!$size!!$rate!!$date";
sub summarylist {
  my $client=$_[0];
  my (@ftpsite, @summary, $i);

  @ftpsite=server_sort($client,sitelist());
  for ( $i=0; $i<=$#ftpsite; $i++) {
    open(S, "$FILELISTDIR/$ftpsite[$i]/summary");
    $summary[$i]=<S>;
    chop($summary[$i]);
    close(S);
    if ( $summary[$i] eq "" ) {
      $summary[$i]="$ftpsite[$i]!!0!!0!!0!!0!!n/a";
    }
  }
  return(@summary);
}

# toplist(n)
# return(totalcount, topcount, @top), $top[i]="$query!!$found!!$count";
sub toplist {
  my $n=$_[0];
  my ($date, $time, $usedtime, $client, $script, $cache, $founditem, $query);
  my (@query, @top, %count, %found);
  my ($totalcount, $topcount, $i);

  open(ULOG, "$USERLOG");
  flock(ULOG, $LOCK_SH);
  foreach (<ULOG>) {
    chop;
    ($date, $time, $usedtime, $client, $script, $cache, $founditem, $query)=split(/\s/);
    $found{$query}=$founditem;
    $count{$query}++;
    $totalcount++;
  }
  flock(ULOG, $LOCK_UN);
  close(ULOG);
  @query=sort { $count{$b}<=>$count{$a} || $found{$b}<=>$found{$a} } keys %count;

  for ( $i=0; $i<$n && $i<=$#query; $i++) {
    $top[$i]="$query[$i]!!$found{$query[$i]}!!$count{$query[$i]}";
    $topcount=$topcount+$count{$query[$i]};
  }
  return($totalcount, $topcount, @top);
}

#############################################################################
############################## map related func #############################
#############################################################################

# return: "host!!info" 
# where info is "lastdate!!totalsize!!ftp1!!ftp2!!..."
sub mymap {
  my (@summary, $totalsize, $lastdate, @ftplist);
  my ($ftp,$dcount,$count,$size,$rate,$date);

  @summary=summarylist();
  $lastdate="";
  foreach (@summary) {
    ($ftp,$dcount,$count,$size,$rate,$date)=split(/!!/);
    push(@ftplist, $ftp);
    $totalsize+=$size;
    if ( $lastdate lt $date && $date ne "n/a" ) {
      $lastdate=$date;
    }
  }
  return my_fqdn()."!!$lastdate!!$FTPLOCATECGIDIR!!$totalsize!!".join('!!', @ftplist);
}

# maplist()
# return: @map, $map[i]="host!!lastdate!!flcgi!!totalsize!!ftp1!!ftp2!!...."
sub localmaplist {
  my @maplist;

  open(M, $MAPLOG);
  while (<M>) {
    chop; if ( ! /!!/ ) { next; }
    push (@maplist, $_);
  }
  close(M);
  return(@maplist);
}
  
sub remotemaplist {
  my ($url)=@_;
  my ($remote_sock, $content, $length, @remotemaplist);

  $remote_sock=http_open($url);
  return 0 unless ($remote_sock);

  $content=join("\n", localmaplist())."\n";
  $content="cmd=get_maplist&maplist=".CGI::escape($content);
  $length=length($content);

  print $remote_sock "POST $url HTTP/1.0
Content-type: application/x-www-form-urlencoded
Content-length: $length\n\n$content";

  while (<$remote_sock>) {
    chop;
    if ( m!^Content-Type: text/plain!i ) { last; }
  }
  while (<$remote_sock>) {
    chop;
    if ( $_ =~ "!!" ) {
      push (@remotemaplist, $_);
    }
  }
  close($remote_sock);

  return(@remotemaplist);
}


sub maplistmerge {
  my @maplist=@_;
  my ($host, $info, %map, @host, @s);

  foreach (@maplist) {
    ($host, $info)=split(/!!/,$_,2);
    if ( $host eq "" || $info !~ /!!/ ) { next; }	# format error

    # since the date is 1st field and date is fixed length in $info
    # the comparison of $info represents the comparison of $date
    if ( defined($map{$host}) && $map{$host} gt $info ) {
      next;				# old is newer, skip
    } else {
      $map{$host}=$info;
    }
  }

  @host=sort { $map{$b} cmp $map{$a} } keys %map;
  foreach (@host) {
    push(@s, "$_!!$map{$_}" );
  }
  return (@s);
}

sub maplistupdate {
  my @maplist=@_;

  open(M, ">$MAPLOG") || return(0);
  flock(M, $LOCK_EX); seek(M, 0, 2);
  foreach (@maplist) {
    print M "$_\n";
  }
  flock(M, $LOCK_UN);
  close(M);

  return 1;
}

1;
