#! /usr/bin/perl -w

# vim:syntax=perl

use strict;
use lib '/usr/local/share/perl5';
use Lire::DlfSchema;
use Lire::Program qw( :msg :dlf );
use Lire::WWW::URL;

init_dlf_converter( "proxy" );
my $schema = eval { Lire::DlfSchema::load_schema( "proxy" ) };
lr_err( "failed to load proxy schema: $@" ) if $@;
my $dlf_maker =
  $schema->make_hashref2asciidlf_func( qw/time client_ip client_host
		    user duration
                    cache_result req_result bytes operation protocol
                    dst_host dst_port requested_url type result_src_code
                    result_src_ip result_src_host/ );

sub parse_line {
    my ( $line ) = @_;
    my %dlf = ();

    my ($urlstring, $source, $type);
    # 979992041.366    502 192.168.1.160 TCP_IMS_HIT/304 216 
    #   GET http://www.example.com/home2.html - DIRECT/www.example.net
    #   text/html

    (
        $dlf{'time'},
        $dlf{'duration'},
        $dlf{'client_ip'},
        $dlf{'cache_result'},    # to do: normalise this with ms_isa
        $dlf{'req_result'},
        $dlf{'bytes'},
        $dlf{'operation'},
        $urlstring,
        $dlf{'user'},
        $source,
        $type,
    ) = $line =~ m|^
        ([0-9]+\.[0-9]*)\s+      # time
        ([0-9]+)\s+              # elapsed
        ([^ ]+)\s+               # remotehost
        ([A-Z_]+)\/              # cache_result  (e.g. TCP_MISS)
        ([0-9-]+)\s+              # http_status (e.g. 503). Netcache may use -
        ([0-9]+)\s+              # bytes
        ([A-Z_]+)\s+             # method (e.g. GET)
        (\S+)\s+                 # url
        ([^ ]+)\s+               # rfc931
        ([^ ]+)\s+		 # Result Src Code/Peer 
				 # Some logs also contains -
        (.*)			 # mime type
    $|x or die "squid lexer failed on '$line'\n";

    $dlf{duration} = sprintf '%.3f', $dlf{duration} / 1_000;

    $dlf{client_host} = $dlf{client_ip};
    my $url = Lire::WWW::URL->new( $urlstring );
    $url->{'scheme'} and $dlf{'protocol'} = $url->{'scheme'};
    $url->{'host'} and $dlf{'dst_host'} = $url->{'host'};
    $url->{'path'} and $dlf{'requested_url'} = $url->{'path'};
    $url->{'port'} and $dlf{'dst_port'} = $url->{'port'};

    # Handle Netcache quoted MIME type
    if ( $type =~ /^"([^"]+)"\s*$/ ) {
	$type = $1;
    } 
    $dlf{'type'} = $type;

    my $peer;
    ( $dlf{'result_src_code'}, $peer ) = 
      $source =~ m|^([A-Z_]+)/		# result_src_code, aka 
					# cache_result_code (e.g.
					#  NONE or DIRECT)
	           ([-0-9a-zA-Z\.]+)	# peer (could be host, ip or 
				        # can be just '-')
		  $|x;

    if ( $dlf{'result_src_code'} ) {
	if ( $peer =~ /^(\d{1,3}\.){3}\d{1,3}$/ ) {
	    # looks like an ip adress
	    $dlf{'result_src_ip'}   = $peer;
	    $dlf{'result_src_host'} = $peer;
	} else {
	    $dlf{'result_src_host'} = $peer;
        }
    } else {
	# FIXME: Is this really what Netcache - means?
	$dlf{'result_src_code'} = 'NONE';
    }
    $dlf_maker->( \%dlf );
}

my $lines       = 0;
my $dlflines    = 0;
my $errorlines  = 0;

init_dlf_converter( "proxy" );

while (<>) {
    lire_chomp();
    $lines++;

    # Skip empty lines
    next if /^\s*$/;
    
    # Skip Netcache directives
    next if /^#/;

    eval {
        my $dlf = parse_line( $_ );
        print join( " ", @$dlf), "\n";
        $dlflines++;
    };

    if ( $@ ) {
        lr_warn( $@ );
        lr_notice( qq{cannot convert line $. "$_" to proxy dlf, skipping} );
        $errorlines++;
    }
}

end_dlf_converter( $lines, $dlflines, $errorlines );

__END__

=pod

=head1 NAME

squid_access2dlf - convert squid logs to dlf format

=head1 SYNOPSIS

B<squid_access2dlf>

=head1 DESCRIPTION

squid_access2dlf expects an access.log log file, as produced by the SQUID Web Proxy
Cache ( http://www.squid-cache.org/ ) on it's stdin.

These log files are whitespace separated, columns are

 time elapsed remotehost code/status bytes method URL rfc931
  peerstatus/peerhost type

, a typical logline looks like e.g.

 979992041.366 502 192.168.1.160 TCP_IMS_HIT/304 216 GET 
  http://example.com/home2.html - NONE/- text/html

Meaning of fields is:

=over

=item time

A Unix timestamp as UTC seconds with a millisecond resolution.

=item elapsed

The elapsed time considers how many milliseconds the transaction busied the
cache.

=item remotehost

The IP address of the requesting instance, the client IP address.

=item code/status

This column is made up of two entries separated by a slash. This column encodes
the transaction result.

The cache result of the request contains information on the kind of request,
how it was satisfied, or in what way it failed.  The status part contains the
HTTP result codes with some Squid specific extensions. Squid uses a subset of
the RFC defined error codes for HTTP.

=over

=item TCP_HIT

A valid copy of the requested object was in the cache.

=item TCP_MISS

The requested object was not in the cache.

=item TCP_REFRESH_HIT

The requested object was cached but STALE. The IMS query for the object
resulted in "304 not modified".

=item TCP_REF_FAIL_HIT

The requested object was cached but STALE. The IMS query failed and the stale
object was delivered.

=item TCP_REFRESH_MISS

The requested object was cached but STALE. The IMS query returned the new
content.

=item TCP_CLIENT_REFRESH_MISS

The client issued a "no-cache" pragma, or some analogous cache control command
along with the request. Thus, the cache has to refetch the object.

=item TCP_IMS_HIT

The client issued an IMS request for an object which was in the cache and
fresh.

=item TCP_SWAPFAIL_MISS

The object was believed to be in the cache, but could not be accessed.

=item TCP_NEGATIVE_HIT

Request for a negatively cached object, e.g. "404 not found", for which the
cache believes to know that it is inaccessible.

=item TCP_MEM_HIT

A valid copy of the requested object was in the cache and it was in memory,
thus avoiding disk accesses.

=item TCP_DENIED

Access was denied for this request.

=item TCP_OFFLINE_HIT

The requested object was retrieved from the cache during offline mode. The
offline mode never validates any object, see offline_mode in squid.conf file.

=item  UDP_HIT

A valid copy of the requested object was in the cache.

=item UDP_MISS

The requested object is not in this cache.

=item UDP_DENIED

Access was denied for this request.

=item UDP_INVALID

An invalid request was received.

=item UDP_MISS_NOFETCH

During "C<-Y>" startup, or during frequent failures, a cache in hit only mode
will return either UDP_HIT or this code. Neighbours will thus only fetch hits.

=item NONE

Seen with errors and cachemgr requests.

=back

=item bytes

The size is the amount of data delivered to the client. Mind that this does not
constitute the net object size, as headers are also counted.

=item method

The request method to obtain an object.  Methods are:

 method    defined    cachabil.  meaning
 --------- ---------- ---------- --------------------
 GET       HTTP/0.9   possibly   object retrieval and
                                  simple searches.
 HEAD      HTTP/1.0   possibly   metadata retrieval.
 POST      HTTP/1.0   CC or Exp. submit data (to a
                                  program).
 PUT       HTTP/1.1   never      upload data (e.g. to
                                  a file).
 DELETE    HTTP/1.1   never      remove resource (e.g.
                                  file).
 TRACE     HTTP/1.1   never      appl. layer trace of
                                  request route.
 OPTIONS   HTTP/1.1   never      request available
                                  comm.  options.
 CONNECT   HTTP/1.1r3 never      tunnel SSL connection.
 ICP_QUERY Squid      never      used for ICP based
                                  exchanges.
 PURGE     Squid      never      remove object from
                                  cache.
 PROPFIND  rfc2518    ?          retrieve properties
                                  of an object.
 PROPATCH  rfc2518    ?          change properties of
                                  an object.
 MKCOL     rfc2518    never      create a new
                                  collection.
 MOVE      rfc2518    never      create a duplicate of
                                  src in dst.
 COPY      rfc2518    never      atomically move src
                                  to dst.
 LOCK      rfc2518    never      lock an object
                                  against
                                  modifications.
 UNLOCK    rfc2518    never      unlock an object.

=item URL

This column contains the URL requested. Please note that the log file may
contain whitespaces for the URI.  In the Lire DLF, the URL is split up in
proto, host and path.

=item rfc931

The eigth column may contain the ident lookups for the requesting client.

=item peerstatus/peerhost

results from neigbouring caches.

=item type

The content type of the object as seen in the HTTP reply header.

=back

=head1 VERSION

$Id: squid_access2dlf.in,v 1.1 2002/08/15 21:08:49 vanbaal Exp $

=head1 AUTHORS

Joost Bekkers <joost@jodocus.org>, Joost van Baal <joostvb@logreport.org>

=head1 COPYRIGHT

Copyright (C) 2001 Joost Bekkers <joost@jodocus.org>
Copyright (C) 2001,2002 Stichting LogReport Foundation LogReport@LogReport.org

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
 
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
 
You should have received a copy of the GNU General Public License
along with this program (see COPYING); if not, check with
http://www.gnu.org/copyleft/gpl.html or write to the Free Software 
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111, USA.

=cut

# Local Variables:
# mode: cperl
# End:

