#!/bin/sh -e

# $Id: lr_run_tests.in,v 1.10 2002/08/18 19:50:55 flacoste Exp $

#
# Copyright (C) 2001, 2002 Stichting LogReport Foundation LogReport@LogReport.org
# 
#     This program is free software; you can redistribute it and/or modify
#     it under the terms of the GNU General Public License as published by
#     the Free Software Foundation; either version 2 of the License, or
#     (at your option) any later version.
# 
#     This program is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU General Public License for more details.
# 
#     You should have received a copy of the GNU General Public License
#     along with this program (see COPYING); if not, check with
#     http://www.gnu.org/copyleft/gpl.html or write to the Free Software 
#     Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111, USA.
#

PROGRAM=lr_run_tests

# dereference sysconfdir's prefix dependency
prefix="/usr/local"
etcdir="${prefix}/etc/lire"
. $etcdir/profile_lean
. $etcdir/defaults

tag="all all ${LR_ID:-UNSET} $PROGRAM"

echo >&2 "$tag info started with $*"

if test $# -lt 1
then
  echo >&2 "$tag err expecting argument. syntax: $PROGRAM testdir"
  exit 1
fi
if ! test -d "$1"
then
    echo >&2 "$tag err directory $d doesn't exists"
    exit 1
fi
# Make it absolute
testdir="`cd $1 && pwd`"
shift

for d in logs mail output tmp
do
    if test ! -d $testdir/$d
    then
	echo >&2 "$tag err no $d/ subdirectory in $testdir. Exiting"
	exit 1
    fi
done

if ls -- $testdir/logs/*.log > /dev/null 2>&1
then
    logs=`ls -- $testdir/logs/*.log`
else
    echo >&2 "$tag warning no logs in $testdir/logs were found"
    logs=""
fi

if ls -- $testdir/mail/*.mail > /dev/null 2>&1
then
    mail=`ls -- $testdir/mail/*.mail`
else
    echo >&2 "$tag warning no mail in $testdir/mail were found."
    mail=""
fi

# Create the output directories
for d in xml html html-i pdf pdf-i xhtml xhtml-i txt excel95 mail-txt mail-pdf mail-html mail-excel95
do
    # Cleanup from previous run
    rm -fr $testdir/output/$d
    mkdir -p $testdir/output/$d
done

# Create a temporary Lire configuration directory
mkdir -p $testdir/tmp/.lire/etc
mkdir -p $testdir/tmp/.lire/data

cat > $testdir/tmp/.lire/etc/defaults <<EOF
LR_DEBUG=
LR_KEEP=
LR_ARCHIVE=
LR_INCLUDEIMAGES=
LR_ARCHIVEDIR="$testdir/tmp/.lire/data"
EOF
HOME="$testdir/tmp"
export HOME

tested=
# Create an XML report and ASCII report for each log type
for l in $logs
do
    service=`basename $l |sed -e 's,[-.].*$,,'`
    base=`basename $l .log`
    outputbase="$testdir/output/xml/$base"

    echo >&2 "$tag info generating XML report for $base"
    lr_run lr_log2report -o xml $service < $l \
	3> $outputbase.xml 2>&1 1>&3 3>&- | tee $outputbase.log

    if grep ' warning \| crit \| err ' $outputbase.log > /dev/null 2> /dev/null
    then
	echo >&2 "$tag info XML report for $base generated warnings in $outputbase.log" 
    fi
    if test -z "$tested"
    then
	tested="^$service\$"
    else
	tested="$tested\\|^$service\$"
    fi
done

# Test other formats
for format in txt html xhtml pdf excel95
do
    if ! lr_check_prereq $format 2> /dev/null
    then
	echo >&2 "$tag info skipping $format tests because of unavailable requirements"
	continue
    fi

    for l in $logs
    do
	service=`basename $l |sed -e 's,[-.].*$,,'`
	base=`basename $l .log`
	xmlreport="$testdir/output/xml/$base.xml"
	outputbase="$testdir/output/$format/$base"

	test -s $xmlreport || continue

	if test $format = "xhtml" || test $format = "html"
	then
	    ext="tar"
	elif test $format = "excel95"
	then
	    ext="xls"
	else
	    ext="$format"
	fi
	echo >&2 "$tag info generating $format report for $base"
	lr_run lr_xml2report -o $format $xmlreport 3> $outputbase.$ext \
	    2>&1 1>&3 3>&- | tee $outputbase.log
	if grep ' warning \| crit \| err ' $outputbase.log > /dev/null 2> /dev/null
	then
	    echo >&2 "$tag info $format report for $base generated warnings in $outputbase.log" 
	fi

	# Handle (X)HTML tar file
	test -s $outputbase.tar || continue
	olddir=`pwd`
	cd $testdir/output/$format
	tar xf $outputbase.tar
	mv report $base
	cd $olddir
	rm -f $outputbase.tar
    done

    test $format = txt || test $format = excel95 && continue

    # Check with images
    if ! lr_check_prereq -i $format 2> /dev/null
    then
	echo >&2 "$tag info skipping $format (with images) tests because of unavailable requirements"
	continue
    fi
    
    for l in $logs
    do
	service=`basename $l |sed -e 's,[-.].*$,,'`
	base=`basename $l .log`
	xmlreport="$testdir/output/xml/$base.xml"
	outputbase="$testdir/output/$format-i/$base"

	test -s $xmlreport || continue
	
	echo >&2 "$tag info generating $format report (with images) for $base"
	case $format in
	html|xhtml)
	    lr_run lr_xml2report -i -o $format $xmlreport 3> $outputbase.tar \
		2>&1 1>&3 3>&- | tee $outputbase.log
	    if grep ' warning \| crit \| err ' $outputbase.log > /dev/null 2> /dev/null
	    then
		echo >&2 "$tag info $format report (with images) for $base generated warnings in $outputbase.log" 
	    fi
	    test -s $outputbase.tar || continue
	    olddir=`pwd`
	    cd $testdir/output/$format-i
	    tar xf $outputbase.tar
	    mv report $base
	    cd $olddir
	    rm -f $outputbase.tar
	    ;;
	pdf)
	    lr_run lr_xml2report -i -o $format $xmlreport 3> $outputbase.pdf \
		2>&1 1>&3 3>&- | tee $outputbase.log
	    if grep ' warning \| crit \| err ' $outputbase.log > /dev/null 2> /dev/null
	    then
		echo >&2 "$tag info $format report (with images) for $base generated warnings in $outputbase.log" 
	    fi
	    ;;
	esac
    done
done

# Test most responder code, lr_spoold and lr_spool aren't tested by this test
for format in txt html pdf excel95
do
    if ! lr_check_prereq $format 2> /dev/null
    then
	echo >&2 "$tag info skipping mail $format tests because of unavailable requirements"
	continue
    fi
    if lr_check_prereq -i $format 2> /dev/null
    then
	echo "LR_INCLUDEIMAGES=1" >> $testdir/tmp/.lire/etc/defaults
    fi

    echo "LR_DEFAULT_OUTPUT_FORMAT=$format" >> $testdir/tmp/.lire/etc/defaults

    for m in $mail
    do
	service=`basename $m |sed -e 's,[-.].*$,,'`
	base=`basename $m .mail`
	outputbase=$testdir/output/mail-$format/$base

	echo >&2 "$tag info testing lr_processmail on $base"
	lr_run lr_processmail $service < $m 2>&1 | tee $outputbase.log
	if grep ' warning \| crit \| err ' $outputbase.log > /dev/null 2> /dev/null
	then
	    echo >&2 "$tag info $format report (with images) for $base generated warnings in $outputbase.log" 
	fi
    done
done

# Cleanup empty files
find $testdir/output -size 0 -exec rm {} \;
rm -fr $testdir/tmp/.lire
rm -fr $testdir/tmp/tmp

not_tested=`lr_check_service -l 2>/dev/null | grep -v "$tested"`
if test -n "$not_tested"
then
    echo >&2 "$tag info The following services weren't tested:" 
    echo >&2 "$not_tested"
else
    echo >&2 "$tag info All supported services were tested" 
fi

exit 0

POD=<<'EOPOD'

=pod

=head1 NAME

lr_run_tests - Run several tests on a Lire installation

=head1 SYNOPSIS

B<lr_run_tests> I<testdir>

=head1 DESCRIPTION

B<lr_run_tests> exercises several aspect of a Lire system. The I<testdir>
argument should be a directory which has the following directory structure

  testdir/
     logs/
     output/
     mail/
     tmp/

The F<logs> and F<mail> subdirectories contain the input that will be
used to test Lire.

=head2 TEST INPUT

The F<logs/> subdirectory should contain log files named
I<service>.log or I<service>-I<ext>.log. Example: F<ipchains.log>,
F<combined.log> or F<combined-1.log>.

The F<mail/> subdirectory should contain email messages that represent
requests to the online responder. The message should be named
I<service>.mail or I<service>-I<ext>.mail. Example: F<xferlog.mail>
or F<xferlog-1.mail>.

=head2 LR_LOG2REPORT TESTS

B<lr_run_tests> will create an XML report for each log file contained
in the F<logs> subdirectory. The XML reports will be saved under
F<output/xml/I<basename>.xml>.

An ASCII report will then be generated in
F<output/txt/I<basename>.txt>.

After that, if the requirements needed to generate HTML, XHTML, PDF or
Excel95 reports are installed, a report in that format will be created
in F<output/I<format>/I<basename>.I<format>>. If the requirements
needed to generate images are available, a report with images will
also be created under F<output/I<format>-i/I<basename>>. This will be
a directory in the case of the HTML or XHTML output format.

=head2 LR_PROCESSMAIL TESTS

All emails in the F<mail/> subdirectory will be processed. Make sure
that the input emails have a reachable email address in the From: or
Reply-To: header so that you can receive the result of the test.  Each
email should contain one log file, either as a (possibly gzipped)
attachment or as plain text in the email body.

=head2 LOGGING

The tests are run with I<LR_DEBUG>, I<LR_KEEP> and I<LR_ARCHIVE> set to off.
(See the Configuration Parameters section in the Lire User Manual for details
about these variables.)

All messages with a level of notice or higher will be saved in the
appropriate output subdirectory under the name I<basename>.log, where
I<basename> matches the input file. All those messages will also be
output on STDERR. Additionally, if an error file was generated while
the XML report was computed, you will find this file under
F<output/xml/I<basename>.error>.

Additionally, B<lr_run_tests> will output info level messages
specifying input files that generated warnings or higher level messages
or that had other problems.

=head2 USING LR_RUN_TESTS

You should check all generated reports intended for users (all except
the XML one) for sanity. Also, you should make sure that all error
messages are legitimate (they represent bogus input, really invalid
log lines or missing installed components.)

The idea of this script is to make it possible to run systematically a
bunch of tests and analyze the input later rather then the iterative
approach where it is easy to forget something.

=head1 SEE ALSO

lr_log2report(1), lr_processmail(1), documentation in the Lire User Manual

=head1 VERSION

This manpage: $Id: lr_run_tests.in,v 1.10 2002/08/18 19:50:55 flacoste Exp $

=head1 AUTHOR

Francis J. Lacoste <flacoste@logreport.org>

=head1 COPYRIGHT

Copyright (C) 2001, 2002 Stichting LogReport Foundation LogReport@LogReport.org
 
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
 
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
 
You should have received a copy of the GNU General Public License
along with this program (see COPYING); if not, check with
http://www.gnu.org/copyleft/gpl.html or write to the Free Software 
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111, USA.
=cut

EOPOD
