Main modifications | Notes | Files | graphdefang.pl | graphdefanglib.pl | My Graphdefang config | MIMEDefang filter | Mail stats | GNU GPL
I no longer use this myself, so it is entirely possible that it no longer works with my filter. It is possible that I will start to use this again, or maybe I will recreate it from scratch using a SQL database, or something...
I've done some modifications to GraphDefang. You can view the result of the at our mail stats page if you're curious, and you can view my version of the index.php (the stuff that shows my graphs).
As there's allways some possibility that others might want what I wanted, I've also made the modified stuff available here.
$senderdomain and $recipientdomain to the collected data values.
yearly_totals
monthly_totals
daily_totals
hourly_totals
weekdaily_totals
yeardaily_totals
totals (only works with graphs with no axis).
yearly_average (only works with graphs with no axis)
monthly_average
daily_average
hourly_average
weekdaily_average
yeardaily_average
pie has been added (only works with summary graphs).
stacked_area because I like it.
draw_*_graph() functions with one draw_graph().
all together with other data types,
the other data types will be excepted from the graph.
hbar and stacked_hbar
(h stands for horizontal), because they can be useful.
compute_bars_sz:
If this is true for a horizontal bar chart,
y_graph_size specifies the bredth of the bars, and the actual height of of
graph will be computed depending on the number of bars.
legend_columns:
Specifies the number of columns in the legend, and is necessary for
add_legend_sz to work.add_legend_sz:
If this is true for a graph with a legend, the size of the legend will be added to
the graphs height.
reversed:
If this is true, the graph values will be sorted in reverse order.
Regards
/Jonas Eckerman
#!/usr/bin/perl -w
# $Id: graphdefang.pl,v 1.21.03 2004/07/04 16:11:36 jonas Exp $
#
# GraphDefang -- a set of tools to create graphs of your mimedefang
# spam and virus logs.
#
# Written by: John Kirkland
# jpk@bl.org
#
# Copyright (c) 2002-2003, John Kirkland
#
# Modified by Jonas Eckerman, 2003-2004
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#=============================================================================
use strict;
use vars qw($MYDIR $OUTPUT_DIR $SUMMARYDB $VALUEDB $TEMPDATABASE $QUIET $NODB $DATAFILE @DATAFILES @GRAPHS %TZ $WHOLE_HOURS);
# Argument parsing
use Getopt::Long;
use Pod::Usage;
$QUIET = 0; # No output
$NODB = 0; # Don't use SummaryDB, just produce charts from logfile
my $trim = 0; # Trim database
my $nomax = 0; # Ignore max date/time
my $help = 0; # Show help?
my $man = 0; # Show bigger help?
my $file; # Log file to parse (optional)
my $forcegraphs = 0; # Create graphs even if there's no new lines
my $nographs = 0; # Don't create graphs.
my $nosummarize = 0; # Don't summarize log data.
my $rewritedb = 0; # Rewrite database
my $backupto; # Backup database to
my $listdata = 0; # List database to stdout
my $countdata = 0; # Count the database
GetOptions( 'quiet' => \$QUIET,
'nodb' => \$NODB,
'trim' => \$trim,
'nomax' => \$nomax,
'help|?' => \$help,
'man' => \$man,
'forcegraphs' => \$forcegraphs,
'nographs' => \$nographs,
'nosummarize' => \$nosummarize,
'file=s' => \$file,
'rewrite' => \$rewritedb,
'list-data' => \$listdata,
'count-data' => \$countdata,
'backup-to=s' => \$backupto
) or pod2usage(2);;
pod2usage(1) if $help;
pod2usage(-exitstatus => 0, -verbose => 2) if $man;
# Get the directory from where graphdefang.pl is running
use File::Basename ();
($MYDIR) = (File::Basename::dirname($0) =~ /(.*)/);
# Get graph configurations
require("$MYDIR/graphdefang-config");
# Require the graphdefang library file
require ("$MYDIR/graphdefanglib.pl");
#
# Path to summary database
#
#$SUMMARYDB = "/var/spool/MIMEDefang/graphdefang.dwhdb";
$SUMMARYDB = "/var/spool/MIMEDefang/graphdefang.summarydb";
$VALUEDB = "/var/spool/MIMEDefang/graphdefang.valuedb";
$TEMPDATABASE = 0;
# Do we do a database trim?
if ($trim) {
print "Beginning SummaryDB Trim\n" if (!$QUIET);
trim_database();
print "Completed SummaryDB Trim\n" if (!$QUIET);
print "Beginning ValueDB Trim\n" if (!$QUIET);
valuedb trim_values();
print "Completed ValueDB Trim\n" if (!$QUIET);
save_database();
#exit;
}
# Count records?
copy_database('#') if ($countdata);
# Rewrite databases?
copy_database(0) if ($rewritedb);
# Backup databases?
copy_database($backupto) if (defined($backupto));
# List all data?
copy_database('|') if ($listdata);
my $NewLines = 0;
my $stopat;
if ($WHOLE_HOURS) {
$stopat = get_unixtime_by_timesummary('hourly',time());
}
if (!$nosummarize) {
print "Summarizing\n" if (!$QUIET);
# Did the user specify a file on the command line?
$DATAFILE = $file if (defined($file));
if ($DATAFILE) {
# Open DATAFILE and Summarize It
$NewLines = read_and_summarize_data($DATAFILE, $nomax, $stopat);
#print "\tNo valid mimedefang logs in $DATAFILE\n" if (!($NewLines || $QUIET));
} elsif (@DATAFILES) {
my $NL;
foreach my $datafile (@DATAFILES) {
$NL = read_and_summarize_data($datafile, $nomax, $stopat);
#print "\tNo valid mimedefang logs in $datafile\n" if (!($NL || $QUIET));
$NewLines+=$NL;
}
} else {
# No DATAFILE or DATAFILES specified!
die "No DATAFILES specified on the command line or in your config file";
}
save_database() if ($NewLines);
}
if ($forcegraphs || ($NewLines && !$nographs)) {
print "Graphing\n" if (!$QUIET);
# Draw graphs
foreach my $settings (@GRAPHS) {
graph(\%{$settings},$stopat);
}
}
# Close database if it's open
close_database();
__END__
=head1 graphdefang.pl
Application for generating graphs from mimedefang log files.
=head1 SYNOPSIS
graphdefang.pl [options]
Options:
--help brief help message
--man full documentation
--quiet quiet output
--nodb do not update SummaryDB
--trim trim the SummaryDB
--nomax ignore the max date/time in SummaryDB
--file optional log file to parse
--forcegraphs create graphs even if there's no new lines
--nographs don't create graphs.
--nosummarize don't summarize log data.
--rewrite rewrite databases.
--list-data list saved data.
--count-data count records in the databases.
--backup-to backup databases to text file.
If called with no options, graphdefang.pl will parse the
logfile as defined by the $DATAFILE variable.
=head1 OPTIONS
=over 8
=item B<--help>
Print a brief help message and exits.
=item B<--man>
Prints the manual page and exits.
=item B<--quiet>
Do not produce status output from mimedefang.pl.
=item B<--nodb>
Do not use nor update the SummaryDB, just parse the file and draw graphs from it.
=item B<--trim>
Trim the SummaryDB to cut out old data. It trims out:
1. hourly data older than 1.25x$NUM_HOURS_SUMMARY hours
2. daily data older than 1.25x$NUM_DAYS_SUMMARY days
3. all but top 25 sender, recipient, value1, value2, subject values
for all dates prior to the current hour, day, and month..
=item B<--nomax>
Ignore the max date/time in the SummaryDB; add all lines from the parsed
file to the database.
=item B<--file>
Optional log file to parse. If this option is not set, graphdefang
will use the $DATAFILE variable.
=item B<--forcegraphs>
Create the graphs even if no new lines were found in the log.
=item B<--nographs>
Don't create any graphs.
=item B<--nosummarize>
Don't read the log file(s) for new data.
=back
=head1 DESCRIPTION
B<graphdefang.pl> will read a file that contains syslog messages from
mimedefang, update its internal summary database, and produce graphs
as requested by the user.
=cut
#!/usr/bin/perl -w
# $Id: graphdefanglib.pl,v 1.47.05 2004/06/04 16:11:55 jonas Exp $
#
# GraphDefang -- a set of tools to create graphs of your mimedefang
# spam and virus logs.
#
# Written by: John Kirkland
# jpk@bl.org
#
# Copyright (c) 2002-2003, John Kirkland
#
# Modified by Jonas Eckerman, 2003-2004
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#=============================================================================
use strict;
use Time::Local;
use Time::Zone;
use Date::Parse;
use Date::Format;
use File::ReadBackwards;
use Fcntl;
use Tie::DNS;
use DB_File;
use DWH_File qw(DB_File);
use File::Copy;
use Date::Set;
use Date::ICal;
use GD::Graph::linespoints;
use GD::Graph::bars;
use GD::Graph::hbars;
use GD::Graph::area;
use GD::Graph::pie;
# X and Y Graph Sizes in pixels
my $X_GRAPH_SIZE = 700;
my $Y_GRAPH_SIZE = 300;
# Number of hours, days, and months in the hourly, daily, and monthly charts, respectively.
my $NUM_HOURS_SUMMARY = 48;
my $NUM_DAYS_SUMMARY = 60;
my $NUM_MONTH_SUMMARY = 24;
# The dataset. Global so we can untie it in the END block if something bad happens.
use vars qw(%data %values $database_open %dns);
$database_open = 0;
sub put_value($) {
my ($v) = @_;
return undef if (!defined($v));
return 0 if (!$v);
my $n = $values{"v2n|$v"};
if (!$n) {
$values{'count'} ++;
$n = $values{'count'};
$values{"n2v|$n"} = $v;
$values{"v2n|$v"} = $n;
}
return $n;
}
sub get_value($) {
my ($n) = @_;
return undef if (!defined($n));
return '' if (!$n);
return $values{"n2v|$n"};
}
sub del_value($) {
my ($n) = @_;
return if (!$n);
my $v = $values{"n2v|$n"};
delete $values{"v2n|$v"};
delete $values{"n2v|$n"};
}
sub database_name() {
return "$SUMMARYDB.work" if ($TEMPDATABASE);
return $SUMMARYDB;
}
sub valuebase_name() {
$VALUEDB = "$SUMMARYDB.values" if (!$VALUEDB);
return "$VALUEDB.work" if ($TEMPDATABASE);
return $VALUEDB;
}
sub copy_counter ($$$$$) {
my ($f,$b,$c1,$c2,$c3) = @_;
if ($f || ($c1 % 37) == 0) {
my $s = '';
if ($c1>0) {
my $e = time()-$b;
$s = ' - ';
if ($e) {
$s .= int($c1/$e)."r/s";
} else {
$s .= $c1."r/s";
}
}
print STDERR "\r$c1"."r, $c2"."cr, $c3"."vr$s ";
}
}
sub trim_values($) {
my %valuerefs = ();
my $dvcnt = 0;
open_database();
while (my ($k,$v) = each(%values)) {
if ($k =~ /^n2v\|(.*)$/) {
$valuerefs{$1} = 0;
}
}
while (my ($ktop,$vtop) = each(%data)) {
if ($ktop =~ /^(monthly|daily|hourly)$/) {
while (my ($ktp,$vtp) = each(%{$vtop})) {
while (my ($ket,$vet) = each(%{$vtp})) {
while (my ($kvt,$vvt) = each(%{$vet})) {
if ($kvt ne 'summary') {
while (my ($kvd,$vvd) = each(%{$vvt})) {
if (defined($vvd)) {
$valuerefs{$kvd}++ if ($kvd);
}
}
}
}
}
}
}
}
while (my ($kvr,$vvr) = each(%valuerefs)) {
if (!$vvr) {
del_value($kvr);
$dvcnt++;
}
}
print "\tTrimmed $dvcnt 'values' from ValueDB\n" if (!$QUIET);
#save_database();
}
sub restore_database($) {
die "Not implemented yet!";
}
sub copy_database($) {
my ($action) = @_;
return 0 if ($NODB);
my $fake;
my $backup = 0;
my $list = 0;
my $rewrite = 0;
my $counter = 1;
if (!$action || $action =~ /^\d+$/) {
$fake = $action if ($action =~ /^\d+$/);
print "Rewriting database\n";
$rewrite = 1;
} elsif ($action eq '|') {
$list = 1;
$counter = 0;
} elsif ($action !~ /^.$/) {
print "Backing up database\n";
$backup = 1;
} elsif ($action ne '#') {
return 0;
}
$fake = 0 if (!$fake || $fake<0);
print "Faking ($fake)!\n" if ($fake);
open_database(1);
my $cvttonum = (!(defined($data{'misc'}{'databaseversion'}) && $data{'misc'}{'databaseversion'}>1));
my %newdata = ();
my %newvalues = ();
my $time = time();
if ($rewrite) {
print "Converting to database version 2\n" if ($cvttonum);
if ($fake<2) {
unlink("$VALUEDB.rewritten");
unlink("$SUMMARYDB.rewritten");
tie(%newvalues,'DB_File',"$VALUEDB.rewritten",O_RDWR|O_EXLOCK|O_CREAT,0644) or die "Could not open the new database!";
tie(%newdata,'DWH_File',"$SUMMARYDB.rewritten",O_RDWR|O_EXLOCK|O_CREAT,0644) or die "Could not open the new database!";
}
} elsif ($backup && !$fake) {
open(BF,">$action") or die ("Could not open output file");
print BF "# Graphdefang database version 2\n";
}
$newvalues{'count'} = 0 if ($fake<3);
my $cntt = 0;
my $cntc = 0;
my $cntv = 0;
my $start = time();
copy_counter(1,$start,$cntt,$cntc,$cntv) if ($counter);
if ($counter && !($backup || $list || $rewrite || $cvttonum)) {
while (my ($k,$v) = each(%values)) {
if ($k =~ /^n2v\|/) {
$cntt++;
copy_counter(0,$start,$cntt,$cntc,$cntv) if ($counter);
}
}
copy_counter(1,$start,$cntt,$cntc,$cntv) if ($counter);
$cntv = $cntt;
$cntt = 0;
print STDERR "\n";
$start = time();
}
while (my ($ktop,$vtop) = each(%data)) {
if ($ktop eq 'misc' || $ktop eq 'maxhosttime') {
while (my ($kcd,$vcd) = each(%{$vtop})) {
if ($rewrite) {
$newdata{$ktop}{$kcd} = $vcd if ($fake<2);
} elsif ($backup) {
print BF "$ktop\x1D$kcd\x1D$vcd\n" if (!$fake);
} elsif ($list) {
print "$ktop/$kcd=$vcd\n";
}
$cntt++;
copy_counter(0,$start,$cntt,$cntc,$cntv) if ($counter);
}
} elsif ($ktop eq 'totals') {
while (my ($kn,$vn) = each(%{$vtop})) {
if ($kn =~ /^\d+$/) {
while (my ($ket,$vet) = each(%{$vn})) {
while (my ($kvt,$vvt) = each(%{$vet})) {
if ($rewrite) {
$newdata{$ktop}{$kn}{$ket}{$kvt} = $vvt if ($fake<2);
} elsif ($backup) {
print BF "$ktop\x1D$kn\x1D$ket\x1D$kvt=$vvt\n" if (!$fake);
} elsif ($list) {
print "$ktop/$kn/$ket/$kvt=$vvt\n";
}
$cntt++;
copy_counter(0,$start,$cntt,$cntc,$cntv) if ($counter);
}
}
}
}
} elsif ($ktop =~ /^(yeardaily|yearly|hourly|weekdaily|monthly|daily)_totals$/) {
while (my ($ktp,$vtp) = each(%{$vtop})) {
while (my ($ket,$vet) = each(%{$vtp})) {
while (my ($kvt,$vvt) = each(%{$vet})) {
if ($rewrite) {
$newdata{$ktop}{$ktp}{$ket}{$kvt} = $vvt if ($fake<2);
} elsif ($backup) {
print BF "$ktop\x1D$ktp\x1D$ket\x1D$kvt\x1D$vvt\n" if (!$fake);
} elsif ($list) {
print "$ktop/$ktp/$ket/$kvt=$vvt\n";
}
$cntt++;
copy_counter(0,$start,$cntt,$cntc,$cntv) if ($counter);
}
}
}
} elsif ($ktop =~ /^(monthly|daily|hourly)$/) {
while (my ($ktp,$vtp) = each(%{$vtop})) {
while (my ($ket,$vet) = each(%{$vtp})) {
while (my ($kvt,$vvt) = each(%{$vet})) {
if ($kvt eq 'summary') {
$newdata{$ktop}{$ktp}{$ket}{$kvt} = $vvt if ($fake<2);
$cntt++;
copy_counter(0,$start,$cntt,$cntc,$cntv) if ($counter);
} else {
while (my ($kvd,$vvd) = each(%{$vvt})) {
if (defined($vvd) && ($list || $backup || $rewrite)) {
my $n;
my $v;
if ($cvttonum) {
$v = $kvd;
} else {
$v = get_value($kvd);
}
if ($rewrite) {
if ($fake < 3) {
if ($cvttonum) {
$v = $kvd;
} else {
$v = get_value($kvd);
}
if (!defined($v)) {
$n = undef;
} elsif (!$v) {
$n = 0;
} else {
$n = $newvalues{"v2n|$v"};
if (!$n) {
$newvalues{'count'} ++;
$n = $newvalues{'count'};
$newvalues{"n2v|$n"} = $v;
$newvalues{"v2n|$v"} = $n;
$cntv++;
}
}
}
$newdata{$ktop}{$ktp}{$ket}{$kvt}{$n} = $vvd if ($fake<2);
} elsif ($backup) {
print BF "$ktop\x1D$ktp\x1D$ket\x1D$kvt\x1D$v\x1D$vvd\n" if (!$fake);
} elsif ($list) {
print "$ktop/$ktp/$ket/$kvt/$v=$vvd\n";
}
}
$cntt++;
$cntc++;
copy_counter(0,$start,$cntt,$cntc,$cntv) if ($counter);
}
}
}
}
}
}
}
if ($rewrite && $fake<2) {
$newdata{'misc'}{'databaseversion'} = 2;
$newvalues{'databaseversion'} = 2;
untie(%newdata);
untie(%newvalues);
}
copy_counter(1,$start,$cntt,$cntc,$cntv) if ($counter);
print STDERR "\n" if ($counter);
if ($rewrite) {
if (!$fake) {
close_database();
unlink("$SUMMARYDB.backup");
unlink("$VALUEDB.backup");
move($SUMMARYDB,"$SUMMARYDB.backup");
move($VALUEDB,"$VALUEDB.backup");
move("$SUMMARYDB.rewritten",$SUMMARYDB);
move("$VALUEDB.rewritten",$VALUEDB);
}
print "Database rewritten\n";
} elsif ($backup) {
if (!$fake) {
open(BF,">$action") or die ("Could not open output file");
print BF "# Graphdefang database version 2\n";
}
print "Database backed up\n";
}
return 1;
}
sub open_database {
my $ignore_version = shift;
return 1 if ($NODB);
if (!$database_open) {
print "\tOpening database\n" if (!$QUIET);
copy($SUMMARYDB,database_name()) if ($TEMPDATABASE);
copy($VALUEDB,valuebase_name()) if ($TEMPDATABASE);
tie(%values,'DB_File',valuebase_name(),O_RDWR|O_EXLOCK|O_CREAT,0644) or die "Could not open the database!";
tie(%data,'DWH_File',database_name(),O_RDWR|O_EXLOCK|O_CREAT,0644) or die "Could not open the database!";
if (!$ignore_version &&
!(defined($data{'misc'}{'databaseversion'}) && $data{'misc'}{'databaseversion'}>1) &&
!(defined($values{'misc'}{'databaseversion'}) && $values{'misc'}{'databaseversion'}>1)) {
untie(%data);
untie(%values);
die "You'r database is in an old format, you need to run a rewrite on it.";
}
$database_open = 1;
}
return $database_open;
}
sub close_database() {
if ($database_open) {
#print "\tClosing database\n" if (!$QUIET);
untie %data;
untie %values;
move(database_name(),$SUMMARYDB) if ($TEMPDATABASE);
move(valuebase_name(),$VALUEDB) if ($TEMPDATABASE);
$database_open = 0;
return 1;
}
return 0;
}
sub save_database() {
if ($database_open) {
print "\tSaving database\n" if (!$QUIET);
untie %data;
untie %values;
tie(%values,'DB_File',valuebase_name(),O_RDWR|O_EXLOCK|O_CREAT,0644) or die "Could not open the database!";
tie(%data,'DWH_File',database_name(),O_RDWR|O_EXLOCK|O_CREAT,0644) or die "Could not open the database!";
return 1;
}
return 0;
}
BEGIN {
tie(%dns,'Tie::DNS');
}
END {
if ($database_open) {
untie %data;
untie %values;
$database_open = 0;
}
untie %dns;
}
sub date_set($$$) {
my ($rule,$start,$end) = @_;
$start = time2str('%Y%m%dT%H%M%S',$start);
$end = time2str('%Y%m%dT%H%M%S',$end);
my $x = Date::Set->event(
rule => $rule,
at => [[ $start, $end ]]
);
$x =~ s/^\[(\d+\.\.)?(.*)\]$/$2/;
return $x;
}
sub zero_counts($) {
my ($t) = @_;
my @c;
for (my $i=0; $i<$t; $i++) { $c[$i] = 0; }
return @c;
}
sub year_count($$) {
my ($start,$end) = @_;
$start = time2str('%Y',$start);
$end = time2str('%Y',$end);
return (($end+1)-$start);
}
sub month_counts($$) {
my ($start,$end) = @_;
my @counts = zero_counts(12);
my @dates = split(/,/, date_set('FREQ=MONTHLY;INTERVAL=1',$start,$end));
foreach my $date (@dates) {
my $ical = Date::ICal->new( ical => $date );
$counts[$ical->month-1]++;
}
return @counts;
}
sub day_counts($$) {
my ($start,$end) = @_;
my @counts = zero_counts(31);
my @dates = split(/,/, date_set('FREQ=DAILY;INTERVAL=1',$start,$end));
foreach my $date (@dates) {
my $ical = Date::ICal->new( ical => $date );
$counts[$ical->day-1]++;
}
return @counts;
}
sub weekday_counts($$) {
my ($start,$end) = @_;
my @counts = zero_counts(7);
my @dates = split(/,/, date_set('FREQ=DAILY;INTERVAL=1',$start,$end));
foreach my $date (@dates) {
my $ical = Date::ICal->new( ical => $date );
my $day = $ical->day_of_week;
$day = 7 if ($day == 0);
$counts[$day-1]++;
}
return @counts;
}
sub yearday_counts($$) {
my ($start,$end) = @_;
my @counts = zero_counts(366);
my @dates = split(/,/, date_set('FREQ=DAILY;INTERVAL=1',$start,$end));
foreach my $date (@dates) {
my $year = $date;
my $month = $date;
$year =~ s/^(\d\d\d\d).*/$1/;
$month =~ s/^\d\d\d\d(\d\d).*/$1/;
$date =~ s/^\d{6}(\d\d).*/$1/;
my $yd = Date::ICal::days_this_year($date,$month,$year);
$counts[$yd]++;
}
return @counts;
}
sub hour_counts($$) {
my ($start,$end) = @_;
my $istart = Date::ICal->new( epoch => $start );
my $iend = Date::ICal->new( epoch => $end );
my @counts = zero_counts(24);
$istart->add( 'day' );
my $days = $iend-$istart;
for (my $d=1; $d<=abs($days->as_days); $d++) {
for (my $h=0; $h<24; $h++) { $counts[$h]++; }
}
for (my $h=$istart->hour; $h<24; $h++) { $counts[$h]++; }
for (my $h=0; $h<=$iend->hour; $h++) { $counts[$h]++; }
return @counts;
}
sub average_str($) {
my ($float) = @_;
return sprintf('%.0f',$float) if ($float >= 10);
return sprintf('%.1f',$float);
}
sub get_unixtime_by_timesummary($$) {
my $timesummary = shift;
my $unixtime = shift;
# Get the number of seconds past the day for a given unixtime
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($unixtime);
# zero out appropriate seconds,minutes,hours,days,etc...
$sec = 0; #if ($timesummary =~ m/hourly|daily|monthly/);
$min = 0; #if ($timesummary =~ m/hourly|daily|monthly/);
$hour = 0 if ($timesummary ne 'hourly');
$mday = 1 if ($timesummary eq 'monthly');
# get unixtime for our new values
$unixtime = timelocal($sec, $min, $hour, $mday, $mon, $year);
return $unixtime;
}
sub get_stripped_unixtime_by_timesummary($$) {
my $timesummary = shift;
my $unixtime = shift;
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($unixtime);
$wday = 7 if ($wday == 0);
return $hour if ($timesummary eq 'hourly');
return $mday if ($timesummary eq 'daily');
return ($mon + 1) if ($timesummary eq 'monthly');
return $wday if ($timesummary eq 'weekdaily');
return ($yday +1) if ($timesummary eq 'yeardaily');
return ($year + 1900) if ($timesummary eq 'yearly');
return -1;
}
sub trim_database() {
open_database();
# Start the DB Trim
my $now = time();
my $trimcounter = 0;
# Delete hourly data older than 1.25*$NUM_HOURS_SUMMARY hours
my $deletetime = get_unixtime_by_timesummary('hourly',$now - 1.25*$NUM_HOURS_SUMMARY*60*60);
foreach my $entrytime (keys %{$data{'hourly'}}) {
if ($entrytime < $deletetime) {
delete($data{'hourly'}{$entrytime});
$trimcounter++;
}
}
print "\tTrimmed $trimcounter 'hourly' entries from SummaryDB\n" if (!$QUIET);
# Delete daily data older than 1.25*$NUM_DAYS_SUMMARY days
$deletetime = get_unixtime_by_timesummary('daily',$now - 1.25*$NUM_DAYS_SUMMARY*60*60*24);
$trimcounter=0;
foreach my $entrytime (keys %{$data{'daily'}}) {
if ($entrytime < $deletetime) {
delete $data{'daily'}{$entrytime};
$trimcounter++;
}
}
print "\tTrimmed $trimcounter 'daily' entries from SummaryDB\n" if (!$QUIET);
# Delete all but Top25 entries in hours, days, and months
# other than the current one!
my @DeleteTimes = ('hourly', 'daily', 'monthly');
$trimcounter = 0;
foreach my $deletetime (@DeleteTimes) {
my $nowdeletetime = get_unixtime_by_timesummary($deletetime,$now);
foreach my $entrytime (keys %{$data{$deletetime}}) {
if ($entrytime < $nowdeletetime ) {
foreach my $event (keys %{$data{$deletetime}{$entrytime}}) {
foreach my $type (keys %{$data{$deletetime}{$entrytime}{$event}}) {
if ($type ne 'summary') {
my %total = ();
foreach my $value (keys %{$data{$deletetime}{$entrytime}{$event}{$type}}) {
$total{$value} = $data{$deletetime}{$entrytime}{$event}{$type}{$value};
}
# Create list of top 25 items.
my $i = 0;
my %keep = ();
foreach my $TopName (sort { $total{$b} <=> $total{$a} } keys %total) {
$keep{$TopName} = 1;
$i++;
last if $i >= 25;
}
# delete the entries unless it is in the topList.
foreach my $value (keys %{$data{$deletetime}{$entrytime}{$event}{$type}}) {
if (!defined($keep{$value})) {
delete $data{$deletetime}{$entrytime}{$event}{$type}{$value};
$trimcounter++;
}
}
}
}
}
}
}
}
print "\tTrimmed $trimcounter 'non top25' entries from SummaryDB\n" if (!$QUIET);
#save_database();
}
sub read_and_summarize_data($$$) {
use vars qw(%event $text $pid %spamd %user_unknown $event $value1 $value2 $value3 $value4 $sender $recipient $subject $NumEvents $FoundNewRow $unixtime $MaxDBUnixTime $recipientdomain $senderdomain);
my $fn = shift;
my $nomax = shift;
my $stopat = shift;
# Temporary variable for lookup information
%spamd = ();
my %NumNewLines;
my $TotNumNewLines = 0;
# Set graphtimes
my @GraphTimes = ("hourly","daily","monthly");
# Load event processing perl code from the events subdirectory
my $dirname = "$MYDIR/event";
opendir(DIR, $dirname) or die "can't opendir $dirname: $!";
while (defined(my $file = readdir(DIR))) {
if (!($file =~ m/^\./) and !($file =~ m/^CVS/)) {
# do nothing if file starts with '.'
opendir(SUBDIR, "$dirname/$file") or die "can't opendir $dirname/$file: $!";
while (defined(my $file2 = readdir(SUBDIR))) {
if (!($file2 =~ m/^\./) and !($file2 =~ m/^CVS/)) {
require "$dirname/$file/$file2";
}
}
}
}
closedir(SUBDIR);
closedir(DIR);
open_database();
print "\tProcessing data file: $fn\n" if (!$QUIET);
# Open log file
tie *ZZZ, 'File::ReadBackwards', $fn || die("can't open datafile: $!");
# Get max unixtime value from DBM file
# This is left here for backwards compatibility... we now track MAX times per host
# to support log files from multiples hosts.
$MaxDBUnixTime = 0;
if (!$nomax && defined($data{'max'})) {
$MaxDBUnixTime = $data{'max'};
# delete the max entry 'cuz we won't use it again
delete($data{'max'});
print "\tConverting to host-based max times\n" if (!$QUIET and !$NODB);
#print "\tPrevious Max Unixtime from SummaryDB: $MaxDBUnixTime\n" if (!$QUIET and !$NODB);
}
# print out the list of max times per host
my %ReadMaxHostTime;
if (defined($data{'maxhosttime'})) {
foreach my $host (sort keys %{$data{'maxhosttime'}}) {
$ReadMaxHostTime{$host} = $data{'maxhosttime'}{$host};
#print "\tMax Unixtime from SummaryDB for $host: $data{'maxhosttime'}{$host}\n" if (!$QUIET and !$NODB);
}
}
while (<ZZZ>) {
chomp;
# Parse syslog line
m/^(\S+\s+\d+\s+\d+:\d+:\d+)\s # datestring -- 1
(\S+)\s # host -- 2
(\S+?) # program -- 3
(?:\[(\d+)\])?:\s # pid -- 4
(?:\[ID\ \d+\ [a-z0-9]+\.[a-z]+\]\ )? # Solaris stuff -- not used
(.*)/x; # text -- 5
my $datestring = $1;
my $host = $2;
my $program = $3;
$pid = $4;
$text = $5;
# Parse date string from syslog using any TIMEZONE info from the config file.
if (defined $TZ{$host}) {
my $zone = tz2zone($TZ{$host});
$unixtime=str2time($datestring,$zone);
} else {
$unixtime=str2time($datestring);
}
# don't examine the line if it is greater than 5 minutes
# older than the maximum time in our DB. The 5 minutes
# comes from the PID, From, and Relay caching with sendmail
# and spamd that occurs below.
$MaxDBUnixTime = $ReadMaxHostTime{$host} if (!$nomax && defined($ReadMaxHostTime{$host}));
last if ($unixtime < ($MaxDBUnixTime-60*5));
# Whole hour stuff
next if ($stopat && ($unixtime >= $stopat));
$event = '';
$value1 = '';
$value2 = '';
$value3 = '';
$value4 = '';
$sender = '';
$recipient = '';
$subject = '';
$recipientdomain = '';
$senderdomain = '';
$NumEvents = 1;
$FoundNewRow = 0;
if (defined $event{$program}) {
foreach my $subroutine (sort keys %{$event{$program}}) {
$event{$program}{$subroutine}->();
last if ($FoundNewRow);
}
}
if ($FoundNewRow) {
# Increment Number of New Lines Found
$NumNewLines{$host}++;
$TotNumNewLines++;
$sender = "" if (defined($recipient) && $sender eq "?");
$recipient = "" if (defined($recipient) && $recipient eq "?");
$sender =~ s/^<// if ($sender);
$sender =~ s/>$// if ($sender);
$recipient =~ s/^<// if ($recipient);
$recipient =~ s/>$// if ($recipient);
$senderdomain = $sender;
$senderdomain =~ s/^[^@]*@// if ($senderdomain);
$recipientdomain = $recipient;
$recipientdomain =~ s/^[^@]*@// if ($recipientdomain);
#$value1 =~ s/^\[// if ($value1);
#$value1 =~ s/\]$// if ($value1);
#$value2 =~ s/^\[// if ($value1);
#$value2 =~ s/\]$// if ($value1);
$sender = put_value($sender);
$recipient = put_value($recipient);
$senderdomain = put_value($senderdomain);
$recipientdomain = put_value($recipientdomain);
$subject = put_value($subject);
$value1 = put_value($value1);
$value2 = put_value($value2);
$value3 = put_value($value3);
$value4 = put_value($value4);
my $summarytime;
# rollup hourly, daily, and monthly summaries for every variable
foreach my $timesummary (@GraphTimes) {
$summarytime = get_unixtime_by_timesummary($timesummary, $unixtime);
$data{$timesummary}{$summarytime}{$event}{'summary'}=0 if (!defined($data{$timesummary}{$summarytime}{$event}{'summary'}));
$data{$timesummary}{$summarytime}{$event}{'value1'}{$value1}=0 if (!defined($data{$timesummary}{$summarytime}{$event}{'value1'}{$value1}));
$data{$timesummary}{$summarytime}{$event}{'value2'}{$value2}=0 if (!defined($data{$timesummary}{$summarytime}{$event}{'value2'}{$value2}));
$data{$timesummary}{$summarytime}{$event}{'value3'}{$value3}=0 if (!defined($data{$timesummary}{$summarytime}{$event}{'value3'}{$value3}));
$data{$timesummary}{$summarytime}{$event}{'value4'}{$value4}=0 if (!defined($data{$timesummary}{$summarytime}{$event}{'value4'}{$value4}));
$data{$timesummary}{$summarytime}{$event}{'sender'}{$sender}=0 if (!defined($data{$timesummary}{$summarytime}{$event}{'sender'}{$sender}));
$data{$timesummary}{$summarytime}{$event}{'recipient'}{$recipient}=0 if (!defined($data{$timesummary}{$summarytime}{$event}{'recipient'}{$recipient}));
$data{$timesummary}{$summarytime}{$event}{'subject'}{$subject}=0 if (!defined($data{$timesummary}{$summarytime}{$event}{'subject'}{$subject}));
$data{$timesummary}{$summarytime}{$event}{'recipientdomain'}{$recipientdomain}=0 if (!defined( $data{$timesummary}{$summarytime}{$event}{'recipientdomain'}{$recipientdomain}));
$data{$timesummary}{$summarytime}{$event}{'senderdomain'}{$senderdomain}=0 if (!defined($data{$timesummary}{$summarytime}{$event}{'senderdomain'}{$senderdomain}));
$data{$timesummary}{$summarytime}{$event}{'summary'}+=$NumEvents;
$data{$timesummary}{$summarytime}{$event}{'value1'}{$value1}+=$NumEvents if ($value1 ne '');
$data{$timesummary}{$summarytime}{$event}{'value2'}{$value2}+=$NumEvents if ($value2 ne '');
$data{$timesummary}{$summarytime}{$event}{'value3'}{$value3}+=$NumEvents if ($value3 ne '');
$data{$timesummary}{$summarytime}{$event}{'value4'}{$value4}+=$NumEvents if ($value4 ne '');
$data{$timesummary}{$summarytime}{$event}{'sender'}{$sender}+=$NumEvents if ($sender ne '');
$data{$timesummary}{$summarytime}{$event}{'recipient'}{$recipient}+=$NumEvents if ($recipient ne '');
$data{$timesummary}{$summarytime}{$event}{'subject'}{$subject}+=$NumEvents if ($subject ne '');
$data{$timesummary}{$summarytime}{$event}{'recipientdomain'}{$recipientdomain}+=$NumEvents if ($recipientdomain ne '');
$data{$timesummary}{$summarytime}{$event}{'senderdomain'}{$senderdomain}+=$NumEvents if ($senderdomain ne '');
# Store the maximum unixtime per timesummary for later reference
$summarytime = get_stripped_unixtime_by_timesummary($timesummary, $unixtime);
$data{$timesummary . '_totals'}{$summarytime}{$event}{'summary'}=0 if (!defined($data{$timesummary . '_totals'}{$summarytime}{$event}{'summary'}));
$data{$timesummary . '_totals'}{$summarytime}{$event}{'summary'}+=$NumEvents;
}
$summarytime = get_stripped_unixtime_by_timesummary("yearly", $unixtime);
$data{'yearly_totals'}{$summarytime}{$event}{'summary'}=0 if(!defined($data{'yearly_totals'}{$summarytime}{$event}{'summary'}));
$data{'yearly_totals'}{$summarytime}{$event}{'summary'}+=$NumEvents;
$summarytime = get_stripped_unixtime_by_timesummary("yeardaily", $unixtime);
$data{'yeardaily_totals'}{$summarytime}{$event}{'summary'}=0 if(!defined($data{'yeardaily_totals'}{$summarytime}{$event}{'summary'}));
$data{'yeardaily_totals'}{$summarytime}{$event}{'summary'}+=$NumEvents;
$summarytime = get_stripped_unixtime_by_timesummary("weekdaily", $unixtime);
$data{'weekdaily_totals'}{$summarytime}{$event}{'summary'}=0 if(!defined($data{'weekdaily_totals'}{$summarytime}{$event}{'summary'}));
$data{'weekdaily_totals'}{$summarytime}{$event}{'summary'}+=$NumEvents;
$data{'totals'}{'1'}{$event}{'summary'}=0 if (!defined($data{'totals'}{'1'}{$event}{'summary'}));
$data{'totals'}{'1'}{$event}{'summary'}+=$NumEvents;
$data{'misc'}{'firsttotal'} = $unixtime
if (!defined($data{'misc'}{'firsttotal'}));
$data{'misc'}{'lasttotal'} = $unixtime
if (!defined($data{'misc'}{'lasttotal'}) || $unixtime>$data{'misc'}{'lasttotal'});
$data{'maxhosttime'}{$host} = $unixtime
if (!defined($data{'maxhosttime'}{$host})
or $unixtime > $data{'maxhosttime'}{$host});
}
}
close (ZZZ);
if (!$NODB) {
#save_database();
if (%NumNewLines) {
foreach my $host (sort keys %NumNewLines) {
print "\t$NumNewLines{$host} new log lines processed for $host\n" if (!$QUIET);
}
} else {
print "\t0 new log lines processed\n" if (!$QUIET);
}
}
return $TotNumNewLines;
}
sub graph($) {
my $settings = shift;
my $stopat = shift;
open_database();
foreach my $grouping_time (@{$settings->{grouping_times}}) {
$settings->{grouping_time} = $grouping_time;
# Set the settings for the graph we've been asked to draw
set_graph_settings($settings);
print "\t$settings->{chart_filename}\n" if (!$QUIET);
# Get the data for the graph we've been asked to draw
my @GraphData = get_graph_data($settings,$stopat);
# Draw Graph
draw_graph($settings->{graph_type},$settings,\@GraphData);
# Output interval used for totals/average?
if ($grouping_time =~ /.*(totals|average)$/) {
save_totals_interval();
}
}
}
sub set_graph_settings($) {
my $settings = shift;
# Set the graph title and filename according to the options chosen
# Initialize the title and filename
$settings->{chart_title} = "";
$settings->{chart_filename} = "";
my $autotitle = "";
my $autofilename = "";
# Set graph x & y dimensions
$settings->{x_graph_size} = $X_GRAPH_SIZE if (!defined($settings->{x_graph_size}));
$settings->{y_graph_size} = $Y_GRAPH_SIZE if (!defined($settings->{y_graph_size}));
# Add "Top N" to the beginning of the Title if necessary
if ($settings->{top_n}) {
$autotitle = "Top $settings->{top_n} ";
}
# Set Data Type Title
my $i = 0;
foreach my $data_type (@{$settings->{data_types}}) {
# Uppercase the first letter of the data_type
$autotitle .= "\u$data_type";
$autofilename .= $data_type;
$i++;
if ( $i == ($#{$settings->{data_types}}) ) {
$autotitle .= " and "
} elsif ( $i < ($#{$settings->{data_types}}) ) {
$autotitle .= ", "
}
}
# Set Grouping Title
if ($settings->{grouping} eq 'summary') {
$autotitle = $autotitle . " Total Counts ";
} elsif ($settings->{grouping} eq 'value1') {
if (defined($settings->{value1_title})) {
$autotitle = $autotitle . " Counts by $settings->{value1_title}";
} else {
$autotitle = $autotitle . " Counts by Value1";
}
} elsif ($settings->{grouping} eq 'value2') {
if (defined($settings->{value2_title})) {
$autotitle = $autotitle . " Counts by $settings->{value2_title}";
} else {
$autotitle = $autotitle . " Counts by Value2";
}
} elsif ($settings->{grouping} eq 'value3') {
if (defined($settings->{value3_title})) {
$autotitle = $autotitle . " Counts by $settings->{value3_title}";
} else {
$autotitle = $autotitle . " Counts by Value3";
}
} elsif ($settings->{grouping} eq 'value4') {
if (defined($settings->{value4_title})) {
$autotitle = $autotitle . " Counts by $settings->{value4_title}";
} else {
$autotitle = $autotitle . " Counts by Value4";
}
} elsif ($settings->{grouping} eq 'sender') {
$autotitle = $autotitle . " Counts by Sender";
} elsif ($settings->{grouping} eq 'recipient') {
$autotitle = $autotitle . " Counts by Recipient";
} elsif ($settings->{grouping} eq 'subject') {
$autotitle = $autotitle . " Counts by Subject";
} elsif ($settings->{grouping} eq 'recipientdomain') {
$autotitle = $autotitle . " Counts by Recipient Domain";
} elsif ($settings->{grouping} eq 'senderdomain') {
$autotitle = $autotitle . " Counts by Sender Domain";
} else {
die ("Invalid settings{grouping} value");
}
# Put top_n in the filename?
if ($settings->{top_n}) {
$autofilename .= "_$settings->{top_n}";
} else {
$autofilename .= "_";
}
$autofilename .= "$settings->{grouping}_$settings->{graph_type}";
# The final portion of the title will be set in the section below
if ($settings->{grouping_time} eq 'hourly') {
$settings->{x_axis_num_values} = $NUM_HOURS_SUMMARY; # Number of x-axis values on graph
$settings->{x_axis_num_values} = $settings->{num_hourly_values} if defined($settings->{num_hourly_values});
$settings->{x_axis_num_sec_incr}= 60*60; # Incremental number of seconds represented by each x-axis value
#$settings->{x_axis_date_format} = "%h %d, %H"; # Format of date string on x-axis
$settings->{x_axis_date_format} = "%H";
$settings->{x_label} = 'Hours';
$settings->{y_label} = 'Counts per Hour';
$settings->{chart_title} = $autotitle . " per Hour (last $settings->{x_axis_num_values} hours)"
unless defined($settings->{title});
$autofilename = "hourly_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'daily') {
$settings->{x_axis_num_values} = $NUM_DAYS_SUMMARY;
$settings->{x_axis_num_values} = $settings->{num_daily_values} if defined($settings->{num_daily_values});
$settings->{x_axis_num_sec_incr}= 60*60*24;
#$settings->{x_axis_date_format} = "%h %d";
$settings->{x_axis_date_format} = "%d";
$settings->{x_label} = 'Days';
$settings->{y_label} = 'Counts per Day';
$settings->{chart_title} = $autotitle . " per Day (Last $settings->{x_axis_num_values} days)"
unless defined($settings->{title});
$autofilename = "daily_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'monthly') {
$settings->{x_axis_num_values} = $NUM_MONTH_SUMMARY;
$settings->{x_axis_num_values} = $settings->{num_monthly_values} if defined($settings->{num_monthly_values});
$settings->{x_axis_num_sec_incr}= 60*60*24*31;
#$settings->{x_axis_date_format} = "%h";
$settings->{x_axis_date_format} = "%m";
$settings->{x_label} = 'Months';
$settings->{y_label} = 'Counts per Month';
$settings->{chart_title} = $autotitle . " per Month (Last $settings->{x_axis_num_values} months)"
unless defined($settings->{title});
$autofilename = "monthly_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'yearly_totals') {
$settings->{x_label} = 'Years';
$settings->{y_label} = 'Totals per Year';
$settings->{chart_title} = "$autotitle per Year" unless defined($settings->{title});
$autofilename = "yearly_totals_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'monthly_totals') {
$settings->{x_label} = 'Months';
$settings->{y_label} = 'Totals per Month';
$settings->{chart_title} = "$autotitle per Month" unless defined($settings->{title});
$autofilename = "monthly_totals_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'weekdaily_totals') {
$settings->{x_label} = 'Days of the Week';
$settings->{y_label} = 'Totals per Day of Week';
$settings->{chart_title} = "$autotitle per Day of Week" unless defined($settings->{title});
$autofilename = "weekdaily_totals_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'yeardaily_totals') {
$settings->{x_label} = 'Days of the Year';
$settings->{y_label} = 'Totals per Day of Year';
$settings->{chart_title} = "$autotitle per Day of Year" unless defined($settings->{title});
$autofilename = "yeardaily_totals_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'daily_totals') {
$settings->{x_label} = 'Days of the Month';
$settings->{y_label} = 'Totals per Day of Month';
$settings->{chart_title} = "$autotitle per Day of Month" unless defined($settings->{title});
$autofilename = "monthdaily_totals_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'hourly_totals') {
$settings->{x_label} = 'Hours';
$settings->{y_label} = 'Totals per Hour';
$settings->{chart_title} = "$autotitle per Hour" unless defined($settings->{title});
$autofilename = "hourly_totals_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'totals') {
$settings->{x_label} = 'Totals';
$settings->{y_label} = 'Total Count';
$settings->{chart_title} = "$autotitle" unless defined($settings->{title});
$autofilename = "totals_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'yearly_average') {
$settings->{x_label} = 'Average';
$settings->{y_label} = 'Average per Year';
$settings->{chart_title} = "$autotitle per Year" unless defined($settings->{title});
$autofilename = "yearly_average_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'monthly_average') {
$settings->{x_label} = 'Months';
$settings->{y_label} = 'Average per Month';
$settings->{chart_title} = "$autotitle per Month" unless defined($settings->{title});
$autofilename = "monthly_average_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'weekdaily_average') {
$settings->{x_label} = 'Days of the Week';
$settings->{y_label} = 'Average per Day of Week';
$settings->{chart_title} = "$autotitle per Day of Week" unless defined($settings->{title});
$autofilename = "weekdaily_average_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'yeardaily_average') {
$settings->{x_label} = 'Days of the Year';
$settings->{y_label} = 'Average per Day of Year';
$settings->{chart_title} = "$autotitle per Day of Year" unless defined($settings->{title});
$autofilename = "yeardaily_average_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'daily_average') {
$settings->{x_label} = 'Days of the Month';
$settings->{y_label} = 'Average per Day of Month';
$settings->{chart_title} = "$autotitle per Day of Month" unless defined($settings->{title});
$autofilename = "monthdaily_average_" . $autofilename;
} elsif ($settings->{grouping_time} eq 'hourly_average') {
$settings->{x_label} = 'Hours';
$settings->{y_label} = 'Average per Hour';
$settings->{chart_title} = "$autotitle per Hour" unless defined($settings->{title});
$autofilename = "hourly_average_" . $autofilename;
}
if (defined $settings->{filter_name}) {
my $filter;
($filter = $settings->{filter_name}) =~ s/\W/_/g;
$settings->{chart_title} .= " filtered by $settings->{filter_name}";
$autofilename .= "_$filter";
}
# Use the title from graphdefang-config if specified, else use the autotitle
$settings->{chart_title} = $settings->{title} if (defined($settings->{title}));
# Use the filename from graphdefang-config if specified, else use the autofilename
$settings->{chart_filename} = $autofilename;
$settings->{chart_filename} = "$settings->{grouping_time}_$settings->{filename}" if (defined($settings->{filename}));
$settings->{chart_filename} =~ s/\//_/g; # Replace any '/' chars with '_'
}
sub nice_graph_name($) {
my ($n) = @_;
$n =~ s/_/ /g;
$n =~ s/([\/ ])([a-z])/$1\u$2/g;
$n =~ s/^([a-z])/\u$1/;
return $n;
}
sub get_graph_data($) {
my $settings = shift;
my $stopat = shift;
# Calculate the date cutoff for our graph
my $currenttime = time();
$currenttime = $stopat-1 if ($stopat && ($currenttime >= $stopat));
my $cutofftime;
my $currentyear;
my $currentmon;
my $currentisdst;
my $xincr = $settings->{x_axis_num_sec_incr};
my $totals = ($settings->{grouping_time} =~ /.*(totals|average)$/);
my $average = ($settings->{grouping_time} =~ /.*average$/);
my $yearly_average = ($settings->{grouping_time} eq 'yearly_average');
my $noaxis = ($settings->{graph_type} eq 'pie' || (defined($settings->{legend_columns}) && $settings->{legend_columns}<1));
die "Only pies and unstacked bars can be without legend!" if ($noaxis && $settings->{graph_type} !~ /^(pie|h?bar)$/);
die "Only summary graphs can be without axis!" if ($noaxis && $settings->{'grouping'} ne 'summary');
die "Most average graphs must have axes!" if ($noaxis && $average && !$yearly_average);
if ($settings->{grouping_time} eq 'monthly') {
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($currenttime);
$currentyear = $year;
$currentmon = $mon;
$currentisdst = $isdst;
# Decrement the month/year value n times
for (my $i = 0; $i < $settings->{x_axis_num_values}-1; $i++) {
if ($mon == 0) {
$year--;
$mon = 11;
} else {
$mon--;
}
}
# get unixtime for our new values
$mday = 1; # Get around a bug that only shows itself on the 30th or 31st of the month
$cutofftime = timelocal($sec, $min, $hour, $mday, $mon, $year);
} elsif (!$totals) {
$cutofftime = $currenttime - ($settings->{x_axis_num_sec_incr}*($settings->{x_axis_num_values}-1));
} else {
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($currenttime);
$xincr = 1;
$cutofftime = 1;
if ($settings->{grouping_time} =~ /^yearly.*/ && !$average) {
my ($fsec,$fmin,$fhour,$fmday,$fmon,$fyear,$fwday,$fyday,$fisdst) = localtime($data{'misc'}{'firsttotal'});
$cutofftime = ($fyear+1900);
$currenttime = ($year+1900);
$cutofftime = ($currenttime-1) if ($cutofftime >= $currenttime);
#print "$cutofftime -> $currenttime\n";
#$settings->{x_axis_num_values} = ($currenttime - $cutofftime) +1;
} elsif ($settings->{grouping_time} =~ /^monthly.*/) {
$currenttime = 12;
} elsif ($settings->{grouping_time} =~ /^yeardaily.*/) {
my ($fsec,$fmin,$fhour,$fmday,$fmon,$fyear,$fwday,$fyday,$fisdst) = localtime($data{'misc'}{'firsttotal'});
$cutofftime = ($fyday+1) if ($year <= $fyear);
$currenttime = 365;
$currenttime = ($yday+1) if ($year <= $fyear);
$cutofftime = ($currenttime - 1) if ($cutofftime >= $currenttime);
#print "$cutofftime -> $currenttime\n";
#print "yd=$yday fyd=$fyday y=$year fy=$fyear cot=$cutofftime ct=$currenttime\n";
} elsif ($settings->{grouping_time} =~ /^hourly.*/) {
$cutofftime = 0;
$currenttime = 23;
} elsif ($settings->{grouping_time} =~ /^weekdaily.*/) {
$currenttime = 7;
} elsif ($settings->{grouping_time} =~ /^daily.*/) {
$currenttime=31;
} elsif ($settings->{grouping_time} =~ /^(totals|yearly_average)$/) {
$currenttime = 1;
die "Only graphs without axis allowed for yearly average and all time totals!" if (!($noaxis));
} else {
die "Bad graph type!";
}
}
# Calculate on totals if average
if ($yearly_average) {
$settings->{grouping_time} = 'totals';
} elsif ($average) {
$settings->{grouping_time} =~ s/average$/totals/;
}
# Create Data Array for Graph
my @GraphData = ();
my @TopNNames = ();
my %Total = ();
my @Legend = ();
my $grandtotalsum = 0;
#print "Debug get_graph_data 100\n";
# Handle data_types = 'all'
my @allorg;
my $allset = (grep(/^all$/,@{$settings->{'data_types'}}));
if ($allset) {
@allorg = @{$settings->{'data_types'}};
my %all;
if (defined(%{$data{$settings->{grouping_time}}})) {
foreach my $date (keys %{$data{$settings->{grouping_time}}}) {
#print "Debug get_graph_data 110\n";
foreach my $data_type (keys %{$data{$settings->{grouping_time}}{$date}}) {
#print "Debug get_graph_data 111\n";
$all{$data_type} = 1 if (!grep(/^$data_type$/,@{$settings->{'data_types'}}));
}
}
}
#print "Debug get_graph_data 120\n";
$settings->{'data_types'} = ();
foreach my $key (sort keys %all) {
push @{$settings->{'data_types'}}, $key;
}
}
#print "Debug get_graph_data 200\n";
# Summarize totals across time interval
for (my $time=$cutofftime; $time<=$currenttime; $time += $xincr) {
my $date;
if ($totals) {
$date = $time;
} else {
$date = get_unixtime_by_timesummary($settings->{grouping_time},$time);
}
# Get total for summary grouping
if ($settings->{'grouping'} eq 'summary') {
foreach my $datatype (@{$settings->{'data_types'}}) {
if (defined($data{$settings->{grouping_time}}{$date}{$datatype}{'summary'})) {
$Total{$datatype} += $data
{$settings->{grouping_time}}
{$date}
{$datatype}
{'summary'};
$grandtotalsum += $data
{$settings->{grouping_time}}
{$date}
{$datatype}
{'summary'};
} else {
$Total{$datatype} += 0;
}
}
} else {
die "Only summary is possible for totals and average!" if ($totals);
# Get total for other groupings
foreach my $datatype (@{$settings->{'data_types'}}) {
foreach my $value (keys %{$data
{$settings->{grouping_time}}
{$date}
{$datatype}
{$settings->{'grouping'}}} ) {
next if (!defined($data
{$settings->{grouping_time}}
{$date}
{$datatype}
{$settings->{'grouping'}}
{$value}));
$Total{'value'}{$value} += $data
{$settings->{grouping_time}}
{$date}
{$datatype}
{$settings->{'grouping'}}
{$value};
$Total{$date}{$value} += $data
{$settings->{grouping_time}}
{$date}
{$datatype}
{$settings->{'grouping'}}
{$value};
$grandtotalsum += $data
{$settings->{grouping_time}}
{$date}
{$datatype}
{$settings->{'grouping'}}
{$value};
}
}
}
# Recalculate the x_axis_num_sec_incr value if we are graphing monthly.
# Determine the current month, increment it by one, and then get a time delta..
if ($settings->{grouping_time} eq 'monthly') {
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($date);
# Increment the month/year value
if ($mon == 11) {
$year++;
$mon = 0;
} else {
$mon++;
}
# Has dst kicked in this month? If so, adjust for it
my $dstadjustment = 0;
if (($currentyear == $year) && ($currentmon == $mon)) {
if ($currentisdst > $isdst) {
$dstadjustment = -3600;
} elsif ($currentisdst < $isdst) {
$dstadjustment = 3600;
} else {
$dstadjustment = 0;
}
}
# get unixtime for our new values
my $newmonthtime = timelocal($sec, $min, $hour, $mday, $mon, $year);
$settings->{x_axis_num_sec_incr} = $newmonthtime - $date + $dstadjustment;
$xincr = $settings->{x_axis_num_sec_incr};
}
}
#print "Debug get_graph_data 300\n";
# Sort the TopNNames list so we have it largest to smallest and keep only the top N.
if ($settings->{'grouping'} eq 'summary') {
#foreach my $datatype (@{$settings->{'data_types'}}) {
# #push @Legend, "\u$datatype, Total = $Total{$datatype}";
# push @Legend, nice_graph_name($datatype) . ", Total = $Total{$datatype}";
#}
@{$settings->{'data_types'}}=();
my @sorted;
if ($settings->{'reversed'}) {
@sorted = sort{$Total{$a} <=> $Total{$b}} keys %Total
} else {
@sorted = sort{$Total{$b} <=> $Total{$a}} keys %Total
}
foreach my $datatype (@sorted) {
push @{$settings->{'data_types'}}, $datatype;
if ($noaxis) {
push @{$GraphData[0]}, nice_graph_name($datatype) . ": $Total{$datatype}";
} else {
push @Legend, nice_graph_name($datatype) . " ($Total{$datatype})";
}
}
} else {
my @sorted;
if ($settings->{'reversed'}) {
@sorted = sort { $Total{'value'}{$a} <=> $Total{'value'}{$b} } keys %{$Total{'value'}};
} else {
@sorted = sort { $Total{'value'}{$b} <=> $Total{'value'}{$a} } keys %{$Total{'value'}};
}
my $i=0;
foreach my $TopNName (@sorted) {
if (!defined($settings->{'filter'}) or $TopNName =~ m/$settings->{'filter'}/i) {
push @TopNNames, $TopNName;
push @Legend, get_value($TopNName)." ($Total{'value'}{$TopNName})";
$i++;
}
last if (defined($settings->{'top_n'}) and $settings->{'top_n'} > 0 and $i >= $settings->{'top_n'} );
}
}
#print "Debug get_graph_data 400\n";
my $totalsum = 0;
my @sums = ();
for (my $time=$cutofftime; $time<=$currenttime; $time += $xincr) {
my $date;
if ($totals) {
$date = $time;
} else {
$date = get_unixtime_by_timesummary($settings->{grouping_time},$time);
}
my $datestring = '';
if ($totals) {
if ($settings->{grouping_time} =~ /weekdaily.*/) {
if ($date == 1) {
$datestring = 'Mon';
} elsif ($date == 2) {
$datestring = 'Tue';
} elsif ($date == 3) {
$datestring = 'Wed';
} elsif ($date == 4) {
$datestring = 'Thu';
} elsif ($date == 5) {
$datestring = 'Fri';
} elsif ($date == 6) {
$datestring = 'Sat';
} elsif ($date == 7) {
$datestring = 'Sun';
} else {
$datestring = '???';
}
} else {
$datestring = $date;
}
} elsif (defined($TZ{GD_Display})) {
my $zone = tz2zone($TZ{GD_Display});
$datestring = time2str($settings->{x_axis_date_format},$date,$zone);
} else {
$datestring = time2str($settings->{x_axis_date_format},$date);
}
my $i=0;
if (!$noaxis) {
push @{$GraphData[$i]}, $datestring;
}
if ( $settings->{'grouping'} eq 'summary' ) {
foreach my $datatype (@{$settings->{'data_types'}}) {
# Data format:
#$data{$timesummary}{$summarytime}{$event}{'summary'}++;
#$data{$timesummary}{$summarytime}{$event}{'value1'}{$value1}++
$i++;
# Set any undefined values to 0 so GD::Graph
# has something to graph
my $value;
if ( defined($data
{$settings->{grouping_time}}
{$date}
{$datatype}
{'summary'}) ) {
#push @{$GraphData[$i]}, $data->
# {$settings->{grouping_time}}
# {$date}
# {$datatype}
# {'summary'};
$value = $data{$settings->{grouping_time}}
{$date}{$datatype}{'summary'};
} else {
#push @{$GraphData[$i]}, 0;
$value = 0;
}
if ($noaxis) {
push @{$GraphData[1]}, $value;
} else {
push @{$GraphData[$i]}, $value;
}
$totalsum += $value;
}
} else {
# iterate over top_n values if they exist, else push 0
if ($#TopNNames > -1) {
foreach my $TopNName (@TopNNames) {
$i++;
# Set any undefined values to 0 so GD::Graph
# has something to graph
if ( defined ($Total{$date}{$TopNName}) ) {
push @{$GraphData[$i]}, $Total{$date}{$TopNName};
$totalsum += $Total{$date}{$TopNName};
} else {
push @{$GraphData[$i]}, 0;
}
}
} else {
$i++;
push @{$GraphData[$i]}, 0;
}
}
# Recalculate the x_axis_num_sec_incr value if we are graphing monthly.
# Determine the current month, increment it by one, and then get a time delta..
if ($settings->{grouping_time} eq 'monthly') {
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($date);
# Increment the month/year value
if ($mon == 11) {
$year++;
$mon = 0;
} else {
$mon++;
}
# Has dst kicked in this month? If so, adjust for it
my $dstadjustment = 0;
if (($currentyear == $year) && ($currentmon == $mon)) {
if ($currentisdst > $isdst) {
$dstadjustment = -3600;
} elsif ($currentisdst < $isdst) {
$dstadjustment = 3600;
} else {
$dstadjustment = 0;
}
}
# get unixtime for our new values
my $newmonthtime = timelocal($sec, $min, $hour, $mday, $mon, $year);
$settings->{x_axis_num_sec_incr} = $newmonthtime - $date + $dstadjustment;
$xincr = $settings->{x_axis_num_sec_incr};
}
}
# Calculate averages based on the totals...
# We replaced average with totals above, set it back
my $averagesum = "";
if ($yearly_average) {
$settings->{grouping_time} = 'yearly_average';
my $years = year_count($data{'misc'}{'firsttotal'},$data{'misc'}{'lasttotal'});
my $i = 0;
foreach my $label (@{$GraphData[0]}) {
@{$GraphData[1]}[$i] = @{$GraphData[1]}[$i]/$years;
$label =~ s/\d+$//;
@{$GraphData[0]}[$i] = $label.average_str(@{$GraphData[1]}[$i]);
$i++;
}
$averagesum = "A=".average_str($totalsum/$years).", ";
} elsif ($average) {
$settings->{grouping_time} =~ s/totals$/average/;
my @counts;
if ($settings->{grouping_time} =~ /^monthly.*/) {
@counts = month_counts($data{'misc'}{'firsttotal'},$data{'misc'}{'lasttotal'});
} elsif ($settings->{grouping_time} =~ /^yeardaily.*/) {
@counts = yearday_counts($data{'misc'}{'firsttotal'},$data{'misc'}{'lasttotal'});
} elsif ($settings->{grouping_time} =~ /^hourly.*/) {
@counts = hour_counts($data{'misc'}{'firsttotal'},$data{'misc'}{'lasttotal'});
} elsif ($settings->{grouping_time} =~ /^weekdaily.*/) {
@counts = weekday_counts($data{'misc'}{'firsttotal'},$data{'misc'}{'lasttotal'});
} elsif ($settings->{grouping_time} =~ /^daily.*/) {
@counts = day_counts($data{'misc'}{'firsttotal'},$data{'misc'}{'lasttotal'});
}
if ($cutofftime > 0) {
$cutofftime--;
$currenttime--;
}
my $totcount = 0;
for (my $time=$cutofftime; $time<=$currenttime; $time+=$xincr) {
my $idx = $time-$cutofftime;
$totcount += $counts[$time];
my $i = 0;
foreach my $datatype (@{$settings->{'data_types'}}) {
$i++;
if ($counts[$time] && @{$GraphData[$i]}[$idx]) {
@{$GraphData[$i]}[$idx] = @{$GraphData[$i]}[$idx]/$counts[$time];
}
}
}
my $i=0;
foreach my $spec (@Legend) {
my $num = $spec;
$spec =~ s/ \(\d*\)$//;
$num =~ s/.* \((\d*)\)$/$1/;
$Legend[$i] = "$spec (A=".average_str($num/$totcount).")";
#$Legend[$i] = "$spec (A=".average_str($num/$totcount).", T=$num)";
#$Legend[$i] = "$spec (T=$num)";
$i++;
}
$averagesum = "A=".average_str($totalsum/$totcount).", ";
} elsif (!$noaxis) {
my $i=0;
foreach my $spec (@Legend) {
my $num = $spec;
$spec =~ s/ \(\d*\)$//;
$num =~ s/.* \((\d*)\)$/$1/;
$Legend[$i] = "$spec (T=$num)";
$i++;
}
}
#print "Debug get_graph_data 500\n";
# If we have no legend, create one so graph doesn't error
push @Legend,"No values of this type!" if (!@Legend);
@{$settings->{legend}} = @Legend;
@{$settings->{'data_types'}} = @allorg if ($allset);
#$settings->{x_label} = "$settings->{x_label}: $totalsum" if ($noaxis);
my $sumdisp = "T=$totalsum";
$sumdisp = "$sumdisp, GT=$grandtotalsum" if ($totalsum != $grandtotalsum);
$settings->{chart_title} = "$settings->{chart_title} ($averagesum$sumdisp)"; #if (!$noaxis);
#if (open DF, "> $OUTPUT_DIR/$settings->{chart_filename}.datadump") {
# print DF dump(@GraphData);
# close DF;
#}
return @GraphData;
}
sub draw_graph($$$$) {
my $graph_type = shift;
my $settings = shift;
my $data = shift;
my $my_graph;
my $noaxis = ($graph_type eq 'pie');
my $nolegend = ($noaxis || (defined($settings->{legend_columns}) && $settings->{legend_columns}<1));
my $ysize = $settings->{y_graph_size};
if ($noaxis) {
if ($graph_type eq 'pie') {
$my_graph = new GD::Graph::pie($settings->{x_graph_size}, $ysize);
$my_graph->set(
label => $settings->{y_label},
'3d' => 1,
start_angle => 95,
);
} else {
die ("Invalid GraphSettings{graph_type} = $graph_type");
}
} else {
if ($graph_type =~ /hbar$/ && $settings->{compute_bars_sz}) {
$ysize = 49+($ysize*@{@$data[1]});
}
if (defined($settings->{legend_columns}) && $settings->{legend_columns} > 0 &&
$settings->{add_legend_sz}) {
my $legrows = @{$settings->{legend}}/$settings->{legend_columns};
$legrows = (int($legrows+1)) if ($legrows != int($legrows));
$ysize = $ysize + (16*$legrows);
#print "$legrows => $ysize\n";
}
my $countsum = 0;
if ($graph_type eq 'line') {
$my_graph = new GD::Graph::linespoints($settings->{x_graph_size}, $ysize);
$my_graph->set(
marker_size => 0,
line_width => 1,
);
} elsif ($graph_type =~ /^stacked_(h?)bar$/) {
if ($1 eq 'h') {
$my_graph = new GD::Graph::hbars($settings->{x_graph_size}, $ysize);
} else {
$my_graph = new GD::Graph::bars($settings->{x_graph_size}, $ysize);
}
$my_graph->set(
cumulate => 1,
correct_width => 0,
);
$countsum = 1;
} elsif ($graph_type eq 'stacked_area') {
$my_graph = new GD::Graph::area($settings->{x_graph_size}, $ysize);
$my_graph->set(
marker_size => 0,
line_width => 0,
cumulate => 1,
);
$countsum = 1;
} elsif ($graph_type =~ /^(h?)bar$/) {
if ($1 eq 'h') {
$my_graph = new GD::Graph::hbars($settings->{x_graph_size}, $ysize);
} else {
$my_graph = new GD::Graph::bars($settings->{x_graph_size}, $ysize);
}
$my_graph->set(
marker_size => 0,
line_width => 0,
correct_width => 0,
cumulate => 0,
overwrite => 0,
);
$my_graph->set(cycle_clrs => 1)
if (@$data<3 && defined($settings->{legend_columns}) && ($settings->{legend_columns}<1));
} else {
die ("Invalid GraphSettings{graph_type} = $graph_type");
}
$my_graph->set(lg_cols => $settings->{legend_columns})
if (defined($settings->{legend_columns}) && $settings->{legend_columns} > 0);
my $maxvalue = 0;
if ($countsum) {
my @sums = ();
for (my $i=1; $i<@$data; $i++) {
for (my $j=0; $j<@{@$data[$i]}; $j++) {
$sums[$j]+=@{@$data[$i]}[$j];
}
}
foreach my $sum (@sums) {
$maxvalue = $sum if ($sum > $maxvalue);
}
} else {
for (my $i=1; $i<@$data; $i++) {
for (my $j=0; $j<@{@$data[$i]}; $j++) {
$maxvalue = @{@$data[$i]}[$j] if (@{@$data[$i]}[$j] > $maxvalue);
}
}
}
$maxvalue = int($maxvalue+1) if ($maxvalue != int($maxvalue));
my $ticks = int(sprintf('%0.1g',$maxvalue / 10));
if ($ticks < 10 && $ticks > 5) {
$ticks = 10;
} elsif ($ticks < 5 && $ticks >2) {
$ticks = 5;
} elsif ($ticks == 0) {
$ticks = 1;
}
while ($maxvalue % $ticks > 0) {
$maxvalue++;
}
$ticks = $maxvalue/$ticks;
my $skip = 1;
$skip = 2 if ($ticks > 20);
if ($maxvalue == 0) {
print STDERR "$graph_type \"$settings->{chart_title}\": max y value is zero!\n";
return;
}
$my_graph->set(
y_label => $settings->{y_label},
x_labels_vertical => ($graph_type !~ /^(stacked_)?hbar$/),
x_label_position => 1/2,
y_tick_number => $ticks,
y_max_value => $maxvalue,
y_min_value => 0,
y_label_skip => $skip,
long_ticks => 1,
skip_undef => 1,
);
if (!$nolegend) {
my $xticks = 1;
$xticks = 0 if (@{@$data[0]} > $settings->{x_graph_size} / 4);
my $xskip = 1;
if (@{@$data[0]} > 200) {
$xskip = 10;
} elsif (@{@$data[0]} > 100) {
$xskip = 5;
} elsif (@{@$data[0]} > 69) {
$xskip = 2;
}
#my $cnti = @{@$data[0]};
#print "$cnti:$xskip\n";
$my_graph->set(
x_label => $settings->{x_label},
x_label_skip => $xskip,
x_tick_offset => $xskip-1,
);
}
}
$my_graph->set(
bgclr => 'white',
fgclr => 'gray',
boxclr => 'lgray',
transparent => 0,
title => $settings->{chart_title},
);
$my_graph->set( dclrs => [ qw(lgreen lred lblue lyellow lpurple cyan lorange dred dgreen dblue dyellow dpurple) ] );
$my_graph->set_legend( @{$settings->{legend}} )
if (!$nolegend);
$my_graph->plot($data);
save_chart($my_graph, "$OUTPUT_DIR/$settings->{chart_filename}");
}
sub save_chart($$) {
my $chart = shift or die "Need a chart!";
my $name = shift or die "Need a name!";
local(*OUT);
my $ext = $chart->export_format;
open(OUT, ">$name.png") or
die "Cannot open $name.$ext for write: $!";
binmode OUT;
print OUT $chart->gd->png;
close OUT;
}
sub save_totals_interval() {
local(*OUT);
open(OUT, ">$OUTPUT_DIR/.totals_interval") or
die "Cannot open .totals_interval for write: $!";
print OUT time2str('%Y-%m-%d %H:%M %z',$data{'misc'}{'firsttotal'})."\n";
print OUT time2str('%Y-%m-%d %H:%M %z',$data{'misc'}{'lasttotal'})."\n";
close OUT;
}
1;
[graphdefang-config] [event/mimedefang.pl/general] [event/mimedefang.pl/greylist] [event/sendmail/reject] [event/sendmail/user_unknown]
Below are my configuration and custom event files for Graphdefang. They are parsing MDLOG data as well as lines from sendmail and my custom MIMEDefang log lin es (the greylisting stuff for example).
#!/usr/bin/perl -w
# $Id: graphdefang-config,v 1.1.10 2003/12/16 16:26:00 jonas Exp $
#
# GraphDefang -- a set of tools to create graphs of your mimedefang
# spam and virus logs.
#
# Written by: John Kirkland
# jpk@bl.org
#
# Copyright (c) 2002-2003, John Kirkland
#
# Modified by Jonas Eckerman, 2003
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#=============================================================================
#
# Path to incoming log file or log files
# If you have only one log file, use $DATAFILE.
#
# If you have more than one log file to parse, say from different hosts,
# use @DATAFILES as follows and you MUST COMMENT OUT OR DELETE $DATAFILE:
#
# $DATAFILES[0] = '/var/log/maillog.host1';
# $DATAFILES[1] = '/var/log/maillog.host2';
$DATAFILES[0] = '/var/log/maillog.7';
$DATAFILES[1] = '/var/log/maillog.6';
$DATAFILES[2] = '/var/log/maillog.5';
$DATAFILES[3] = '/var/log/maillog.4';
$DATAFILES[4] = '/var/log/maillog.3';
$DATAFILES[5] = '/var/log/maillog.2';
$DATAFILES[6] = '/var/log/maillog.1';
$DATAFILES[7] = '/var/log/maillog.0';
$DATAFILES[8] = '/var/log/maillog';
#$DATAFILE = './maillog';
#
# If true, the log file will only be parsed until the last entry
# before the last whole hour.
#
$WHOLE_HOURS = 1;
#
# Optional Timezone variable by host name. The host name must match
# the host name presented in the syslog file(s). This variable is
# useful when you have a central syslog server collecting logs for
# machines that are in different timezones. By default, graphdefang
# uses the timezone that is local to the machine upon which it is
# running. It is not necessary to define the TZ for EVERY host, but
# only for the ones that aren't in the same timezone as the log
# server. The timezone must be understood by the Time::Zone perl
# module.
#
# $TZ{'westover'} = 'cst6cdt';
# $TZ{'GD_Display'} = 'cst6cdt';
#
#
# Output directory for png files that are created
#
$OUTPUT_DIR = '/var/spool/MIMEDefang/graphs';
# Set graph settings
#
# Possible settings:
#
# Name: data_types (required)
# Description: Array of events from the mimedefang log file to
# graph.
# Supported Values: These event names are not fixed. If you put an event
# in mimedefang-filter with md_log('event'), then you
# can use it here. The values used in the example
# mimedefang-filter are 'spam', 'virus',
# 'suspicious_chars', 'message/partial', 'bad_filename',
# 'non_rfc822', 'non_multipart'.
# 'all' is also supported, but it must be listed by itself.
#
# Name: graph_type (required)
# Description: Type of graph to output.
# Supported Values: 'line' or 'stacked_bar'
#
# Name: grouping (required)
# Description: Which value to graph from the md_log file.
# Supported Values: 'summary', 'value1', 'value2', 'sender', 'recipient'
# 'subject'. value1 and value2 are the optional
# parameters that can be logged with the md_log command
# from mimedefang-filter.
#
# Name: grouping_times (required)
# Description: Array of Time intervals to use for grouping
# Supported Values: 'hourly', 'daily', or 'monthly'
#
# Name: top_n (optional)
# Description: Limit number of values to the top n. This
# value is recommended when looking at
# senders, recipients, or subjects.
#
# Name: value1_title (optional)
# Description: Title used in the header if value1 is
# graphed.
#
# Name: value2_title (optional)
# Description: Title used in the header if value2 is
# graphed.
#
# Name: filter (optional)
# Description: Regular expression filter that can be
# used with value1, value2, sender,
# recipient, and subject
# Common uses:
# '@westover.org' to filter sender or recipient by domain
# '^(?:(?!klez).)*$' to filter OUT klez in a virusname
#
# Name: filter_name (optional)
# Description: If a filter is used, the filtername will be appended
# to the Graph Title as "filtered by $filter_name" and
# appended to the end of the filename.
#
# Name: $GraphSettings{'title'} (optional)
# Description: Assigns a title for the chart.
#
# Name: $GraphSettings{'filename'} (optional)
# Description: Sets the filename for a given chart (note:
# the grouping_time is appended to this variable
# to determine the final file name.
#
# Name: $GraphSettings{'x_graph_size'} (optional)
# Name: $GraphSettings{'y_graph_size'} (optional)
# Description: Size, in pixels, for this graph. This overrides the
# default.
#
# Name: $GraphSettings{'num_hourly_values'} (optional)
# Name: $GraphSettings{'num_daily_values'} (optional)
# Name: $GraphSettings{'num_monthly_values'} (optional)
# Description: Number of data points for this graph. This overrides
# the default.
# helo
# mail_from
# host
# suspicious_chars
# message/partial
# virus
# bad_filename
# non_multipart
# non_rfc822
# spam
# accepted
my %GraphSettings;
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '000_Summary',
'title' => 'Summary',
'data_types' => ['all','grey','white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet'],
'graph_type' => 'pie',
'grouping' => 'summary',
'grouping_times' => ['totals','yearly_average'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '600',
'y_graph_size' => '300',
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '001_Summary',
'title' => 'Summary',
'data_types' => ['all','grey','white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet'],
'graph_type' => 'hbar',
'grouping' => 'summary',
'grouping_times' => ['totals','yearly_average'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '600',
#'y_graph_size' => '300',
'y_graph_size' => 16,
'compute_bars_sz' => 1,
'legend_columns' => 0,
'reversed' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '002_IncomingSummary',
'title' => 'Incoming Summary',
'data_types' => ['all','grey','white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet','passed'],
'graph_type' => 'pie',
'grouping' => 'summary',
'grouping_times' => ['totals','yearly_average'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '600',
'y_graph_size' => '300',
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '003_IncomingSummary',
'title' => 'Incoming Summary',
'data_types' => ['all','grey','white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet','passed'],
'graph_type' => 'hbar',
'grouping' => 'summary',
'grouping_times' => ['totals','yearly_average'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '600',
#'y_graph_size' => '300',
'y_graph_size' => 16,
'compute_bars_sz' => 1,
'legend_columns' => 0,
'reversed' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '004_Summary',
'title' => 'Summary',
'data_types' => ['all','grey','white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet'],
'graph_type' => 'stacked_area',
'grouping' => 'summary',
'grouping_times' => ['hourly','daily','monthly','hourly_totals','weekdaily_totals','daily_totals','yeardaily_totals','monthly_totals','yearly_totals','hourly_average','weekdaily_average','daily_average','yeardaily_average','monthly_average'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '300',
'legend_columns' => 4,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '007_Incoming',
'title' => 'Incoming',
'data_types' => ['all','grey','white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet','passed'],
'graph_type' => 'stacked_area',
'grouping' => 'summary',
'grouping_times' => ['hourly','daily','monthly','hourly_totals','weekdaily_totals','daily_totals',,'yeardaily_totals','monthly_totals','yearly_totals','hourly_average','weekdaily_average','daily_average','yeardaily_average','monthly_average'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '300',
'legend_columns' => 4,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '010_Rejected',
'title' => 'Rejected',
'data_types' => ['all','grey','accepted','passed','white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet'],
#'data_types' => ['unknown_user','spam','helo','host','virus','mail_from','suspicious_chars','message/partial','bad_filename','non_multipart','non_rfc822','check_mail','check_relay','check_rcpt'],
'graph_type' => 'stacked_area',
'grouping' => 'summary',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '300',
'legend_columns' => 5,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '020_Accepted',
'title' => 'Accepted',
'value1_title' => 'Host',
'value2_title' => 'Relay',
'data_types' => ['accepted'],
'graph_type' => 'stacked_area',
'grouping' => 'recipientdomain',
#'top_n' => '24',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '300',
'legend_columns' => 4,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '025_Passed',
'title' => 'Passed',
'value1_title' => 'Host',
'value2_title' => 'Relay',
'data_types' => ['passed'],
'graph_type' => 'stacked_area',
'grouping' => 'value1',
#'top_n' => '24',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 4,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '028_Blocked',
'title' => 'Blocked',
'value1_title' => 'Match',
'value2_title' => 'Relay',
'data_types' => ['helo','mail_from','host'],
'graph_type' => 'stacked_bar',
'grouping' => 'summary',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '300',
);
#push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '030_AcceptedBySystem',
'title' => 'Accepted by Operating System',
'value1_title' => 'Host',
'value2_title' => 'Relay',
'value3_title' => 'System',
'data_types' => ['accepted'],
'graph_type' => 'stacked_area',
'grouping' => 'value3',
#'top_n' => '24',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 6,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '032_RejectedBySystem',
'title' => 'Rejected by Operating System',
'value3_title' => 'System',
'data_types' => ['all','grey','accepted','passed','white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet'],
'graph_type' => 'stacked_area',
'grouping' => 'value3',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 6,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '033_SpamBySystem',
'title' => 'Spam by Operating System',
'value3_title' => 'System',
'data_types' => ['spam'],
'graph_type' => 'stacked_area',
'grouping' => 'value3',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 6,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '035_AcceptedByRecipient',
'title' => 'Accepted by recipient',
'value1_title' => 'Host',
'value2_title' => 'Relay',
'data_types' => ['accepted'],
'graph_type' => 'stacked_bar',
'grouping' => 'recipient',
'top_n' => '9',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 3,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '036_AcceptedBySender',
'title' => 'Accepted by sender',
'value1_title' => 'Host',
'value2_title' => 'Relay',
'data_types' => ['accepted'],
'graph_type' => 'stacked_bar',
'grouping' => 'senderdomain',
'top_n' => '9',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 3,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '040_SpamByRecipient',
'title' => 'Spam by recipient',
'value1_title' => 'Score',
'value2_title' => 'Relay',
'data_types' => ['spam'],
'graph_type' => 'stacked_bar',
'grouping' => 'recipient',
'top_n' => '9',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 3,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '050_SpamBySender',
'title' => 'Spam by sender',
'value1_title' => 'Score',
'value2_title' => 'Relay',
'data_types' => ['spam'],
'graph_type' => 'stacked_bar',
'grouping' => 'senderdomain',
'top_n' => '9',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 3,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '070_ViriiByVirus',
'title' => 'Virii by virus',
'value1_title' => 'VirusName',
'value2_title' => 'Relay',
'data_types' => ['virus'],
'graph_type' => 'stacked_area',
'grouping' => 'value1',
#'top_n' => '9',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 4,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '080_ViriiByRecipient',
'title' => 'Virii by Recipient',
'value1_title' => 'VirusName',
'value2_title' => 'Relay',
'data_types' => ['virus'],
'graph_type' => 'bar',
'grouping' => 'recipient',
'top_n' => '9',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '300',
);
#push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '505_Greylist_Events',
'title' => 'Greylist Events Summary',
'value1_title' => 'Count',
'value2_title' => 'Relay',
'data_types' => ['white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet'],
'graph_type' => 'pie',
'grouping' => 'summary',
#'top_n' => '9',
'grouping_times' => ['totals'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '600',
'y_graph_size' => '300',
);
push @GRAPHS, { %GraphSettings };
#-------------------------------------------------------------
%GraphSettings = ();
%GraphSettings = (
'filename' => '510_greylist_events',
'title' => 'Greylist Events Summary',
'value1_title' => 'Count',
'value2_title' => 'Relay',
'data_types' => ['white_triplet','black_triplet','new_triplet','old_triplet','reset_triplet'],
'graph_type' => 'stacked_area',
'grouping' => 'summary',
#'top_n' => '9',
'grouping_times' => ['hourly','daily','monthly'],
'num_hourly_values' => '48',
'num_daily_values' => '61',
'num_monthly_values' => '24',
'x_graph_size' => '700',
'y_graph_size' => '250',
'legend_columns' => 5,
'add_legend_sz' => 1,
);
push @GRAPHS, { %GraphSettings };
#!/usr/bin/perl -w
# Sample Rows from mimedefang's md_log()
#Sep 28 21:55:50 westover mimedefang.pl[16803]: MDLOG,g8T2th86016917,mail_out,,,<mimedefang@westover.org>,<defang-admin@westover.org>,SPAM: 21.9, Make Money 100% RISK FREE!
#Sep 28 21:55:52 westover mimedefang.pl[16803]: MDLOG,g8T2th88016917,mail_out,,,<mimedefang@westover.org>,<defang-admin@westover.org>,SPAM: 20.3, Access Your PC from Anywhere - Download Now
#Sep 28 21:55:55 westover mimedefang.pl[16803]: MDLOG,g8T2th8A016917,mail_out,,,<mimedefang@westover.org>,<defang-admin@westover.org>,SPAM: 32.8, /ADV / The Best Business Opportunity on Net
#Sep 28 21:55:57 westover mimedefang.pl[16803]: MDLOG,g8T2th8C016917,mail_out,,,<mimedefang@westover.org>,<defang-admin@westover.org>,SPAM: 16.6, Get the lowest long distance rates available.
#Sep 28 21:56:02 westover mimedefang.pl[16803]: MDLOG,g8T2tt82016932,spam,38,203.167.97.19,<joisie69@hotmail.com>,<vernonm@westover.org>,I did not belive it....
#Sep 28 21:56:09 westover mimedefang.pl[16803]: MDLOG,g8T2u782016945,mail_out,,,<mimedefang@westover.org>,<defang-admin@westover.org>,SPAM: 38, I did not belive it....
use p0fIP2OS;
$event{'mimedefang.pl'}{'general'} =
sub {
if ($text =~ m/^MDLOG,\S+?,(\S+?),(\S*?),(\S*?),(.*?),(.*?),(.*)$/ ) {
# get values from regular expression
# Only summarize data if it is newer than our current MaxDBUnixTime
if ($unixtime > $MaxDBUnixTime) {
$event = $1;
$value1 = $2;
$value2 = $3;
$sender = $4;
$recipient = $5;
$subject = $6;
if ($value2) {
$value3 = ip2os($value2);
#$value4 = $dns{$value2};
#$value4 = $value2 if (!$value4);
}
$value1 = "Localhost/Auth" if ($value1 =~ /^127\.0\.0\.1$/);
$value1 = "Localhost/Auth" if ($value1 =~ /^localhost(\.i)?(\.frukt\.org)?$/);
$value1 = "Local User" if ($value1 =~ /^10-0-[234]-[0-9]+\.i\.frukt\.org$/i);
$value1 = "Local User" if ($value1 =~ /^\[10\.0\.[234]\.[0-9]+\]$/);
$value1 = "VPN User" if ($value1 =~ /^10-0-(8|37|42)-[0-9]+\.i\.frukt\.org$/i);
$value1 = "VPN User" if ($value1 =~ /^\[10\.0\.(8|37|42)\.[0-9]+\]$/);
$FoundNewRow = 1;
}
}
};
#!/usr/bin/perl -w
#gAABdPd17840: ruleset=check_rcpt, arg1=<0206241317.54101.bj@bl.org>, relay=200-207-163-140.dsl.telesp.net.br [200.207.163.140], reject=550 5.0.0 <0206241317.54101.bj@bl.org>... We dont accept mail from br
#gAF0bmd11725: ruleset=check_rcpt, arg1=<mparson@bl.org>, relay=[209.163.156.31], reject=451 4.1.8 Domain of sender address pqwa625@100pesos.com does not resolve
#h1H0wArt006658: ruleset=check_mail, arg1=<bluefunkshorts2002@yahoo.co.uk>, relay=ip68-106-211-59.om.om.cox.net [68.106.211.59], reject=550 5.0.0 <bluefunkshorts2002@yahoo.co.uk>... Error
#Dec 1 19:06:40 chip sm-mta[25849]: hB1I3wXT025849: ruleset=check_mail, arg1=<122nan@gorgeousgeorge.us>, relay=as19-5-5.lk.bonet.se [217.215.176.117], reject=451 4.1.8 Domain of sender address 122nan@gorgeousgeorge.us does not resolve
#Dec 1 19:09:23 chip sm-mta[25932]: hB1I6eXT025932: ruleset=check_mail, arg1=<hwlmc4tih@gorgeousgeorge.us>, relay=as19-5-5.lk.bonet.se [217.215.176.117], reject=451 4.1.8 Domain of sender address hwlmc4tih@gorgeousgeorge.us does not resolve
#Dec 1 19:12:34 chip sm-mta[26053]: hB1ICVXT026053: ruleset=check_rcpt, arg1=<bob.henson@the>, relay=[203.156.64.103], reject=550 5.7.1 <bob.henson@the>... Relaying denied. IP name lookup failed [203.156.64.103]
use p0fIP2OS;
$event{'sm-mta'}{'reject'} =
sub {
if ($text =~ m/^.* ruleset=(.*), arg1=(.*), relay=(.*), reject=([0-9]+) (.*)$/) {
if ($unixtime > $MaxDBUnixTime) {
$event = $1;
$sender = $2;
$value1 = "$4 $5";
$value2 = $3;
#$value1 = "Localhost/Auth" if ($value1 =~ /^127\.0\.0\.1$/);
#$value1 = "Localhost/Auth" if ($value1 =~ /^localhost(\.i)?(\.frukt\.org)?$/);
#$value1 = "Local User" if ($value1 =~ /^10-0-[234]-[0-9]+\.i\.frukt\.org$/i);
#$value1 = "Local User" if ($value1 =~ /^\[10\.0\.[234]\.[0-9]+\]$/);
#$value1 = "VPN User" if ($value1 =~ /^10-0-(8|37|42)-[0-9]+\.i\.frukt\.org$/i);
#$value1 = "VPN User" if ($value1 =~ /^\[10\.0\.(8|37|42)\.[0-9]+\]$/);
$value2 =~ s/.*\[(\d+\.\d+\.\d+\.\d+)\].*/$1/;
if ($value2) {
$value3 = ip2os($value2);
#$value4 = $dns{$value2};
#$value4 = $value2 if (!$value4);
}
$FoundNewRow = 1;
}
}
};
#!/usr/bin/perl -w
#gAABdPd17840: ruleset=check_rcpt, arg1=<0206241317.54101.bj@bl.org>, relay=200-207-163-140.dsl.telesp.net.br [200.207.163.140], reject=550 5.0.0 <0206241317.54101.bj@bl.org>... We dont accept mail from br
#gAF0bmd11725: ruleset=check_rcpt, arg1=<mparson@bl.org>, relay=[209.163.156.31], reject=451 4.1.8 Domain of sender address pqwa625@100pesos.com does not resolve
#h1H0wArt006658: ruleset=check_mail, arg1=<bluefunkshorts2002@yahoo.co.uk>, relay=ip68-106-211-59.om.om.cox.net [68.106.211.59], reject=550 5.0.0 <bluefunkshorts2002@yahoo.co.uk>... Error
#Dec 1 19:06:40 chip sm-mta[25849]: hB1I3wXT025849: ruleset=check_mail, arg1=<122nan@gorgeousgeorge.us>, relay=as19-5-5.lk.bonet.se [217.215.176.117], reject=451 4.1.8 Domain of sender address 122nan@gorgeousgeorge.us does not resolve
#Dec 1 19:09:23 chip sm-mta[25932]: hB1I6eXT025932: ruleset=check_mail, arg1=<hwlmc4tih@gorgeousgeorge.us>, relay=as19-5-5.lk.bonet.se [217.215.176.117], reject=451 4.1.8 Domain of sender address hwlmc4tih@gorgeousgeorge.us does not resolve
#Dec 1 19:12:34 chip sm-mta[26053]: hB1ICVXT026053: ruleset=check_rcpt, arg1=<bob.henson@the>, relay=[203.156.64.103], reject=550 5.7.1 <bob.henson@the>... Relaying denied. IP name lookup failed [203.156.64.103]
use p0fIP2OS;
$event{'sm-mta'}{'reject'} =
sub {
if ($text =~ m/^.* ruleset=(.*), arg1=(.*), relay=(.*), reject=([0-9]+) (.*)$/) {
if ($unixtime > $MaxDBUnixTime) {
$event = $1;
$sender = $2;
$value1 = "$4 $5";
$value2 = $3;
#$value1 = "Localhost/Auth" if ($value1 =~ /^127\.0\.0\.1$/);
#$value1 = "Localhost/Auth" if ($value1 =~ /^localhost(\.i)?(\.frukt\.org)?$/);
#$value1 = "Local User" if ($value1 =~ /^10-0-[234]-[0-9]+\.i\.frukt\.org$/i);
#$value1 = "Local User" if ($value1 =~ /^\[10\.0\.[234]\.[0-9]+\]$/);
#$value1 = "VPN User" if ($value1 =~ /^10-0-(8|37|42)-[0-9]+\.i\.frukt\.org$/i);
#$value1 = "VPN User" if ($value1 =~ /^\[10\.0\.(8|37|42)\.[0-9]+\]$/);
$value2 =~ s/.*\[(\d+\.\d+\.\d+\.\d+)\].*/$1/;
if ($value2) {
$value3 = ip2os($value2);
#$value4 = $dns{$value2};
#$value4 = $value2 if (!$value4);
}
$FoundNewRow = 1;
}
}
};
#!/usr/bin/perl -w
# matching against:
#Feb 16 18:58:13 westover sendmail[6660]: h1H0wCrt006660: <andresg@moi.net>... User unknown
#Feb 16 18:58:13 westover sendmail[6660]: h1H0wCrt006660: <andref@moi.net>... User unknown
#Feb 16 18:58:13 westover sendmail[6660]: h1H0wCrt006660: <andrea@moi.net>... User unknown
#Feb 16 18:58:14 westover sendmail[6660]: h1H0wCrt006660: <andrae@moi.net>... User unknown
#Feb 16 18:58:15 westover sendmail[6660]: h1H0wCrt006660: <andie@moi.net>... User unknown
#Feb 16 18:58:16 westover sendmail[6660]: h1H0wCrt006660: <andreaj@moi.net>... User unknown
#Feb 16 18:58:17 westover sendmail[6660]: h1H0wCrt006660: <anderso1@moi.net>... User unknown
#Feb 16 18:58:18 westover sendmail[6660]: h1H0wCrt006660: <andrewb@moi.net>... User unknown
#Feb 16 18:58:19 westover sendmail[6660]: h1H0wCrt006660: <andrew1@moi.net>... User unknown
#Feb 16 18:58:20 westover sendmail[6660]: h1H0wCrt006660: <andre_b@moi.net>... User unknown
#Feb 16 18:58:21 westover sendmail[6660]: h1H0wCrt006660: <andrade@moi.net>... User unknown
#Feb 16 18:58:22 westover sendmail[6660]: h1H0wCrt006660: <andih@moi.net>... User unknown
#Feb 16 18:58:23 westover sendmail[6660]: h1H0wCrt006660: <anderton@moi.net>... User unknown
#Feb 16 18:58:24 westover sendmail[6660]: h1H0wCrt006660: <705aO12943L14764c67@moi.net>... User unknown
#Feb 16 18:58:25 westover sendmail[6660]: h1H0wCrt006660: <andreasd@moi.net>... User unknown
#Feb 16 18:58:26 westover sendmail[6660]: h1H0wCrt006660: <andream@moi.net>... User unknown
#Feb 16 18:58:27 westover sendmail[6660]: h1H0wCrt006660: <andreah@moi.net>... User unknown
#Feb 16 18:58:28 westover sendmail[6660]: h1H0wCrt006660: <andrea2@moi.net>... User unknown
#Feb 16 18:58:29 westover sendmail[6660]: h1H0wCrt006660: <andreh@moi.net>... User unknown
#Feb 16 18:58:30 westover sendmail[6660]: h1H0wCrt006660: <andreasw@moi.net>... User unknown
#Feb 16 18:58:31 westover sendmail[6660]: h1H0wCrt006660: <andrev@moi.net>... User unknown
#Feb 16 18:58:32 westover sendmail[6660]: h1H0wCrt006660: <andino@moi.net>... User unknown
#Feb 16 18:58:33 westover sendmail[6660]: h1H0wCrt006660: <andresr@moi.net>... User unknown
#Feb 16 18:58:34 westover sendmail[6660]: h1H0wCrt006660: <andreg@moi.net>... User unknown
#Feb 16 18:58:35 westover sendmail[6660]: h1H0wCrt006660: <andreaz@moi.net>... User unknown
#Feb 16 18:58:36 westover sendmail[6660]: h1H0wCrt006660: <andreap@moi.net>... User unknown
#Feb 16 18:58:37 westover sendmail[6660]: h1H0wCrt006660: <andersom@moi.net>... User unknown
#Feb 16 18:58:38 westover sendmail[6660]: h1H0wCrt006660: <andrejs@moi.net>... User unknown
#Feb 16 18:58:38 westover sendmail[6660]: h1H0wCrt006660: from=<jennyhorseface@yahoo.co.uk>, size=0, class=0, nrcpts=0, proto=ESMTP, daemon=MTA, relay=cae168-215-231.sc.rr.com [24.168.215.231]
use p0fIP2OS;
$event{'sendmail'}{'user_unknown'} =
sub {
if ($text =~ m/^(\S+): from=(.+), size=.+ nrcpts=0, proto=.+ relay=(.*)$/) {
# Create a temp data hash to store the from and relay info for
# user unknown attempts if there were no valid recipients in the
# entire message (nrcpts=0).
if ($unixtime > ($MaxDBUnixTime)) {
my $id = $1;
my $from = $2;
my $relay = $3;
$user_unknown{$id}{'from'} = $2;
$user_unknown{$id}{'relay'} = $3;
}
} elsif ($text =~ m/^(\S+): (\<\S+\>)\.\.\. User unknown$/) {
if ($unixtime > $MaxDBUnixTime) {
$event = 'user_unknown';
my $id = $1;
$recipient = $2;
# extract the domain from the unknown user's email address
my $domain;
if ($recipient =~ m/<.*@(.*)>/) {
$domain = $1;
}
$value1 = "none";
$value1 = $domain if defined($domain);
# extract the 'from' and 'relay' from the temp user_unknown hash
$sender = "unknown";
$sender = $user_unknown{$id}{'from'} if (defined($user_unknown{$id}{'from'}));
$value2 = "none";
if (defined($user_unknown{$id}{'relay'})) {
$value2 = $user_unknown{$id}{'relay'};
$value2 =~ s/.*\[(\d+\.\d+\.\d+\.\d+)\].*/$1/;
if ($value2) {
$value3 = ip2os($value2);
#$value4 = $dns{$value2};
#$value4 = $value2 if (!$value4);
}
}
$FoundNewRow = 1;
}
}
};
(2006-11-18)