#!/usr/bin/perl
#***********************************************************************
#
# mdf-log-handler
#
# mimedefang-filter mail log real time analyzer
#
# can be used like this in syslogd.conf:
# mail.info |exec /usr/local/sbin/mdf-log-handler.pl
#
# $Id: mdf-log-handler.pl,v 1.10 2009/01/16 17:54:37 jonas Exp $
#
# This program may be distributed under the terms of the GNU General
# Public License, Version 2, or (at your option) any later version.
#
#***********************************************************************

use strict;
use Socket;
use DBI;
use Sys::Syslog;
use Date::Parse;
use Date::Format;
use Text::CSV_XS;

my $debug = 0;

#***********************************************************************
# Configuration.
#***********************************************************************

my %Features;
$Features{'Path:SPOOLDIR'} = '/var/spool/MIMEDefang';

# Add setting to config parser
my %cfgcfg = ();
$cfgcfg{'@i'}{i} = 0;
$cfgcfg{'@i'}{c} = 0;
sub add_cfg_cfg {
	my ($c,$v,$d,$t,$f) = @_;
	$d = '' unless (defined($d));
	$t = 's' unless ($t);
	$f = '' unless ($f);
	$c =~ s/[-_]+//g;
	$c = lc($c);
	$$v = $d;
	$cfgcfg{'@i'}{c} ++;
	$cfgcfg{$c}{v} = $v;
	$cfgcfg{$c}{t} = lc($t);
	$cfgcfg{$c}{f} = lc($f);
	$cfgcfg{$c}{i} = $cfgcfg{'@i'}{c};
	$cfgcfg{$c}{x} = 0;
}

use vars qw($WhiteNets);
add_cfg_cfg('WhiteNets',\$WhiteNets,'10.0.0.0/255.255.0.0','mn');

use vars qw($debug_loglevel);
add_cfg_cfg('DebugLogLevel',\$debug_loglevel,0,'i');

use vars qw($dc_unknown_user $dc_lost_conn $dc_timeout $dc_bad_sender $dc_noop $dc_abuse);
add_cfg_cfg('dc_unknown_user',\$dc_unknown_user,0,'b');
add_cfg_cfg('dc_lost_connection',\$dc_lost_conn,0,'b');
add_cfg_cfg('dc_timeout',\$dc_timeout,0,'b');
add_cfg_cfg('dc_bad_sender',\$dc_bad_sender,0,'b');
add_cfg_cfg('dc_noop',\$dc_noop,0,'b');
add_cfg_cfg('dc_abuse',\$dc_abuse,0,'b');

use vars qw($database_spec $database_user $database_pass);
add_cfg_cfg('database_spec',\$database_spec,'dbi:SQLite:dbname=%s/filterdata.db','p');
add_cfg_cfg('database_user',\$database_user,'','s');
add_cfg_cfg('database_pass',\$database_pass,'','s');


#***********************************************************************
# Code.
#***********************************************************************

# Some initialization
openlog("mdlogh","pid","mail");
my $sqldb = undef;
my $sqldbd = '?';
read_cfg_cfg();
$debug_loglevel = 999 if ($debug);
my $logssv = Text::CSV_XS->new({sep_char=>';',quote_char=>'"',binary=>1});

unlink('/tmp/mdlogh.log') if ($debug && (-f '/tmp/mdlogh.log'));
sub logfile {
	return unless ($debug);
	return unless (open(LF,'>>','/tmp/mdlogh.log'));
	foreach my $ll (@_) {
		$ll =~ s/[\r\n]+$//s;
		$ll =~ s/^[\r\n]+//s;
		foreach my $l (split(/[\r\n]+/,$ll)) {
			print LF sprintf("%s:  %s\n",time2str('%Y-%m-%m %H:%M:%S',time()),$l) if ($l ne '');
		}
	}
	close(LF);
}

sub logsys {
	syslog(@_) unless ($debug);
	print sprintf("%s: %s\n",shift @_,sprintf(shift @_,@_)) if ($debug);
}
sub md_syslog {
	logsys(shift @_,'%s',@_);
}
sub log_msg {
	logsys('notice',@_);
}
sub debug_log {
	my $level = shift;
	my $msg = shift;
	logsys($level<0?'err':'info',"MDLDBG: $msg",@_) unless ($level>$debug_loglevel);
}

# Get a file path name 
sub get_file_path_name {
	my $f = shift;
	return $f if ($f =~ /[\/\\]/);
	foreach my $d ((sprintf('%s/%s',$Features{'Path:CONFDIR'},'filter'),$Features{'Path:CONFDIR'},
			'/usr/local/etc/mimedefang/filter','/etc/mimedefang/filter',
			'/usr/local/etc/mimedefang','/etc/mimedefang',
			'/usr/local/etc/mail','/etc/mail', )) {
		if ($d =~ /mimedefang/) {
			return "$d/$f" if (-f "$d/$f");
			return "$d/mimedefang-$f" if (-f "$d/mimedefang-$f");
		} else {
			return "$d/mimedefang-$f" if (-f "$d/mimedefang-$f");
			return "$d/$f" if (-f "$d/$f");
		}
	}
	return '';
}

# Read file into array
sub read_a_file {
	my ($fn,$match,$noclean) = @_;
	my @l = ();
	if ($fn && (-f $fn) && (open(CF,'<',$fn))) {
		while (my $l = <CF>) {
			unless ($noclean) {
				next if ($l =~ /^[;#]/);
				next if ($l =~ /^[\s\r\n]*$/s);
				next if ($match && $l !~ /^$match/i);
			}
			push @l, $l;
		}
		close(CF);
	}
	return \@l;
}

# read the configuration file
sub read_cfg_cfg {
	my $cfgfn = get_file_path_name('filter.conf');
	die('No filter config!') unless ($cfgfn);
	die('Cannot read filter config!') unless (open(F,'<',$cfgfn));
	md_syslog('info',"Filter config: $cfgfn");
	while (my $l = <F>) {
		$l =~ s/[\r\n]+//gs;
		next unless ($l);
		next if ($l =~ /^\s*[#;]/);
		if ($l =~ /^\$(\S+):\s*(.*?)\s*\$?$/) {
			md_syslog('info',"Config $1: $2");
		} elsif ($l =~ /^\s*(\S+)\s*?[\s:=]\s*(\S.*?)\s*$/) {
			my $c = lc($1);
			my $v = $2;
			$c =~ s/[-_]+//g;
			next unless ($c);
			next if ($c =~ /^\@/);
			if (defined($cfgcfg{$c})) {
				if ($cfgcfg{$c}{x} && $cfgcfg{$c}{t} =~ /^m/i) {
					${$cfgcfg{$c}{v}} .= ';' if (${$cfgcfg{$c}{v}} ne '');
					${$cfgcfg{$c}{v}} .= $v;
				} elsif (($cfgcfg{$c}{t} eq 'l' || $cfgcfg{$c}{t} =~ /^m/i) && $v =~ /^\s*\@\{\s*(.*?)\s*\}\s*$/) {
					my $lst = read_a_file(get_file_path_name($1,'.*\S'));
					if ($cfgcfg{$c}{t} eq 'l') {
						${$cfgcfg{$c}{v}} = '';
						foreach my $le (@{$lst}) {
							$le =~ s/[\r\n]+//gs;
							unless ($le =~ /\\[.@]/) {
								$le =~ s/\./\\./g;
								$le =~ s/\@/\\\@/g;
							}
							next unless ($le);
							${$cfgcfg{$c}{v}} .= '|' if (${$cfgcfg{$c}{v}} ne '');
							${$cfgcfg{$c}{v}} .= $le
						}
					} elsif ($cfgcfg{$c}{t} =~ /^m/i) {
						foreach my $le (@{$lst}) {
							$le =~ s/[\r\n]+//gs;
							${$cfgcfg{$c}{v}} .= ';' if (${$cfgcfg{$c}{v}} ne '');
							${$cfgcfg{$c}{v}} .= $le;
						}
					}
				} else {
					${$cfgcfg{$c}{v}} = $v;
					$cfgcfg{$c}{x} = 1;
				}
			}
		}
	}
	close(F);
	#return;
	my @ck = sort { $cfgcfg{$a}{i} <=> $cfgcfg{$b}{i} } keys %cfgcfg;
	foreach my $c (@ck) {
		next if ($c =~ /^\@/);
		${$cfgcfg{$c}{v}} = ${$cfgcfg{$cfgcfg{$c}{f}}{v}} if ($cfgcfg{$c}{f} && !${$cfgcfg{$c}{v}});
	}
	foreach my $c (@ck) {
		next if ($c =~ /^\@/);
		if ($cfgcfg{$c}{t} eq 'l') {
			${$cfgcfg{$c}{v}} = '' unless (${$cfgcfg{$c}{v}});
			${$cfgcfg{$c}{v}} =~ s/\s//g;
			unless (${$cfgcfg{$c}{v}} =~ /\\[.@]/) {
				${$cfgcfg{$c}{v}} =~ s/\./\\./g;
				${$cfgcfg{$c}{v}} =~ s/\@/\\\@/g;
			}
			${$cfgcfg{$c}{v}} =~ s/,/\|/g if (${$cfgcfg{$c}{v}} !~ /[\(\)\|\{\}]/ && ${$cfgcfg{$c}{v}} =~ /\,/);
			${$cfgcfg{$c}{v}} = sprintf('(%s)',${$cfgcfg{$c}{v}}) if (${$cfgcfg{$c}{v}} !~ /[\(\)]/ && ${$cfgcfg{$c}{v}} =~ /\|/);
		} elsif ($cfgcfg{$c}{t} eq 'a') {
			${$cfgcfg{$c}{v}} = '' unless (${$cfgcfg{$c}{v}});
			next unless (${$cfgcfg{$c}{v}});
			${$cfgcfg{$c}{v}} .= '@'.${$cfgcfg{'myfilterhostname'}{v}} if (${$cfgcfg{$c}{v}} =~ /^[^@]+$/);
		} elsif ($cfgcfg{$c}{t} eq 't') {
			${$cfgcfg{$c}{v}} = '' unless (${$cfgcfg{$c}{v}});
			${$cfgcfg{$c}{v}} = sprintf(${$cfgcfg{$c}{v}},${$cfgcfg{'myfilterhostname'}{v}});
			${$cfgcfg{$c}{v}} =~ s/[\r\n]*$/\n\n/s;
		} elsif ($cfgcfg{$c}{t} eq 'b') {
			${$cfgcfg{$c}{v}} = 0 unless (${$cfgcfg{$c}{v}});
			${$cfgcfg{$c}{v}} = 0 if (${$cfgcfg{$c}{v}} =~ /^\s*(false|no?|off|0+)\s*$/);
			${$cfgcfg{$c}{v}} = 1 if (${$cfgcfg{$c}{v}});
		} elsif ($cfgcfg{$c}{t} eq 'i') {
			${$cfgcfg{$c}{v}} = 0 unless (${$cfgcfg{$c}{v}});
			${$cfgcfg{$c}{v}} = eval(${$cfgcfg{$c}{v}});
		} elsif ($cfgcfg{$c}{t} eq 'ps') {
			${$cfgcfg{$c}{v}} = $Features{'Path:SPOOLDIR'} unless (${$cfgcfg{$c}{v}});
			${$cfgcfg{$c}{v}} = sprintf('%s/%s',$Features{'Path:SPOOLDIR'},${$cfgcfg{$c}{v}}) if (${$cfgcfg{$c}{v}} !~ /[\/\\]/);
		} elsif ($cfgcfg{$c}{t} eq 'pc') {
			${$cfgcfg{$c}{v}} = $Features{'Path:CONFDIR'} unless (${$cfgcfg{$c}{v}});
			${$cfgcfg{$c}{v}} = sprintf('%s/%s',$Features{'Path:CONFDIR'},${$cfgcfg{$c}{v}}) if (${$cfgcfg{$c}{v}} !~ /[\/\\]/);
		} elsif ($cfgcfg{$c}{t} eq 'p') {
			${$cfgcfg{$c}{v}} = $Features{'Path:CONFDIR'} unless (${$cfgcfg{$c}{v}});
			${$cfgcfg{$c}{v}} = sprintf(${$cfgcfg{$c}{v}},$Features{'Path:SPOOLDIR'},$Features{'Path:CONFDIR'});
		} elsif ($cfgcfg{$c}{t} eq 'mpsm') {
			my @pil = split(/\s*;\s*/,${$cfgcfg{$c}{v}});
			for (my $i=0;$i<@pil;$i++) {
				my ($fn,$ft,$fo,$x) = split(/\s*,\s*/,$pil[$i]);
				$fn = sprintf('%s/%s',${$cfgcfg{'sendmailconfig'}{v}},$fn) if ($fn !~ /[\/\\]/);
				$fo = '?' unless ($fo);
				unless ($ft) {
					if ($fn =~ /table/i) {
						$ft = 'table'
					} else {
						$ft = 'list';
					}
				}
				$pil[$i] = join(',',$fn,lc($ft),lc($fo));
			}
			${$cfgcfg{$c}{v}} = join(';',@pil);
		} elsif ($cfgcfg{$c}{t} eq 'mbs') {
			${$cfgcfg{$c}{v}} = 0 unless (${$cfgcfg{$c}{v}});
			${$cfgcfg{$c}{v}} = 0 if (${$cfgcfg{$c}{v}} =~ /^\s*(false|no|n|off|0+)\s*$/);
			${$cfgcfg{$c}{v}} = 1 if (${$cfgcfg{$c}{v}} =~ /^\s*(true|yes|y|on|\d*[1-9]\d*)\s*$/);
		} else {
			${$cfgcfg{$c}{v}} = '' unless (${$cfgcfg{$c}{v}});
		}
	}
	if ($database_spec =~ /sqlite/i) {
		$sqldbd = 'L';
	} elsif ($database_spec =~ /mysql/i) {
		$sqldbd = 'M';
	}
}

# Strip strings
sub address_strip_nc($) {
	my ($a) = @_;
	$a = '' unless (defined($a));
	$a =~ s/[\r\n]+//gs;
	$a =~ s/^[<\[]//;
	$a =~ s/[>\]]$//;
	#$a =~ s/^(.*\@)?\s*(\S+)(\s.*)?$/$1$2/;
	return $a;
}
sub address_strip($) {
	my ($a) = @_;
	return lc(address_strip_nc($a));
}

# Checks against a small internal list
sub check_ip_in_list($$) {
	my $ip = address_strip(shift);
	return 0 unless ($ip && $ip =~ /^\d+\.\d+\.\d+\.\d+$/);
	my $addr = inet_aton($ip);
	return 0 unless ($addr);
	foreach my $lst (@_) {
		next unless ($lst);
		foreach my $net (split(/;/,$lst)) {
			$net =~ s/\s+//g;
			next unless ($net);
			my ($na_s,$nm_s) = split(/\//,$net);
			$nm_s = '255.255.255.255' unless ($nm_s);
			my $na = inet_aton($na_s);
			my $nm = inet_aton($nm_s);
			next unless ($na && $nm);
			return 1 if (($addr & $nm) eq ($na & $nm));
		}
	}
	return 0;
}

# Checks against a small internal IP address white list
sub check_internal_whitelist($) {
	my($ip) = @_;
	return check_ip_in_list($ip,"127.0.0.1/255.255.255.255;$WhiteNets");
}

# Checks if IP is a black net
sub check_black_nets($) {
	my($ip) = @_;
	return check_ip_in_list($ip,'10.0.0.0/255.0.0.0;172.16.0.0/255.240.0.0;192.168.0.0/255.255.0.0;127.0.0.0/255.255.255.0');
}


#***********************************************************************
# SQL.
#***********************************************************************

sub sql_translate {
	my ($cmd) = @_;
	if ($sqldbd eq 'M') {
		$cmd =~ s/^INSERT OR IGNORE /INSERT IGNORE /;
	}
	return $cmd;
}

sub sql_do_commands {
	for (my $i=0;$i<@_;$i++) {
		my $cmd = sql_translate($_[$i]);
		#debug_log(0,'sql_do_commands: %s ',$cmd);
		return 0 unless ($cmd && defined($sqldb->do($cmd)));
	}
	return 1;
}

sub sql_exec_commands {
	for (my $i=0;$i<@_;$i++) {
		my @pars = @{$_[$i]};
		my $cmd = sql_translate(shift @pars);
		return 0 unless ($cmd);
		my $st = $sqldb->prepare_cached($cmd);
		return 0 unless ($st);
		#debug_log(7,'sql_exec_commands: %s : %s',$cmd,join(' | ',@pars));
		my $res = $st->execute(@pars);
		$st->finish;
		return 0 unless ($res);
	}
	return 1;
}

sub sql_disconnect {
	$sqldb->disconnect() if ($sqldb);
	$sqldb = undef;
}

sub sql_connect {
	my $f = shift;
	sql_disconnect() if ($f);
	if ($sqldb) {
		return 1 if ($sqldbd eq 'L');
		sql_disconnect() unless (eval { $sqldb->ping });
		return 1 if ($sqldb);
		debug_log(0,'SQL Connect Old (%s)',$sqldbd);
	}
	$sqldb = DBI->connect($database_spec,$database_user,$database_pass,{RaiseError=>0}); # (AutoCommit=>$sqldbd ne 'M') ??
	unless ($sqldb) {
		debug_log(-1,'SQL Connect Failed (%s) %s',$sqldbd,$DBI::errstr);
		return 0;
	}
	if ($sqldbd eq 'L') {
		sql_do_commands('PRAGMA SYNCHRONOUS=OFF');
	}
	#debug_log(0,'SQL Connect Ok (%s)',$sqldbd);
	return 1;
}

sub sql_quote {
	my $s = shift;
	sql_connect() unless ($sqldb);
	return $s unless ($sqldb);
	return $sqldb->quote($s);
}
sub sql_quotes {
	my @r = ();
	while (my $s = shift @_) {
		push @r, sql_quote($s);
	}
	return @r;
}

sub sql_do {
	return 0 unless (sql_connect());
	my $ok = sql_do_commands(@_);
	return $ok if ($sqldb->{AutoCommit});
	if ($ok) {
		#debug_log(0,'sql_do: commit');
		return $sqldb->commit;
	}
	debug_log(-1,'sql_do: rollback');
	$sqldb->rollback;
	return 0;
}

sub sql_execute_multi {
	return 0 unless (sql_connect());
	my $ok = sql_exec_commands(@_);
	return $ok if ($sqldb->{AutoCommit});
	return $sqldb->commit if ($ok);
	debug_log(-1,'sql_exec: rollback');
	$sqldb->rollback;
	return 0;
}
sub sql_execute {
	return sql_execute_multi(\@_);
}

sub sql_select {
	my $cmd = shift;
	return undef unless (sql_connect());
	return undef unless ($cmd);
	my $st;
	if (@_) {
		#debug_log(7,'sql_select: %s : %s',$cmd,join(' | ',@_));
		$st = $sqldb->prepare_cached($cmd);
		$st->execute(@_) if ($st);		
	} else {
		#debug_log(7,'sql_select: %s',$cmd);
		$st = $sqldb->prepare_cached("$cmd");
		$st->execute if ($st);
	}
	debug_log(-1,'sql prepare error: %s',$cmd) unless ($st);
	return $st;
}

sub sql_select_one_row {
	my $st = sql_select(@_);
	return undef unless ($st);
	my @res = $st->fetchrow_array;
	$st->finish;
	return \@res;
}

sub sql_select_one {
	my $res = sql_select_one_row(@_);
	return undef unless ($res && @{$res});
	return $res->[0];
}


#***********************************************************************
# Application.
#***********************************************************************

my $tim;
my $mid;
my $clnd = time();
my %mids = ();
my %rads = ();
my %rrls = ();

sub nopar {
	my $str = join(' ',@_);
	$str =~ s/\(.*?\)//;
	$str =~ s/^\s+//;
	$str =~ s/\s+$//;
	$str =~ s/\s+/ /g;
	return $str;
}

sub clean_data {
	return unless (time()-$clnd > 30*60);
	$clnd = time();
	my @dl = ();
	while (my ($mid, $mit) = each %mids) {
		push @dl, $mid if ($clnd-$mit > 2*60*60);
	}
	delete $mids{@dl} if (@dl);
	@dl = ();
	while (my ($hst, $inf) = each %rads) {
		push @dl, $hst if ($clnd-$inf->{t} > 60*60);
	}
	delete $rads{@dl} if (@dl);
	@dl = ();
	while (my ($rly, $rrt) = each %rrls) {
		push @dl, $rly if ($clnd-$rrt > 10*60);
	}
	delete $rrls{@dl} if (@dl);
}

sub dump_data {
	return unless ($debug);
	return unless (open(LF,'>>','/tmp/mdlogh.dat'));
	while (my ($mid, $mit) = each %mids) {
		print LF "M:$mid\n";
	}
	while (my ($hst, $inf) = each %rads) {
		print LF "$hst=$inf->{h}\n";
	}
	while (my ($rly, $rrt) = each %rrls) {
		print LF "R:$rly\n";
	}
	close(LF);
}

sub timnow {
	my $now = $tim ? str2time($tim) : 0;
	return $now ? $now : time();
}

sub xaddr {
	my ($xxx) = @_;
	return $rads{$xxx}->{h} if ($rads{$xxx});
	return 0;
}
sub haddr {
	my ($host) = @_;
	if ($host =~ /^\s*\[(\d+\.\d+\.\d+\.\d+)\]/) { return $1; }
	return xaddr("h:$host");
}
sub maddr {
	my ($mid) = @_;
	return xaddr("m:$mid");
}

sub dict_add {
	my $why = shift;
	my $addr = shift;
	return unless ($addr);
	return if (check_internal_whitelist($addr) || check_black_nets($addr));
	my $time = timnow();
	return if ($why eq 'noop' && $rrls{$addr} && $time-$rrls{$addr}<10*60);
	$mids{$mid} = $time;
	$rrls{$addr} = $time unless ($why eq 'noop');
	sql_execute('INSERT INTO dictionary (dc_stamp,dc_host) VALUES (?,?)',$time,$addr) unless ($debug);
	my @el = ($addr);
	while (@_) {
		my $e = shift @_;
		next if ($e =~ /^\s*$/);
		$e =~ s/^\s+//;
		$e =~ s/\s+$//;
		next if ($e =~ /^\[?$addr\]?$/);
		push @el, $e;
	}
	log_msg('%s: dc_%s %s %s',$mid?$mid:'?',$why,time2str('%Y-%m-%d %H:%M:%S',$time),join(' ',@el));
}

sub sm_rargs {
	my @al = ();
	foreach my $l (@_) {
		while ($l =~ /^.*?(\S+)=(\S.*?), (.*)$/) {
			my $fn = $1;
			my $fv = $2;
			$l = $3;
			push @al, $fv if ($fn =~ /^arg\d+$/);
		}
	}
	return @al;
}

sub handle_sendmail {
	$mid = shift;
	my $log = shift;
	if ($log =~ /\srelay=(\S+)\s+\[(\d+\.\d+\.\d+\.\d+)\](?:,|$)/) {
		my ($addr,$host) = ($2,$1);
		return if (check_internal_whitelist($addr) || check_black_nets($addr));
		my $inf = {t=>timnow(),h=>$addr};
		$rads{"h:$host"} = $inf;
		$rads{"m:$mid"} = $inf if ($mid && $mid !~ /^.$/);
	}
	return if ($mid && $mids{$mid});
	my $did = 1;
	if ($log =~ /^lost (\S+) channel from (\S+\s+)?\[(\d+\.\d+\.\d+\.\d+)\]/) {
		return dict_add('lost_conn',$3,$2,$1) if ($debug || $dc_lost_conn);
	} elsif ($log =~ /^(\S+): (.*?): Connection reset by (\S+)/) {
		return dict_add('lost_conn',haddr($3),$3,$1,$2) if ($debug || $dc_lost_conn);
	} elsif ($log =~ /^timeout waiting for input from (\S+) during (.*)$/) {
		return dict_add('timeout',haddr($1),$1,$2) if ($debug || $dc_timeout);
	} elsif ($log =~ /^(\S+\s+)?\[(\d+\.\d+\.\d+\.\d+)\]\s+(?:\([^\)]*\)\s+)?did not issue MAIL\/EXPN\/VRFY\/ETRN/) {
		return dict_add('noop',$2,$1) if ($debug || $dc_noop);
	} elsif ($log =~ /^(\S+)\s+(?:\([^\)]*\)\s+)?did not issue MAIL\/EXPN\/VRFY\/ETRN/) {
		return dict_add('noop',haddr($1),$1) if ($debug || $dc_noop);
	} elsif ($log =~ /^(\S+\s+)?\[(\d+\.\d+\.\d+\.\d+)\]:\s+.*?, throttling.$/) {
		return dict_add('abuse',$2,$1,'throttle') if ($debug || $dc_abuse);
	} elsif ($log =~ /^rejecting commands from (\S+\s+)?\[(\d+\.\d+\.\d+\.\d+)\]\s+.*pre-greeting/) {
		return dict_add('abuse',$2,$1,'greeting') if ($debug || $dc_abuse);
	} elsif ($log =~ /^POSSIBLE ATTACK\s(?:.+?\s)?from\s(\S+\s+)?\[(\d+\.\d+\.\d+\.\d+)\]:/) {
		return dict_add('abuse',$2,$1,'attack') if ($debug || $dc_abuse);
	} elsif ($log =~ /^POSSIBLE ATTACK\s(?:.+?\s)?from\s(\S+):/) {
		return dict_add('abuse',haddr($1),$1,'attack') if ($debug || $dc_abuse);
	} elsif ($log =~ /^(\S+):\s+(unexpected close|\S+ timeout) .*? from (\S+\s+)?\[(\d+\.\d+\.\d+\.\d+)\]/) {
		return dict_add('lost_conn',$4,$3,$1,$2) if ($debug || $dc_lost_conn);
	} elsif ($log =~ /^(\S+):\s+(unexpected close|\S+ timeout) .*? from (\S+),/) {
		return dict_add('lost_conn',haddr($3),$3,$1,$2) if ($debug || $dc_lost_conn);
	} elsif ($log =~ /^(\S+): premature EOM: Operation (.*?) with (\S+)$/) {
		return dict_add('lost_conn',haddr($3),$3,$1,$2) if ($debug || $dc_lost_conn);
	} elsif ($log =~ /(\S+)\.\.\. (User unknown|Invalid.*address|Unbalanced .?[<>].?)$/) {
		return dict_add('unknown_user',maddr($mid),$mid,$1,lc($2)) if ($debug || $dc_unknown_user);
	} elsif ($log =~ /^(.*?)\s+\(possible attack\)$/) {
		return dict_add('abuse',maddr($mid),$mid,nopar($1)) if ($debug || $dc_abuse);
	} elsif ($log =~ /^ruleset=(\S+),.*?\srelay=(\S+\s+)?\[(\d+\.\d+\.\d+\.\d+)\]/) {
		my ($rn,$hn,$ia) = ($1,$2,$3);
		if ($rn eq 'check_mail') {
			return dict_add('bad_sender',$ia,$hn,sm_rargs($log)) if ($debug || $dc_bad_sender);
		} elsif ($rn eq 'check_rcpt') {
			if ($log =~ /\sRelaying denied$/) {
				return dict_add('abuse',$ia,$hn,sm_rargs($log)) if ($debug || $dc_abuse);
			} else {
				return dict_add('unknown_user',$ia,$hn,sm_rargs($log)) if ($debug || $dc_unknown_user);
			}
		} elsif ($rn eq 'check_relay') {
			if ($log =~ /, reject=.*(limit exceeded|Too many)/) {
				return dict_add('abuse',$ia,$hn,sm_rargs($log),'limit') if ($debug || $dc_abuse);
			} else {
				$did = 0;
			}
		} else {
			$did = 0;
		}
	} elsif ($log =~ /^SYSERR\S+: (\S+): (.*?) on connection from (.*)$/) {
		my ($dw,$es,$aa) = ($1,$2,$3);
		my ($hn,$ia);
		if ($aa =~ /^(\S+)\s+\[(\d+\.\d+\.\d+\.\d+)\]/) {
			$hn = $1;
			$ia = $2;
		} elsif ($aa =~ /^\[(\d+\.\d+\.\d+\.\d+)\]/) {
			$ia = $1;
		} elsif ($aa =~ /^(\S+),/) {
			$hn = $1;
			$ia = haddr($hn);
		}
		if ($ia) {
			if ($es =~ /^(.*) timeout$/) {
				return dict_add('timeout',$ia,$hn,$dw,$1) if ($debug || $dc_timeout);
			} elsif ($es eq 'I/O error') {
				return dict_add('lost_conn',$ia,$hn,$dw,$es) if ($debug || $dc_lost_conn);
			} else {
				$did = 0;
			}
		} else {
			$did = 0;
		}
	} else {
		$did = 0;
	}
	logfile("sm $mid $log") unless ($did || $log =~ /^(?:Milter[:\s]|to=|from=|discarded|[A-Za-z0-9]{14,14}: DSN:|collect: premature EOM: unexpected close)/);
}

sub handle_mdstats {
	$mid = shift;
	my $what = shift;
	my $log = shift;
	if ($what eq 'info' && $logssv->parse($log)) {
		my @log = $logssv->fields();
		shift @log;
		if (@log && $log[0] =~ /^[\[<]?(\d+\.\d+\.\d+\.\d+)[\]>]?$/) {
			my $inf = {t=>timnow(),h=>$log[0]};
			$rads{"m:$mid"} = $inf if ($mid && $mid !~ /^.$/);
			$rads{"h:$log[1]"} = $inf if ($#log && $log[1]);
		}
	} elsif (($what eq 'tempfail' || $what eq 'reject') && $logssv->parse($log)) {
		my @log = $logssv->fields();
		shift @log; shift @log;
		if (@log && $log[0] =~ /^[\[<]?(\d+\.\d+\.\d+\.\d+)[\]>]?$/) {
			my $addr = $1;
			$rads{"m:$mid"} = {t=>timnow(),h=>$addr} if ($mid && $mid !~ /^.$/);
			$rrls{$addr} = timnow() unless (check_internal_whitelist($addr) || check_black_nets($addr));
			return;
		}
	} elsif ($what eq 'relay_info' && $logssv->parse($log)) {
		my @log = $logssv->fields();
		return if ($#log<1 || $log[1] =~ /^\[$log[0]\]$/);
		return if (check_internal_whitelist($log[0]) || check_black_nets($log[0]));
		my $inf = {t=>timnow(),h=>$log[0]};
		$rads{"h:$log[1]"} = $inf;
		$rads{"m:$mid"} = $inf if ($mid && $mid !~ /^.$/);
		return;
	}
	logfile("md $what $mid $log") unless ($what =~ /^(?:modified|quarantine|greylist|deliver|.*info|time|size|.*_.*|stream|discard)$/);
}

my $start = time();
while (my $l = <STDIN>) {
	$l =~ s/[\s\r\n]+$//s;
	$tim = '';
	if ($l =~ /^([A-Z][a-z][a-z]\s+\d+\s+\d+:\d+:\d+)\s+(.*)$/) {
		$tim = $1;
		$l = $2;
	}
	$l =~ s/^\S+\s+//;
	next if ($l =~ /^mdlogh\s/);
	if ($l =~ /^mimedefang.pl(?:\[\S+\])?:\s+(\S+): mdstats;([^;]+);(.*)$/) {
		handle_mdstats($1,$2,$3);
	} elsif ($l =~ /^(?:sendmail|sm-\S+)(?:\[\S+\])?:\s+([A-Za-z0-9]{14,14}):\s+(.*?)$/) {
		handle_sendmail($1,$2);
	} elsif ($l =~ /^(?:sendmail|sm-\S+)(?:\[\S+\])?:\s+(.*?)$/) {
		handle_sendmail('',$1);
	}
	clean_data();
}
sql_disconnect();
dump_data();