#!/usr/local/bin/perl
# 2006-06-01 15:17
use strict;
use IPC::Signal qw(sig_num);
use Date::Parse;
use Date::Format;
use Fcntl;
use File::Copy;
use Getopt::Mixed "nextOption";
use Sys::Syslog;
#use GD::Graph::linespoints;
#use GD::Graph::bars;
use GD::Graph::hbars;
#use GD::Graph::area;
#use GD::Graph::pie;
use DBI;
use NetAddr::IP;
use File::Copy;
my $udplogpid = '/var/run/p0fudplog.pid';
my $del = 1;
my $printsql = 0;
my %hosts = (
'*' => 'total',
'10.0.7.254' => 'Monty',
'10.0.37.254' => 'Satchel',
'10.0.42.254' => 'Fylax'
);
my @ignoreip = (
'10.0.0.0/8',
'195.67.112.192/27',
);
my $logpath = "/home/log/p0f";
my $graphpath = "/home/www/p0f";
my $dbdef = "dbi:mysql:p0f";
my $dbuser = 'p0f';
my $dbpass = 'pwd';
openlog('p0f.graph','pid',(19 << 3));
sub YesNo($) {
my ($b) = @_;
return 'Yes' if ($b);
return 'No';
}
my @ignoreal = ();
foreach my $iip (@ignoreip) {
next unless ($iip);
my $ni = new NetAddr::IP($iip);
next unless ($ni);
push @ignoreal, $ni->re;
}
sub checkignore {
my $ip = shift;
foreach my $ni (@ignoreal) {
return 1 if ($ip =~ /^$ni$/);
}
return 0;
}
my $quiet = 0;
my $nologs = 0;
my $nographs = 0;
my $donothing = 0;
my %onlyhosts = ();
my $verbose = 0;
my $create = 0;
my $createusr = '';
my $createpwd = '';
my $maxfiles = '';
my $timethis = 0;
sub ShowSettings {
print "Log path: $logpath\n".
"Graph path: $graphpath\n".
"Database: $dbdef\n".
"Delete logs and graphs: ".YesNo($del)."\n".
"p0f.udplog PID file: $udplogpid\n".
"Create tables: ".YesNo($create)."\n".
"Max files: $maxfiles\n".
"Don't read logs: ".YesNo($nologs)."\n".
"Don't create graphs: ".YesNo($nographs)."\n".
"Don't do anything: ".YesNo($donothing)."\n";
my $hc = 0;
foreach my $hi (sort keys %hosts) {
if ($hc) {
print " ";
} else {
print "Host names: ";
$hc = 1;
}
print "$hi => $hosts{$hi}\n";
}
$hc = 0;
foreach my $hi (sort keys %onlyhosts) {
if ($hc) {
print " ";
} else {
print "Only read logs for: ";
$hc = 1;
}
print "$hi\n";
}
}
Getopt::Mixed::init('q v h=s c=s no-logs no-graphs do-nothing show-settings max-files=i help quiet>q host>h verbose>v create>c');
while (my ($optn, $optv, $optp) = nextOption()) {
if ($optn eq 'h' || $optn eq 'host') {
$onlyhosts{$optv}=1;
} elsif ($optn eq 'q' || $optn eq 'quiet') {
$quiet = 1;
$verbose = 0;
} elsif ($optn eq 'no-logs') {
$nologs = 1;
} elsif ($optn eq 'no-graphs') {
$nographs = 1;
} elsif ($optn eq 'do-nothing') {
$donothing = 1;
} elsif ($optn eq 'v' || $optn eq 'verbose') {
$verbose = 1;
$quiet = 0;
} elsif ($optn eq 'show-settings') {
ShowSettings();
} elsif ($optn eq 'c' || $optn eq 'create') {
$create = 1;
$createusr = $optv;
$createpwd = $optv;
$createusr =~ s/:.*$//;
$createpwd =~ s/^.*://;
} elsif ($optn eq 'max-files') {
$maxfiles = $optv;
} elsif ($optn eq 'help') {
print "-h<ip> --host=<ip> : Only read logs from <ip>\n".
"-q --quiet : Don't show what's happening".
"--no-logs : Don't read logs\n".
"--no-graphs : Don't create graphs\n".
"--do-nothing : Don't do anything\n".
"--show-settings : Show current settings and options\n".
"-c<u:p> --create=<u:p> : Create databases\n".
"--help : Show this text\n";
}
}
Getopt::Mixed::cleanup();
ShowSettings if ($verbose);
sub show($) {
my ($m) = @_;
print "$m\n" if (!$quiet);
}
sub vshow($) {
my ($m) = @_;
print "$m\n" if ($verbose);
}
if ($donothing) {
show("Not doing anything.");
exit 0;
}
my $sqldb = undef;
my $sqlr;
sub sql_command {
foreach my $cmd (@_) {
return 0 unless ($sqldb && $cmd);
print "$cmd\n" if ($printsql);
$sqlr = $sqldb->do($cmd);
return 0 unless (defined($sqlr));
}
return 1;
}
sub sql_select {
my $cmd = shift;
print "$cmd\n" if ($printsql);
return undef unless ($sqldb && $cmd);
my $st = $sqldb->prepare("$cmd");
return undef unless ($st);
$st->execute;
my @res = ();
while (my @row = $st->fetchrow_array) {
push @res, \@row;
}
$st->finish;
return \@res;
}
sub sql_select_one_row {
my $cmd = shift;
print "$cmd\n" if ($printsql);
return undef unless ($sqldb && $cmd);
my $st = $sqldb->prepare("$cmd");
return undef unless ($st);
$st->execute;
my @res = $st->fetchrow_array;
$st->finish;
return \@res;
}
sub sql_select_one {
my $res = sql_select_one_row(@_);
return undef unless ($res && @{$res});
return $res->[0];
}
sub sql_set_intinfo {
my ($n,$v,$h);
if (@_ < 3) {
($n,$v) = @_;
$h = '*';
} else {
($h,$n,$v) = @_;
}
sql_command("REPLACE INTO intinfo (ii_host,ii_name,ii_value) VALUES ('$h','$n',$v)");
}
sub sql_get_intinfo {
my ($n,$h);
if (@_ < 2) {
($n) = @_;
$h = '*';
} else {
($h,$n) = @_;
}
my $v = sql_select_one("SELECT ii_value FROM intinfo WHERE ii_host='$h' AND ii_name='$n'");
return 0 unless ($v && $v > 0);
return $v;
}
sub create_database() {
$sqldb = DBI->connect($dbdef,$createusr,$createpwd,{AutoCommit=>1});
return 0 unless ($sqldb);
my @tl = $sqldb->tables();
unless (grep(/intinfo/,@tl)) {
syslog('info','Creating int info table');
sql_command(
'CREATE TABLE intinfo ('.
'ii_stamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, '.
'ii_host CHAR(15) NOT NULL, '.
'ii_name CHAR(15) NOT NULL, '.
'ii_value BIGINT UNSIGNED)',
'CREATE UNIQUE INDEX ii_idx_entry ON intinfo (ii_host,ii_name)',
);
sql_set_intinfo('intinfo_c',time());
}
unless (grep(/iplist/,@tl)) {
syslog('info','Creating iplist table');
sql_command(
'CREATE TABLE iplist ('.
'ip_stamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, '.
'ip_ip CHAR(15) PRIMARY KEY, '.
'ip_os VARCHAR(20) NOT NULL, '.
'ip_ver VARCHAR(40) NOT NULL DEFAULT \'\')',
);
sql_set_intinfo('iplist_c',time());
}
unless (grep(/stats/,@tl)) {
syslog('info','Creating stats table');
sql_command(
'CREATE TABLE stats ('.
'st_stamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, '.
'st_ignore BOOL NOT NULL, '.
'st_host CHAR(15) NOT NULL, '.
'st_year INTEGER NOT NULL, '.
'st_month INTEGER NOT NULL, '.
'st_ip CHAR(15) NOT NULL, '.
'st_os VARCHAR(20) NOT NULL, '.
'st_ver VARCHAR(40) NOT NULL, '.
'st_dist INTEGER UNSIGNED NOT NULL, '.
'st_link VARCHAR(20) NOT NULL, '.
'st_sign VARCHAR(40) NOT NULL, '.
'st_count INTEGER UNSIGNED NOT NULL DEFAULT 0)',
'CREATE UNIQUE INDEX st_idx_entry ON stats ('.
'st_ignore,'.
'st_host,'.
'st_year,'.
'st_month,'.
'st_ip,'.
'st_os,'.
'st_ver,'.
'st_dist,'.
'st_link,'.
'st_sign)',
);
sql_set_intinfo('stats_c',time());
}
$sqldb->disconnect();
$sqldb = undef;
return 1;
}
sub open_database() {
$sqldb = DBI->connect($dbdef,$dbuser,$dbpass,{AutoCommit=>1}) unless ($sqldb);
return 0 unless ($sqldb);
return 1;
}
sub close_database() {
if ($sqldb) {
$sqldb->disconnect();
$sqldb = undef;
return 1;
}
return 0;
}
create_database() if ($create);
die("DB?") unless (open_database());
sql_command(sprintf("INSERT IGNORE INTO intinfo (ii_name,ii_value) VALUES ('created',%u)",time()));
#unless ($nologs) {
# if ($udplogpid && sig_num('HUP') && open(PF,'<',$udplogpid")) {
# my $pid = <PF>;
# close(PF);
# $pid =~ s/[\r\n]//g;
# if ($pid =~ /^\d+$/ && $pid > 0) {
# show("Cycling p0f.udplog...");
# kill(sig_num('HUP'),$pid);
# sleep(5);
# }
# }
#}
my @files = ();
my $host = '-';
my $leIP = '';
my $leT = 0;
my $leOS = '';
my $leOSVer = '';
my $leDist = 0;
my $leLink = '';
my $leSign = '';
my $ltcnt = 0;
my $lccnt = 0;
my $nrt = 0;
my $frt = 0;
sub countthis {
return unless ($leOS);
my $ignore = 0;
$ignore = 1 if (checkignore($leIP));
my $cnrt = str2time($leT);
$nrt = $cnrt if ($cnrt && $cnrt > $nrt);
$frt = $cnrt if ($cnrt && ($frt == 0 || $cnrt < $frt));
my ($ss,$mm,$hh,$dd,$month,$year,$zone) = strptime($leT);
$year += 1900;
$month += 1;
$leIP = $sqldb->quote($leIP);
$leOSVer = $sqldb->quote($leOSVer);
$leOS = $sqldb->quote($leOS);
$leDist = 0 unless ($leDist && $leDist > 0);
$leLink = 'unknown' unless ($leLink);
$leLink = $sqldb->quote($leLink);
$leSign = '' unless ($leSign);
$leSign = $sqldb->quote($leSign);
sql_command("INSERT IGNORE INTO stats (".
"st_ignore,st_host,st_year,st_month,".
"st_ip,st_os,st_ver,st_dist,st_link,st_sign) VALUES (".
"$ignore,'$host',$year,$month,".
"$leIP,$leOS,$leOSVer,$leDist,$leLink,$leSign)"
);
sql_command("UPDATE stats SET st_count=st_count+1 WHERE ".
"st_ignore=$ignore AND st_host='$host' AND ".
"st_year=$year AND st_month=$month AND ".
"st_IP=$leIP AND st_os=$leOS AND st_ver=$leOSVer AND ".
"st_dist=$leDist AND st_link=$leLink AND st_sign=$leSign"
);
if ($cnrt) {
sql_command("INSERT IGNORE INTO intinfo (ii_host,ii_name,ii_value) VALUES ('$host','newest',$cnrt)");
sql_command("UPDATE intinfo SET ii_value=GREATEST(ii_value,$cnrt) WHERE ii_host='$host' AND ii_name='newest'");
sql_command("INSERT IGNORE INTO intinfo (ii_host,ii_name,ii_value) VALUES ('$host','oldest',$cnrt)");
sql_command("UPDATE intinfo SET ii_value=LEAST(ii_value,$cnrt) WHERE ii_host='$host' AND ii_name='oldest'");
}
$ltcnt++;
$lccnt++;
$leT = '';
$leOS = '';
$leOSVer = '';
$leDist = 0;
$leLink = '';
$leSign = '';
}
unless ($nologs) {
if (opendir(PD,$logpath)) {
while (my $f=readdir(PD)) {
push @files, $f if ($f =~ /^(\d+\.\d+\.\d+\.\d+)\.[A-Z0-9]+\.p0flog$/ &&
($onlyhosts{$1} || !%onlyhosts));
last if ($maxfiles>0 && @files>=$maxfiles);
}
closedir(PD);
}
}
if (@files && !$nologs) {
show("Reading logs...");
foreach my $cf (sort @files) {
$lccnt = 0;
open (LF,"$logpath/$cf") or die('Ischias!');
$host = $cf;
$host =~ s/^(\d+\.\d+\.\d+\.\d+).*$/$1/;
while (my $l=<LF>) {
$l =~ s/[\r\n]//g;
if ($l =~ /^\s+->.*\((.+)\)$/) {
my @ia = split(/,/,$1);
foreach my $i (@ia) {
$i =~ s/^\s+//;
$i =~ s/\s+$//;
if ($i =~ /^distance\s+(\d+)$/) {
$leDist = $1;
} elsif ($i =~ /^link:\s+(\S+)$/) {
$leLink = $1;
$leLink = 'unknown' if ($leLink =~ /^unknown/i);
}
}
} elsif ($l =~ /^\s+Signature\:\s+\[(.*)\].*$/) {
$leSign = $1;
} elsif ($l =~ /^<([^>]+)>\s+(\d+\.\d+\.\d+\.\d+)\:\d+\s+-\s+(\S+)\s*([^\(\[]*)(.*)/) {
countthis();
$leIP = $2;
$leT = $1;
$leOS = $3;
$leOSVer = $4;
$leOS = lc($leOS) if ($leOS =~ /^UNKNOWN$/i);
$leOSVer = '' if ($leOS eq 'unknown');
$leOSVer =~ s/^\s+//;
$leOSVer =~ s/\s+$//;
$leOSVer = '?' if ($leOS ne 'unknown' && $leOSVer eq '');
$leOS = '' unless ($leOS);
$leOSVer = '' unless ($leOSVer);
$leIP = '' unless ($leIP);
if ($leIP && $leOS) {
my $qIP = $sqldb->quote($leIP);
my $qOS = $sqldb->quote($leOS);
my $qOSVer = $sqldb->quote($leOSVer);
sql_command("REPLACE INTO iplist (ip_ip,ip_os,ip_ver) VALUES ($qIP,$qOS,$qOSVer)");
}
}
}
close(LF);
show("$cf: $lccnt");
unlink("$logpath/$cf") if ($del);
}
countthis();
sql_command("INSERT IGNORE INTO intinfo (ii_host,ii_name,ii_value) VALUES ('*','newest',$nrt)");
sql_command("UPDATE intinfo SET ii_value=GREATEST(ii_value,$nrt) WHERE ii_host='*' AND ii_name='newest'");
sql_command("INSERT IGNORE INTO intinfo (ii_host,ii_name,ii_value) VALUES ('*','oldest',$frt)");
sql_command("UPDATE intinfo SET ii_value=LEAST(ii_value,$frt) WHERE ii_host='*' AND ii_name='oldest'");
show("$ltcnt entries imported") if ($ltcnt);
syslog('info',"$ltcnt entries imported");
}
sub yticks($) {
my ($m) = @_;
$m = int($m+1) if ($m != int($m));
my $t = int(sprintf('%0.1g',$m / 10));
if ($t < 10 && $t > 5) {
$t = 10;
} elsif ($t < 5 && $t >2) {
$t = 5;
} elsif ($t == 0) {
$t = 1;
}
while ($m % $t > 0) {
$m++;
}
$t = $m/$t;
my $s = 1;
$s = 2 if ($t > 20);
return ($m,$t,$s);
}
sub graph_hbars {
my $hi = shift;
my $hn = shift;
my $file = shift;
my $label = shift;
my $byval = shift;
my $selv = shift;
vshow("$file:");
my @data = ();
my $res = sql_select($selv);
return '' unless ($res && @{$res});
my $cnt = 0;
my $max = 0;
my $tcc = 0;
foreach my $row (@{$res}) {
$tcc += $row->[1];
my $l = $row->[0];
$l = '-' if ($l eq '' || $l eq '0');
my $v = $row->[1];
push @{$data[0]}, $l;
push @{$data[1]}, $v;
vshow(" $l: $v");
$max = $v if ($v > $max);
$cnt ++;
}
vshow(" cnt:$cnt, max:$max");
my $title = "$label ($tcc) by $byval";
return '' unless ($cnt>0 && $max>0);
my $graph = new GD::Graph::hbars(700,49+(16*($cnt+1)));
my ($ym,$yt,$ys) = yticks($max);
$graph->set(
title => "[$hn] $title",
y_label => 'Count',
x_label => $label,
y_max_value => $ym,
y_tick_number => $yt,
y_label_skip => $ys,
'3d' => 0,
bgclr => 'white',
fgclr => 'gray',
boxclr => 'lgray',
transparent => 0,
cycle_clrs => 1,
long_ticks => 1,
marker_size => 0,
line_width => 0,
correct_width => 0,
cumulate => 0,
overwrite => 0,
dclrs => [qw(lgreen lred lblue lyellow lpurple cyan lorange dred dgreen dblue dyellow dpurple)],
);
$file =~ s{([^-_A-Za-z0-9])}{'='.sprintf('%02X',ord($1))}eg;
$graph->plot(\@data);
my $fn = "$graphpath/$hi\_-_$file.png";
if (open(IMAGE,'>',"$fn.tmp")) {
binmode(IMAGE);
print IMAGE $graph->gd->png;
close(IMAGE);
syslog('info',"Created graph $file.png");
return "$file.png" if (move("$fn.tmp",$fn));
}
return 0;
}
sub save_text {
my $fn = shift;
if (open(DF,'>',"$graphpath/$fn.tmp")) {
while (@_) {
my $l = shift @_;
$l =~ s/\n?$/\n/;
print DF $l;
}
close(DF);
move("$graphpath/$fn.tmp","$graphpath/$fn");
}
}
sub save_dates {
my $h = shift;
$h = 0 if ($h eq '@');
my $hi = shift;
my $fn = 'data_interval';
$fn = "$hi\_-_$fn" if ($hi);
if ($h) {
save_text($fn,
time2str('%Y-%m-%d %H:%M %z',sql_get_intinfo($h,'oldest')),
time2str('%Y-%m-%d %H:%M %z',sql_get_intinfo($h,'newest')),
time2str('%Y-%m-%d %H:%M %z',$timethis),
);
} else {
save_text($fn,
time2str('%Y-%m-%d %H:%M %z',sql_get_intinfo('oldest')),
time2str('%Y-%m-%d %H:%M %z',sql_get_intinfo('newest')),
time2str('%Y-%m-%d %H:%M %z',$timethis),
);
}
}
sub graphit {
my $hosti = shift;
my $hostn = shift;
my $hostw = shift;
my $filen = shift;
my $title = shift;
my $qvarn = shift;
my $order = shift;
my $qvarw = shift;
my $stats = shift;
$stats = 'cs' unless (defined($stats));
$hostw = '' unless ($hostw);
$hostw = "AND $hostw" if ($hostw);
$qvarw = '' unless ($qvarw);
$qvarw = "AND $qvarw" if ($qvarw);
my ($oc,$os);
if ($order) {
$oc = $order;
$os = $order;
} else {
$oc = "COUNT($qvarn) ASC";
$os = "SUM(st_count) ASC";
}
my $r = 0;
$r ++ if ($stats =~ /[Cc]/ && graph_hbars($hosti,$hostn,"sys_$filen",$title,'System',
"SELECT $qvarn,COUNT($qvarn) FROM stats WHERE st_ignore=0 $hostw $qvarw GROUP BY $qvarn ORDER BY $oc"));
$r ++ if ($stats =~ /[Ss]/ && graph_hbars($hosti,$hostn,"con_$filen",$title,'Connection',
"SELECT $qvarn,SUM(st_count) FROM stats WHERE st_ignore=0 $hostw $qvarw GROUP BY $qvarn ORDER BY $os"));
return $r;
}
sub graphos {
my $hosti = shift;
my $hostn = shift;
my $hostw = shift;
my $stats = shift;
$hostw = '' unless ($hostw);
my $hostww = $hostw;
$hostww = "AND $hostww" if ($hostww);
my $osl;
if ($stats =~ /^[Cc]/) {
$stats = 'C';
$osl = sql_select("SELECT st_os FROM stats WHERE st_ignore=0 $hostww GROUP BY st_os ORDER BY COUNT(st_os) DESC");
} elsif ($stats =~ /^[Ss]/) {
$stats = 'S';
$osl = sql_select("SELECT st_os FROM stats WHERE st_ignore=0 $hostww GROUP BY st_os ORDER BY SUM(st_count) DESC");
} else {
return 0;
}
my @osol = ();
my $r = 0;
foreach my $osr (@{$osl}) {
my $os = $osr->[0];
next unless ($os);
my $osi = lc($os);
next if ($osi eq 'unknown');
my $osq = $sqldb->quote($os);
my $osf = "osver_-_$osi";
my $ddd = 0;
my $rr = graphit($hosti,$hostn,$hostw,$osf,"$os System/Version",'st_ver','',"st_os=$osq",$stats);
push @osol, "$osi: $os" if ($rr);
$r += $rr;
}
if (@osol) {
if ($stats =~ /[Ss]/) {
save_text("$hosti\_-_sys_os_order",@osol);
} elsif ($stats =~ /[Cc]/) {
save_text("$hosti\_-_con_os_order",@osol);
}
}
return $r;
}
sub graphthis {
my $ip = shift;
my $hostnl = shift;
return 0 if ($ip ne '@' && length($ip) < 2);
$timethis = time();
my $r = 0;
my $hostw = '';
my ($hostn,$hosti);
if ($ip eq '@') {
$hostn = 'All';
$hosti = '@';
} else {
$hosts{$ip} = $ip unless ($hosts{$ip});
$hostn = $hosts{$ip};
$hosti = lc($hostn);
$hosti =~ s{([^-_A-Za-z0-9])}{'='.sprintf('%02X',ord($1))}eg;
$hostw = "st_host='$ip'";
}
$r += graphit($hosti,$hostn,$hostw,'os','Operating System (Vendor)','st_os');
$r += graphit($hosti,$hostn,$hostw,'dist','Distance','st_dist','st_dist DESC','st_dist!=0');
$r += graphit($hosti,$hostn,$hostw,'link','Link','st_link');
$r += graphos($hosti,$hostn,$hostw,'C');
$r += graphos($hosti,$hostn,$hostw,'S');
if ($r) {
save_dates($ip,$hosti);
push @{$hostnl}, "$hosti: $hostn" if ($hostnl);
}
return $r;
}
if (!$nographs) {
show("Creating graphs...");
if ($del && opendir(PD,$graphpath)) {
while (my $f=readdir(PD)) {
unlink("$graphpath/$f") if ($f =~ /^.+_-_.+\.png$/);
}
closedir(PD);
}
my @hostnl = ();
my $hostl = sql_select_one_row('SELECT DISTINCT st_host FROM stats');
foreach my $ip (@{$hostl}) {
graphthis($ip,\@hostnl);
}
graphthis('@',\@hostnl) if (@hostnl > 1);
save_dates();
save_text('hosts',sort { $a cmp $b } @hostnl) if (@hostnl);
syslog('info','Saved info');
}
show("Finished");
(2008-01-11)