From 333e1554ce0e40cfa3039303d7a83b04fc09575d Mon Sep 17 00:00:00 2001 From: Sven Hoexter Date: Mon, 9 Jun 2025 13:38:14 +0200 Subject: [PATCH] New upstream version 1.1.11 --- ChangeLog | 34 ++++ pffrombyto.1 | 2 +- pflogsumm | 450 ++++++++++++++++++++++++++++++++++++++++----------- pflogsumm.1 | 55 +++++-- pftobyfrom.1 | 2 +- 5 files changed, 428 insertions(+), 115 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1cc76ce..87b2476 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,40 @@ ChangeLog for pflogsumm http://jimsun.LinxNet.com/postfix_contrib.html.] +rel-1.1.11 20250530 + + Now requires Perl v5.10.0 minimum. + + Added support for postscreen summary and detail data: + + --pscrn-stats - display postscreen summary stats + + --pscrn-detail [cnt] - emit detailed postscreen data, optionally + to the top [cnt] events. + + Improved host/domain/IP address normalization. Now normalizes IPv6 + addresses. + + Bugfix: Potential undefined variable condition in get_smh. (Derives + seconds, minutes, and hours from seconds.) + + Cosmetic: + + Fixed displayed single-digit day-of-month in report heading when + "-d " specified. (Broken in v1.1.6) + + Added day-of-week to report heading when "-d " specified. + + Now always displays "Postfix Log Summaries" heading. + + Minor code optimizations to employ defined-or ("//") and reduce + from List::Util. + + Added -x (debug) option. Debugging emitted to STDERR + + Added --unprocd option. Unprocessed log lines written to + specified filename. + rel-1.1.10 20250529 Bugfix: Messages rejected in latter SMTP processing, after they'd diff --git a/pffrombyto.1 b/pffrombyto.1 index eb0e61a..a5836e0 100644 --- a/pffrombyto.1 +++ b/pffrombyto.1 @@ -55,7 +55,7 @@ .\" ======================================================================== .\" .IX Title "PFFROMBYTO 1" -.TH PFFROMBYTO 1 2025-05-22 1.1.10 "User Contributed Perl Documentation" +.TH PFFROMBYTO 1 2025-05-22 1.1.11 "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff --git a/pflogsumm b/pflogsumm index 53c8b6b..202e1f5 100755 --- a/pflogsumm +++ b/pflogsumm @@ -6,22 +6,25 @@ eval 'exec perl -S $0 "$@"' pflogsumm - Produce Postfix MTA logfile summary -Copyright (C) 1998-2025 by James S. Seymour, Release 1.1.10 +Copyright (C) 1998-2025 by James S. Seymour, Release 1.1.11 =head1 SYNOPSIS pflogsumm -[eq] [-d ] [--detail ] [--bounce-detail ] [--colwidth ] [--deferral-detail ] [-h ] [-i|--ignore-case] [--iso-date-time] [--mailq] - [-m|--uucp-mung] [--no-no-msg-size] [--problems-first] [--rej-add-from] - [--rej-add-to] [--reject-detail ] [--smtp-detail ] - [--smtpd-stats] [--smtpd-warning-detail ] [--srs-mung] - [--syslog-name=string] [-u ] [--use-orig-to] - [--verbose-msg-detail] [--verp-mung[=] [--zero-fill] [file1 [filen]] + [-m|--uucp-mung] [--no-no-msg-size] [--problems-first] + [--pscrn-detail [cnt] [--pscrn-stats] [--rej-add-from] [--rej-add-to] + [--reject-detail ] [--smtp-detail ] [--smtpd-stats] + [--smtpd-warning-detail ] [--srs-mung] [--syslog-name=string] + [-u ] [--unprocd ] [--use-orig-to] + [--verbose-msg-detail] [--verp-mung[=] [-x] [--zero-fill] + [file1 [filen]] pflogsumm -[help|version] - If no file(s) specified, reads from stdin. Output is to stdout. + If no file(s) specified, reads from stdin. Output is to stdout. Errors + and debug to stderr. =head1 DESCRIPTION @@ -139,6 +142,18 @@ Copyright (C) 1998-2025 by James S. Seymour, Release 1.1.10 Emit "problems" reports (bounces, defers, warnings, etc.) before "normal" stats. + --pscrn-detail [cnt] + Emit postscreen detail. + + If the optional cnt is included: Limits postscreen detail + reports to the top cnt. + + --pscrn-stats + Collect and emit postscreen summary stats. + + Note: Postscreen rejects are collected and reported + in any event. + --rej-add-from For those reject reports that list IP addresses or host/domain names: append the email from address to @@ -213,6 +228,10 @@ Copyright (C) 1998-2025 by James S. Seymour, Release 1.1.10 See also: "-h" and "--*-detail" options for further report-limiting options. + --unprocd + + Emit unprocessed logfile lines to file + --use-orig-to Where "orig_to" fields are found, report that in place @@ -225,13 +244,15 @@ Copyright (C) 1998-2025 by James S. Seymour, Release 1.1.10 Note: this can result in quite long lines in the report. - --verp-mung do "VERP" generated address (?) munging. Convert - --verp-mung=2 sender addresses of the form - "list-return-NN-someuser=some.dom@host.sender.dom" + --verp-mung + --verp-mung=2 + Do "VERP" generated address (?) munging. Convert + sender addresses of the form + "list-return-NN-someuser=some.dom@host.sender.dom" to "list-return-ID-someuser=some.dom@host.sender.dom" - In other words: replace the numeric value with "ID". + In other words: replace the numeric value with "ID". By specifying the optional "=2" (second form), the munging is more "aggressive", converting the address @@ -247,6 +268,8 @@ Copyright (C) 1998-2025 by James S. Seymour, Release 1.1.10 --version Print program name and version and bail out. + -x Enable debugging to STDERR + --zero-fill "Zero-fill" certain arrays so reports come out with data in columns that that might otherwise be blank. @@ -374,11 +397,13 @@ Copyright (C) 1998-2025 by James S. Seymour, Release 1.1.10 =head1 REQUIREMENTS + Requires Perl 5.10, minimum + For certain options (e.g.: --smtpd-stats), Pflogsumm requires the Date::Calc module, which can be obtained from CPAN at http://www.perl.com. - Pflogsumm is currently written and tested under Perl 5.8.3. + Pflogsumm is currently written and tested under Perl 5.38. As of version 19990413-02, pflogsumm worked with Perl 5.003, but future compatibility is not guaranteed. @@ -404,21 +429,23 @@ Copyright (C) 1998-2025 by James S. Seymour, Release 1.1.10 =cut +require v5.10.0; use strict; use locale; use Getopt::Long; +use List::Util qw(reduce); eval { require Date::Calc }; -my $hasDateCalc = $@ ? 0 : 1; +my $haveDateCalc = $@ ? 0 : 1; my $mailqCmd = "mailq"; -my $release = "1.1.10"; +my $release = "1.1.11"; # Variables and constants used throughout pflogsumm use vars qw( $progName $usageMsg %opts - @monthNames %monthNums $thisYr $thisMon + @monthNames %monthNums $thisYr $thisMon @dowNames $isoDateTime ); @@ -437,6 +464,7 @@ use constant { %monthNums = qw( Jan 0 Feb 1 Mar 2 Apr 3 May 4 Jun 5 Jul 6 Aug 7 Sep 8 Oct 9 Nov 10 Dec 11); +@dowNames = qw("" Mon Tue Wed Thu Fri Sat Sun); ($thisMon, $thisYr) = (localtime(time()))[4,5]; $thisYr += 1900; @@ -461,7 +489,7 @@ use constant { my ( $cmd, $qid, $addr, $orig_to, $size, $relay, $status, $delay, - $dateStr, $dateStrRFC3339, + $dateStr, $dateStrRFC3339, $dow, %panics, %fatals, %warnings, %masterMsgs, %deferred, %bounced, %noMsgSize, %msgDetail, @@ -476,6 +504,7 @@ my ( %rcvdMsg, $msgsFwdd, $msgsBncd, $msgsDfrdCnt, $msgsDfrd, %msgDfrdFlgs, %connTime, %smtpdPerDay, %smtpdPerDom, $smtpdConnCnt, $smtpdTotTime, + %pscrnConnTime, %pscrnPerDay, %pscrnPerIP, $pscrnConnCnt, $pscrnTotTime, %smtpMsgs ); $dayCnt = $smtpdConnCnt = $smtpdTotTime = 0; @@ -497,17 +526,63 @@ for (0 .. 23) { $smtpdPerHr[$_] = [0,0,0]; } +# +# Postscreen +# +my @pscrnPerHr; +for (0 .. 23) { + $pscrnPerHr[$_] = [0,0,0]; +} + +my @pscrnRegexs = ( + { 'expr' => '(ALLOWLIST VETO) \[(.+)\]:(\d+)' }, + { 'expr' => '(BARE NEWLINE) from \[(.+)\]:(\d+) after .+' }, + { 'expr' => '(BDAT without valid RCPT) from \[(.+)\]:(\d+)' }, + { 'expr' => '(COMMAND COUNT LIMIT) from \[(.+)\]:(\d+) after .+' }, + { 'expr' => '(COMMAND LENGTH LIMIT) from \[(.+)\]:(\d+) after .+' }, + { 'expr' => '(COMMAND PIPELINING) from \[(.+)\]:(\d+) after .+: .+' }, + { 'expr' => '(COMMAND TIME LIMIT) from \[(.+)\]:(\d+) after .+' }, + { 'expr' => '(CONNECT) from \[(.+)\]:(\d+) to \[.+\]:\d+' }, + { 'expr' => '(ENFORCE) \[(.+)\]:(\d+), (PSC_CLIENT_ADDR_PORT.+;)' }, + { 'expr' => '(DATA without valid RCPT) from \[(.+)\]:(\d+)' }, + { 'expr' => '(DISCONNECT) \[(.+)\]:(\d+)' }, + { 'expr' => '(DROP) \[(.+)\]:(\d+), (PSC_CLIENT_ADDR_PORT.+;)' }, + { 'expr' => '(DNSBL rank \d+) for \[(.+)\]:(\d+)' }, + { 'expr' => '(FAIL) \[(.+)\]:(\d+), (PSC_CLIENT_ADDR_PORT.+;)' }, + { 'expr' => '(HANGUP) after .+ from \[(.+)\]:(\d+) in ' }, + { 'expr' => '(NON-SMTP COMMAND) from \[(.+)\]:(\d+) after .+: .+' }, + { 'expr' => '(NOQUEUE: reject: CONNECT) from \[(.+)\]:(\d+): (all server ports busy)' }, + { 'expr' => '(NOQUEUE: reject: CONNECT) from \[(.+)\]:(\d+): (too many connections)' }, + { 'expr' => '(NOQUEUE: reject: RCPT) from \[(.+)\]:(\d+): \d+ ' }, + { 'expr' => '(PASS .+) \[(.+)\]:(\d+)$' }, + { 'expr' => '(PASS .+) \[(.+)\]:(\d+)(?:, )(PSC_CLIENT_ADDR_PORT.+;)' }, + { 'expr' => '(PREGREET) .+ after .+ from \[(.+)\]:(\d+): .+' }, + { 'expr' => '(SKIP) \[(.+)\]:(\d+), (PSC_CLIENT_ADDR_PORT.+;)' }, + { 'expr' => '(reject: connect) from \[(.+)\]:(\d+): (all screening ports busy)' }, + { 'expr' => '(\b\w+LISTED) \[(.+)\]:(\d+)' }, + { 'expr' => '(\b\w+LIST VETO) \[(.+)\]:(\d+)' }, + { 'expr' => '(UNFAIL) \[(.+)\]:(\d+), (PSC_CLIENT_ADDR_PORT.+;)' }, + { 'expr' => '(UNPASS) \[(.+)\]:(\d+), (PSC_CLIENT_ADDR_PORT.+;)' }, +); +# FIXME: Not certain what to do with this one +# { 'expr' => '(\[(.+)\]:\d+: replacing command \\".+\\" with \\".+\\")' }, + +my %pscrnHits; + + ($progName = $0) =~ s/^.*\///; $usageMsg = "usage: $progName -[eq] [-d ] [--detail ] [--bounce-detail ] [--colwidth ] [--deferral-detail ] [-h ] [-i|--ignore-case] [--iso-date-time] [--mailq] - [-m|--uucp-mung] [--no-no-msg-size] [--problems-first] [--rej-add-from] - [--rej-add-to] [--reject-detail ] [--smtp-detail ] - [--smtpd-stats] [--smtpd-warning-detail ] [--srs-mung] - [--syslog-name=string] [-u ] [--use-orig-to] [--verbose-msg-detail] - [--verp-mung[=]] [--zero-fill] [file1 [filen]] + [-m|--uucp-mung] [--no-no-msg-size] [--problems-first] + [--pscrn-detail [cnt] [--pscrn-stats] [--rej-add-from] [--rej-add-to] + [--reject-detail ] [--smtp-detail ] [--smtpd-stats] + [--smtpd-warning-detail ] [--srs-mung] [--syslog-name=string] + [-u ] [--unprocd ] [--use-orig-to] + [--verbose-msg-detail] [--verp-mung[=]] [-x] [--zero-fill] + [file1 [filen]] $progName --[version|help]"; @@ -515,6 +590,7 @@ $usageMsg = $isoDateTime = 0; # Don't use ISO date/time formats GetOptions( "bounce-detail=i" => \$opts{'bounceDetail'}, + "colwidth=i" => \$opts{'colWidth'}, "d=s" => \$opts{'d'}, "deferral-detail=i" => \$opts{'deferralDetail'}, "detail=i" => \$opts{'detail'}, @@ -528,22 +604,25 @@ GetOptions( "m" => \$opts{'m'}, "no-no-msg-size" => \$opts{'noNoMsgSize'}, "problems-first" => \$opts{'pf'}, + "pscrn-detail:i" => \$opts{'pscrnDetail'}, + "pscrn-stats" => \$opts{'pscrnStats'}, "q" => \$opts{'q'}, "rej-add-from" => \$opts{'rejAddFrom'}, "rej-add-to" => \$opts{'rejAddTo'}, "reject-detail=i" => \$opts{'rejectDetail'}, - "colwidth=i" => \$opts{'colWidth'}, "smtp-detail=i" => \$opts{'smtpDetail'}, "smtpd-stats" => \$opts{'smtpdStats'}, "smtpd-warning-detail=i" => \$opts{'smtpdWarnDetail'}, "srs-mung" => \$opts{'srsMung'}, "syslog-name=s" => \$opts{'syslogName'}, "u=i" => \$opts{'u'}, + "unprocd=s" => \$opts{'unProcdFN'}, "use-orig-to" => \$opts{'useOrigTo'}, "uucp-mung" => \$opts{'m'}, "verbose-msg-detail" => \$opts{'verbMsgDetail'}, "verp-mung:i" => \$opts{'verpMung'}, "version" => \$opts{'version'}, + "x" => \$opts{'debug'}, "zero-fill" => \$opts{'zeroFill'} ) || die "$usageMsg\n"; @@ -557,13 +636,22 @@ $opts{'smtpdWarnDetail'} = -1 unless(defined($opts{'smtpdWarnDetail'})); $opts{'rejectDetail'} = -1 unless(defined($opts{'rejectDetail'})); $opts{'colWidth'} = 0 if($opts{'verbMsgDetail'}); $opts{'colWidth'} = -1 unless(defined($opts{'colWidth'})); +# This one's a bit tricky because it works differently +$opts{'pscrnDetail'} = defined($opts{'pscrnDetail'})? ($opts{'pscrnDetail'} == 0? -1 : $opts{'pscrnDetail'}) : 0; # If --detail was specified, set anything that's not enumerated to it if(defined($opts{'detail'})) { - foreach my $optName (qw (h u bounceDetail deferralDetail smtpDetail smtpdWarnDetail rejectDetail)) { + foreach my $optName (qw (h u bounceDetail deferralDetail smtpDetail smtpdWarnDetail rejectDetail pscrnDetail)) { $opts{$optName} = $opts{'detail'} unless($opts{"$optName"} != -1); } } +if(defined $opts{'debug'}) { + if(defined $opts{'pscrnDetail'}) { + print STDERR "\$opts{'pscrnDetail'}: $opts{'pscrnDetail'}\n"; + } else { + print STDERR "\$opts{'pscrnDetail'}: undef\n"; + } +} my $syslogName = $opts{'syslogName'}? $opts{'syslogName'} : "postfix"; @@ -577,39 +665,42 @@ if(defined($opts{'version'})) { exit 0; } -if($hasDateCalc) { +if($haveDateCalc) { # manually import the Date::Calc routine we want # # This looks stupid, but it's the only way to shut Perl up about - # "Date::Calc::Delta_DHMS" used only once" if -w is on. (No, + # "Date::Calc::" used only once" if -w is on. (No, # $^W = 0 doesn't work in this context.) *Delta_DHMS = *Date::Calc::Delta_DHMS; *Delta_DHMS = *Date::Calc::Delta_DHMS; + *Day_of_Week = *Date::Calc::Day_of_Week; + *Day_of_Week = *Date::Calc::Day_of_Week; -} elsif(defined($opts{'smtpdStats'})) { - # If user specified --smtpd-stats but doesn't have Date::Calc - # installed, die with friendly help message. +} elsif(defined($opts{'smtpdStats'}) || defined($opts{'pscrnStats'})) { + # If user specified --smtpd-stats or --pscrn-stats but doesn't + # have Date::Calc installed, die with friendly help message. die < $unProcdFN") || -# die "couldn't open \"$unProcdFN\": $!\n"; +if($opts{'unProcdFN'}) { + open($unProcd, "> $opts{'unProcdFN'}") || + die "couldn't open \"$opts{'unProcdFN'}\": $!\n"; +} while(<>) { next if(defined($dateStr) && ! (/^${dateStr} / || /^${dateStrRFC3339}T/)); @@ -633,7 +724,7 @@ while(<>) { unless((($cmd, $qid) = $logRmdr =~ m#^(?:postfix|$syslogName)(?:/(?:smtps|submission))?/([^\[:]*).*?: ([^:\s]+)#o) == 2 || (($cmd, $qid) = $logRmdr =~ m#^((?:postfix)(?:-script)?)(?:\[\d+\])?: ([^:\s]+)#o) == 2) { - print $unProcd "$_" if $unProcd; + print $unProcd "[01]: $_" if $unProcd; next; } chomp; @@ -758,44 +849,130 @@ while(<>) { } } else { - next unless(defined($opts{'smtpdStats'})); - if($logRmdr =~ /: connect from /) { - $logRmdr =~ /\/smtpd\[(\d+)\]: /; - @{$connTime{$1}} = - ($msgYr, $msgMon + 1, $msgDay, $msgHr, $msgMin, $msgSec); - } elsif($logRmdr =~ /: disconnect from /) { - my ($pid, $hostID) = $logRmdr =~ /\/smtpd\[(\d+)\]: disconnect from (.+?)( unknown=\d+\/\d+)?( commands=\d+\/\d+)?$/; - if(exists($connTime{$pid}) && ($hostID = gimme_domain($hostID))) { - my($d, $h, $m, $s) = Delta_DHMS(@{$connTime{$pid}}, - $msgYr, $msgMon + 1, $msgDay, $msgHr, $msgMin, $msgSec); - delete($connTime{$pid}); # dispose of no-longer-needed item - my $tSecs = (86400 * $d) + (3600 * $h) + (60 * $m) + $s; - - ++$smtpdPerHr[$msgHr][0]; - $smtpdPerHr[$msgHr][1] += $tSecs; - $smtpdPerHr[$msgHr][2] = $tSecs if($tSecs > $smtpdPerHr[$msgHr][2]); - - unless(${$smtpdPerDay{$revMsgDateStr}}[0]++) { - ${$smtpdPerDay{$revMsgDateStr}}[1] = 0; - ${$smtpdPerDay{$revMsgDateStr}}[2] = 0; - } + if($cmd eq 'smtpd') { + next unless(defined($opts{'smtpdStats'})); + if($logRmdr =~ /: connect from /) { + $logRmdr =~ /\/smtpd\[(\d+)\]: /; + @{$connTime{$1}} = + ($msgYr, $msgMon + 1, $msgDay, $msgHr, $msgMin, $msgSec); + } elsif($logRmdr =~ /: disconnect from /) { + my ($pid, $hostID) = $logRmdr =~ /\/smtpd\[(\d+)\]: disconnect from (.+?)( unknown=\d+\/\d+)?( commands=\d+\/\d+)?$/; + if(exists($connTime{$pid}) && ($hostID = gimme_domain($hostID))) { + my($d, $h, $m, $s) = Delta_DHMS(@{$connTime{$pid}}, + $msgYr, $msgMon + 1, $msgDay, $msgHr, $msgMin, $msgSec); + delete($connTime{$pid}); # dispose of no-longer-needed item + my $tSecs = (86400 * $d) + (3600 * $h) + (60 * $m) + $s; + + ++$smtpdPerHr[$msgHr][0]; + $smtpdPerHr[$msgHr][1] += $tSecs; + $smtpdPerHr[$msgHr][2] = $tSecs if($tSecs > $smtpdPerHr[$msgHr][2]); + + unless(${$smtpdPerDay{$revMsgDateStr}}[0]++) { + ${$smtpdPerDay{$revMsgDateStr}}[1] = 0; + ${$smtpdPerDay{$revMsgDateStr}}[2] = 0; + } - ${$smtpdPerDay{$revMsgDateStr}}[1] += $tSecs; - ${$smtpdPerDay{$revMsgDateStr}}[2] = $tSecs - if($tSecs > ${$smtpdPerDay{$revMsgDateStr}}[2]); + ${$smtpdPerDay{$revMsgDateStr}}[1] += $tSecs; + ${$smtpdPerDay{$revMsgDateStr}}[2] = $tSecs + if($tSecs > ${$smtpdPerDay{$revMsgDateStr}}[2]); + + if($hostID){ + unless(${$smtpdPerDom{$hostID}}[0]++) { + ${$smtpdPerDom{$hostID}}[1] = 0; + ${$smtpdPerDom{$hostID}}[2] = 0; + } + ${$smtpdPerDom{$hostID}}[1] += $tSecs; + ${$smtpdPerDom{$hostID}}[2] = $tSecs + if($tSecs > ${$smtpdPerDom{$hostID}}[2]); + } - if($hostID){ - unless(${$smtpdPerDom{$hostID}}[0]++) { - ${$smtpdPerDom{$hostID}}[1] = 0; - ${$smtpdPerDom{$hostID}}[2] = 0; + ++$smtpdConnCnt; + $smtpdTotTime += $tSecs; + } + } + } elsif($cmd eq 'postscreen' && (defined $opts{'pscrnStats'} || $opts{'pscrnDetail'})) { + + my ($pscrnAct, $clientIP, $clientPort, $pscrnAddl, $capCnt); + print STDERR "\n" if($opts{'debug'}); + print STDERR "\$opts{'pscrnStats'}: " . ($opts{'pscrnStats'} // 0) .", \$opts{'pscrnDetail'}: $opts{'pscrnDetail'}\n" if($opts{'debug'}); + foreach my $regEx (@pscrnRegexs) { + print STDERR "\$regEx->{'expr'}: \"$regEx->{'expr'}\"\n" if($opts{'debug'}); + if(($capCnt = (($pscrnAct, $clientIP, $clientPort, $pscrnAddl) = $logRmdr =~ /$regEx->{'expr'}/)) >= 3) { + ++$regEx->{'cnt'}; # Not (currently?) used + if($opts{'debug'}) { + foreach ($pscrnAct, $clientIP, $clientPort, $pscrnAddl) { + print STDERR "capt: \"$_\"\n" if(defined $_ ); + } } - ${$smtpdPerDom{$hostID}}[1] += $tSecs; - ${$smtpdPerDom{$hostID}}[2] = $tSecs - if($tSecs > ${$smtpdPerDom{$hostID}}[2]); + last; + } + } + + print STDERR "\$capCnt: $capCnt\n\$logRmdr: \"$logRmdr\"\n" if($opts{'debug'}); + + my $bump_capt_cnt = sub { + if($capCnt == 4) { + print STDERR "Bumping \$pscrnHits{\"$pscrnAct $pscrnAddl\"}{\"$clientIP\"} on \$logRmdr: \"$logRmdr\"\n" if($opts{'debug'}); + ++$pscrnHits{"$pscrnAct $pscrnAddl"}{$clientIP} if($opts{'pscrnDetail'}); + print STDERR "\$cmd: \"$cmd\", \$logRmdr: \"$logRmdr\"\n" if($opts{'debug'}); + } else { + print STDERR "Bumping \$pscrnHits{\"$pscrnAct\"}{\"$clientIP\"} on \$logRmdr: \"$logRmdr\"\n" if($opts{'debug'}); + ++$pscrnHits{$pscrnAct}{$clientIP} if($opts{'pscrnDetail'}); + print STDERR "\$cmd: \"$cmd\", \$logRmdr: \"$logRmdr\"\n" if($opts{'debug'}); } + }; + + if($capCnt == 3) { + if($pscrnAct eq 'CONNECT') { + @{$connTime{"$clientIP:$clientPort"}} = + ($msgYr, $msgMon + 1, $msgDay, $msgHr, $msgMin, $msgSec); + print STDERR "\@{\$connTime{\"$clientIP:$clientPort\"}}: " . join(' / ', @{$connTime{"$clientIP:$clientPort"}}) . "\n" if($opts{'debug'}); + } elsif($pscrnAct =~ /^(DISCONNECT|HANGUP|PASS (NEW|OLD))$/) { + print STDERR "DISCO: \$pscrnAct: \"$pscrnAct\", \$clientIP: \"$clientIP\", \$clientPort: \"$clientPort\"\n" if($opts{'debug'}); + + if(exists($connTime{"$clientIP:$clientPort"})) { + my($d, $h, $m, $s) = Delta_DHMS(@{$connTime{"$clientIP:$clientPort"}}, + $msgYr, $msgMon + 1, $msgDay, $msgHr, $msgMin, $msgSec); + delete($connTime{"$clientIP:$clientPort"}); # dispose of no-longer-needed item + my $tSecs = (86400 * $d) + (3600 * $h) + (60 * $m) + $s; + print STDERR "DISCONNECT: \$tSecs: $tSecs\n" if($opts{'debug'}); + + ++$pscrnPerHr[$msgHr][0]; + $pscrnPerHr[$msgHr][1] += $tSecs; + $pscrnPerHr[$msgHr][2] = $tSecs if($tSecs > $pscrnPerHr[$msgHr][2]); + + unless(${$pscrnPerDay{$revMsgDateStr}}[0]++) { + ${$pscrnPerDay{$revMsgDateStr}}[1] = 0; + ${$pscrnPerDay{$revMsgDateStr}}[2] = 0; + } + + ${$pscrnPerDay{$revMsgDateStr}}[1] += $tSecs; + ${$pscrnPerDay{$revMsgDateStr}}[2] = $tSecs + if($tSecs > ${$pscrnPerDay{$revMsgDateStr}}[2]); + + unless(${$pscrnPerIP{$clientIP}}[0]++) { + ${$pscrnPerIP{$clientIP}}[1] = 0; + ${$pscrnPerIP{$clientIP}}[2] = 0; + } + + ${$pscrnPerIP{$clientIP}}[1] += $tSecs; + ${$pscrnPerIP{$clientIP}}[2] = $tSecs + if($tSecs > ${$pscrnPerIP{$clientIP}}[2]); + + ++$pscrnConnCnt; + $pscrnTotTime += $tSecs; + + # Want the per-postscreen-action stats? + $bump_capt_cnt->() if($opts{'pscrnDetail'} != 0 && $pscrnAct =~ /^PASS (NEW|OLD)$/); - ++$smtpdConnCnt; - $smtpdTotTime += $tSecs; + } + } else { + $bump_capt_cnt->() if($opts{'pscrnDetail'}); # Want the per-postscreen-action stats? + } + } elsif($capCnt == 4) { + $bump_capt_cnt->() if($opts{'pscrnDetail'}); # Want the per-postscreen-action stats? + } else { + print $unProcd "[02]: $_\n" if($unProcd && (defined $opts{'pscrnStats'} || $opts{'pscrnDetail'} != 0)); } } } @@ -928,7 +1105,7 @@ while(<>) { ++${$msgsPerDay{$revMsgDateStr}}[3]; ++$msgsBncd; } else { - print $unProcd "$_\n" if $unProcd; + print $unProcd "[03]: $_\n" if $unProcd; } } elsif($cmd eq 'pickup' && $logRmdr =~ /: (sender|uid)=/) { @@ -947,7 +1124,7 @@ while(<>) { } elsif($logRmdr =~ /.* connect to ([^[]+)\[\S+?\]: (.+?) \(port \d+\)$/) { ++$smtpMsgs{lc($2)}{$1}; } else { - print $unProcd "$_\n" if $unProcd; + print $unProcd "[04]: $_\n" if $unProcd; } } elsif($cmd =~ /^n?qmgr$/ && $logRmdr =~ /\bremoved$/) { @@ -955,7 +1132,7 @@ while(<>) { } else { - print $unProcd "$_\n" if $unProcd; + print $unProcd "[05]: $_\n" if $unProcd; } } } @@ -973,7 +1150,7 @@ if(my $noSizeCnt = scalar grep { !exists $rcvdMsg{$_}{'size'} } keys %rcvdMsg) { # debugging if($unProcd) { close($unProcd) || - warn "problem closing \"$unProcdFN\": $!\n"; + warn "problem closing \"$opts{'unProcdFN'}\": $!\n"; } # Calculate percentage of messages rejected and discarded @@ -984,9 +1161,13 @@ if(my $msgsTotal = $msgsDlvrd + $msgsRjctd + $msgsDscrdd) { $msgsDscrddPct = int(($msgsDscrdd/$msgsTotal) * 100); } +print "Postfix Log Summaries"; if(defined($dateStr)) { - print "Postfix log summaries for $dateStr\n"; + (my $dispDate = $dateStr) =~ s/\[ 0\]// if($dateStr); + $dow .= ", " if(length($dow)); + print " for ${dow}${dispDate}"; } +print "\n"; print_subsect_title("Grand Totals"); print "messages\n\n"; @@ -1022,6 +1203,20 @@ if(defined($opts{'smtpdStats'})) { } } +if(defined($opts{'pscrnStats'})) { + print "\npostscreen\n\n"; + printf " %6d%s connections\n", adj_int_units($pscrnConnCnt); + printf " %6d%s IP addresses\n", adj_int_units(int(keys %pscrnPerIP)); + printf " %6d avg. connect time (seconds)\n", + ($pscrnConnCnt && $pscrnConnCnt > 0)? ($pscrnTotTime / $pscrnConnCnt) + .5 : 0; + { + my ($sec, $min, $hr) = get_smh($pscrnTotTime); + printf " %2d:%02d:%02d total connect time\n", + $hr, $min, $sec; + } +} + + print "\n"; print_problems_reports() if(defined($opts{'pf'})); @@ -1070,6 +1265,9 @@ sub print_problems_reports { unless($opts{'smtpdWarnDetail'} == 0) { print_nested_hash(\%warnings, "Warnings", $opts{'smtpdWarnDetail'}, $opts{'q'}); } + + print_nested_hash(\%pscrnHits, "postscreen actions", $opts{'pscrnDetail'}, $opts{'q'}) if($opts{'pscrnDetail'}); + print_nested_hash(\%fatals, "Fatal Errors", 0, $opts{'q'}); print_nested_hash(\%panics, "Panics", 0, $opts{'q'}); print_hash_by_cnt_vals(\%masterMsgs,"Master daemon messages", 0, $opts{'q'}); @@ -1104,8 +1302,7 @@ End_Of_Per_Day_Heading printf " $msgMonStr %2d $msgYr", $msgDay; } foreach $value (@{$msgsPerDay->{$_}}) { - my $value2 = $value? $value : 0; - printf " %6d%s", adj_int_units($value2); + printf " %6d%s", adj_int_units($value // 0); } print "\n"; } @@ -1414,8 +1611,7 @@ sub walk_nested_hash { keys(%{$hashRef->{$_}}); # "reset" hash iterator unless(ref($hashVal2) eq 'HASH') { my $rptLines = keys %{$hashRef->{$_}}; - my $rptCnt = 0; - $rptCnt += $_ foreach (values %{$hashRef->{$_}}); + my $rptCnt = reduce { $a + $b } 0, values %{$hashRef->{$_}}; print " ("; print "top $cnt of " if($cnt > 0 && $rptLines > $cnt); print "$rptCnt)"; @@ -1474,18 +1670,66 @@ sub print_subsect_title { } # Normalize IP addr or hostname -# (Note: Makes no effort to normalize IPv6 addrs. Just returns them -# as they're passed-in.) sub normalize_host { - # For IP addrs and hostnames: lop off possible " (user@dom.ain)" bit - my $norm1 = (split(/\s/, $_[0]))[0]; + my ($host) = @_; + # Strip off possible " (user@dom.ain)" bit + my $norm1 = (split(/\s/, $host))[0]; + + # Helper function to normalize IPv6 address + sub normalize_ipv6 { + my ($addr) = @_; + # Convert to lowercase and remove leading/trailing whitespace + $addr = lc($addr); + $addr =~ s/^\s+|\s+$//g; + + # Split into blocks + my @blocks = split /:/, $addr; + my $double_colon_index = -1; + my $block_count = @blocks; + + # Find double colon (::) if it exists + for my $i (0 .. $#blocks) { + if ($blocks[$i] eq '') { + $double_colon_index = $i; + last; + } + } + + # Handle double colon abbreviation + if ($double_colon_index >= 0) { + my $missing_blocks = 8 - $block_count + 1; # +1 for the empty block + my @zeroes = ('0000') x $missing_blocks; + splice @blocks, $double_colon_index, 1, @zeroes; + } + + # Pad each block with leading zeros to 4 digits + @blocks = map { sprintf("%04s", $_) } @blocks; + + # Ensure exactly 8 blocks + if (@blocks < 8) { + push @blocks, ('0000') x (8 - @blocks); + } elsif (@blocks > 8) { + die "Invalid IPv6 address: $addr\n"; + } + + return join '', @blocks; + } - if((my @octets = ($norm1 =~ /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/)) == 4) { - # Dotted-quad IP address - return(pack('U4', @octets)); + if ((my @octets = ($norm1 =~ /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/)) == 4) { + # Dotted-quad IPv4 address + # Validate each octet is in range 0-255 + for my $octet (@octets) { + return "0_$norm1" if $octet > 255; # Treat invalid as hostname + } + # Convert to 8-character hex (2 chars per octet) + my $hex = sprintf("%02x%02x%02x%02x", @octets); + return "2_$hex"; + } elsif ($norm1 =~ /^[\dA-Fa-f]+:/) { + # IPv6 address + return "3_" . normalize_ipv6($norm1); } else { - # Possibly hostname or user@dom.ain - return(join( '', map { lc $_ } reverse split /[.@]/, $norm1 )); + # Hostname or user@dom.ain + return "0_" . join('', map { lc $_ } reverse split /[.@]/, $norm1); } } @@ -1560,7 +1804,7 @@ sub by_count_then_size { # return traditional and RFC3339 date strings to match in log sub get_datestrs { - my ($dateOpt) = $_[0]; + my ($dateOpt, $haveDateCalc) = @_; my $time = time(); @@ -1571,9 +1815,10 @@ sub get_datestrs { die "$usageMsg\n"; } my ($t_mday, $t_mon, $t_year) = (localtime($time))[3,4,5]; + my $dow = ($dateOpt && $haveDateCalc)? $dowNames[Day_of_Week($t_year + 1900, $t_mon + 1, $t_mday)] : ""; - return @{[map {s/ (\d)$/\[ 0\]$1/; $_} sprintf("%s %2d", $monthNames[$t_mon], $t_mday)]}, - sprintf("%04d-%02d-%02d", $t_year+1900, $t_mon+1, $t_mday); + return @{[map {s/ (\d)$/[ 0]$1/; $_} sprintf("%s %2d", $monthNames[$t_mon], $t_mday)]}, + sprintf("%04d-%02d-%02d", $t_year+1900, $t_mon+1, $t_mday), $dow; } # if there's a real domain: uses that. Otherwise uses the IP addr. @@ -1680,10 +1925,16 @@ sub string_trimmer { # Get seconds, minutes and hours from seconds sub get_smh { my $sec = shift @_; - my $hr = int($sec / 3600); - $sec -= $hr * 3600; - my $min = int($sec / 60); - $sec -= $min * 60; + my ($min, $hr); + + if($sec) { + $hr = int($sec / 3600); + $sec -= $hr * 3600; + $min = int($sec / 60); + $sec -= $min * 60; + } else { + ($sec, $min, $hr) = (0, 0, 0); + } return($sec, $min, $hr); } @@ -1707,6 +1958,7 @@ sub proc_smtpd_reject { # Was an IPv6 problem here ($rejTyp, $rejFrom, $rejRmdr) = ($logLine =~ /^.* \b(?:reject(?:_warning)?|hold|discard): (\S+) from (\S+?): (.*)$/); + print STDERR "\$rejTyp: \"$rejTyp\", \$rejReas: \"$rejReas\"\n" if($opts{'debug'} && defined $rejTyp && defined $rejReas); # Next: get the reject "reason" $rejReas = $rejRmdr; @@ -1726,6 +1978,8 @@ sub proc_smtpd_reject { $rejReas =~ s/^Unverified (Client host rejected: Generic - Please relay via ISP).*$/$1/; } elsif($rejTyp eq "MAIL") { # *more* special treatment :-( grrrr... $rejReas =~ s/^\d{3} (?:<.+>: )?([^;:]+)[;:]?.*$/$1/; + } elsif($rejTyp eq "connect") { # and still *more* special treatment :-( *sigh*... + $rejTyp = 'CONNECT'; } else { $rejReas =~ s/^(?:.*[:;] )?([^,]+).*$/$1/; } @@ -1780,7 +2034,7 @@ sub proc_smtpd_reject { $rejData .= " ($from)" if($opts{'rejAddFrom'}); ++$rejects->{$rejTyp}{$rejReas}{$rejData}; } else { - #print STDERR "dbg: unknown/un-enumerated reject reason $rejReas, \$rejFrom: \"$rejFrom\"!\n\n"; + print STDERR "dbg: unknown/un-enumerated reject reason: \$rejReas: \"$rejReas\", \$rejTyp: \"$rejTyp\", \$rejFrom: \"$rejFrom\"!\n" if($opts{'debug'}); my $rejData = gimme_domain($rejFrom); if($opts{'rejAddFrom'} && $opts{'rejAddTo'} && $to) { $rejData .= " ($from -> $to)"; diff --git a/pflogsumm.1 b/pflogsumm.1 index 98a8bc8..cb0ca17 100644 --- a/pflogsumm.1 +++ b/pflogsumm.1 @@ -55,7 +55,7 @@ .\" ======================================================================== .\" .IX Title "PFLOGSUMM 1" -.TH PFLOGSUMM 1 2025-05-29 1.1.10 "User Contributed Perl Documentation" +.TH PFLOGSUMM 1 2025-06-07 1.1.11 "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l @@ -63,22 +63,25 @@ .SH NAME pflogsumm \- Produce Postfix MTA logfile summary .PP -Copyright (C) 1998\-2025 by James S. Seymour, Release 1.1.10 +Copyright (C) 1998\-2025 by James S. Seymour, Release 1.1.11 .SH SYNOPSIS .IX Header "SYNOPSIS" -.Vb 8 +.Vb 10 \& pflogsumm \-[eq] [\-d ] [\-\-detail ] \& [\-\-bounce\-detail ] [\-\-colwidth ] [\-\-deferral\-detail ] \& [\-h ] [\-i|\-\-ignore\-case] [\-\-iso\-date\-time] [\-\-mailq] -\& [\-m|\-\-uucp\-mung] [\-\-no\-no\-msg\-size] [\-\-problems\-first] [\-\-rej\-add\-from] -\& [\-\-rej\-add\-to] [\-\-reject\-detail ] [\-\-smtp\-detail ] -\& [\-\-smtpd\-stats] [\-\-smtpd\-warning\-detail ] [\-\-srs\-mung] -\& [\-\-syslog\-name=string] [\-u ] [\-\-use\-orig\-to] -\& [\-\-verbose\-msg\-detail] [\-\-verp\-mung[=] [\-\-zero\-fill] [file1 [filen]] +\& [\-m|\-\-uucp\-mung] [\-\-no\-no\-msg\-size] [\-\-problems\-first] +\& [\-\-pscrn\-detail [cnt] [\-\-pscrn\-stats] [\-\-rej\-add\-from] [\-\-rej\-add\-to] +\& [\-\-reject\-detail ] [\-\-smtp\-detail ] [\-\-smtpd\-stats] +\& [\-\-smtpd\-warning\-detail ] [\-\-srs\-mung] [\-\-syslog\-name=string] +\& [\-u ] [\-\-unprocd ] [\-\-use\-orig\-to] +\& [\-\-verbose\-msg\-detail] [\-\-verp\-mung[=] [\-x] [\-\-zero\-fill] +\& [file1 [filen]] \& \& pflogsumm \-[help|version] \& -\& If no file(s) specified, reads from stdin. Output is to stdout. +\& If no file(s) specified, reads from stdin. Output is to stdout. Errors +\& and debug to stderr. .Ve .SH DESCRIPTION .IX Header "DESCRIPTION" @@ -198,6 +201,18 @@ Copyright (C) 1998\-2025 by James S. Seymour, Release 1.1.10 \& Emit "problems" reports (bounces, defers, warnings, \& etc.) before "normal" stats. \& +\& \-\-pscrn\-detail [cnt] +\& Emit postscreen detail. +\& +\& If the optional cnt is included: Limits postscreen detail +\& reports to the top cnt. +\& +\& \-\-pscrn\-stats +\& Collect and emit postscreen summary stats. +\& +\& Note: Postscreen rejects are collected and reported +\& in any event. +\& \& \-\-rej\-add\-from \& For those reject reports that list IP addresses or \& host/domain names: append the email from address to @@ -272,6 +287,10 @@ Copyright (C) 1998\-2025 by James S. Seymour, Release 1.1.10 \& See also: "\-h" and "\-\-*\-detail" options for further \& report\-limiting options. \& +\& \-\-unprocd +\& +\& Emit unprocessed logfile lines to file +\& \& \-\-use\-orig\-to \& \& Where "orig_to" fields are found, report that in place @@ -284,13 +303,15 @@ Copyright (C) 1998\-2025 by James S. Seymour, Release 1.1.10 \& \& Note: this can result in quite long lines in the report. \& -\& \-\-verp\-mung do "VERP" generated address (?) munging. Convert -\& \-\-verp\-mung=2 sender addresses of the form -\& "list\-return\-NN\-someuser=some.dom@host.sender.dom" +\& \-\-verp\-mung +\& \-\-verp\-mung=2 +\& Do "VERP" generated address (?) munging. Convert +\& sender addresses of the form +\& "list\-return\-NN\-someuser=some.dom@host.sender.dom" \& to \& "list\-return\-ID\-someuser=some.dom@host.sender.dom" \& -\& In other words: replace the numeric value with "ID". +\& In other words: replace the numeric value with "ID". \& \& By specifying the optional "=2" (second form), the \& munging is more "aggressive", converting the address @@ -306,6 +327,8 @@ Copyright (C) 1998\-2025 by James S. Seymour, Release 1.1.10 \& \& \-\-version Print program name and version and bail out. \& +\& \-x Enable debugging to STDERR +\& \& \-\-zero\-fill "Zero\-fill" certain arrays so reports come out with \& data in columns that that might otherwise be blank. .Ve @@ -438,12 +461,14 @@ Copyright (C) 1998\-2025 by James S. Seymour, Release 1.1.10 .Ve .SH REQUIREMENTS .IX Header "REQUIREMENTS" -.Vb 3 +.Vb 1 +\& Requires Perl 5.10, minimum +\& \& For certain options (e.g.: \-\-smtpd\-stats), Pflogsumm requires the \& Date::Calc module, which can be obtained from CPAN at \& http://www.perl.com. \& -\& Pflogsumm is currently written and tested under Perl 5.8.3. +\& Pflogsumm is currently written and tested under Perl 5.38. \& As of version 19990413\-02, pflogsumm worked with Perl 5.003, but \& future compatibility is not guaranteed. .Ve diff --git a/pftobyfrom.1 b/pftobyfrom.1 index a62a123..64659ed 100644 --- a/pftobyfrom.1 +++ b/pftobyfrom.1 @@ -55,7 +55,7 @@ .\" ======================================================================== .\" .IX Title "PFTOBYFROM 1" -.TH PFTOBYFROM 1 2025-05-22 1.1.10 "User Contributed Perl Documentation" +.TH PFTOBYFROM 1 2025-05-22 1.1.11 "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l -- 2.39.5