Mercurial > repos > other > Puppet

changeset 126:8316d4e55e92 puppet-3.6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix Apache 2.4 Logwatch support Uses tweaked files from https://sourceforge.net/p/logwatch/git/ci/2ecdeb6e3b62814ad69e7371e69d2a26315fc45e/ We need to use "http-error-24.conf" so that Logwatch doesn't apply standard HTTP date formats to the error log and lose all log messages (Also includes a fix to remove a file from Puppet that we removed from disk - not committed separately because I didn't find it until I went to puppet apply this, and I couldn't be bothered rushing to work out how to do partial commits)
author IBBoard <dev@ibboard.co.uk>
date Tue, 13 Sep 2016 21:04:40 +0100
parents ca711ab45f17
children bafd91f195de
files common/logwatch/http-error common/logwatch/log-http-error.conf common/logwatch/services-http-error.conf manifests/templates.pp modules/postfix/manifests/init.pp
diffstat 5 files changed, 101 insertions(+), 143 deletions(-) [+]
line wrap: on
line diff
--- a/common/logwatch/http-error	Tue Sep 13 20:53:16 2016 +0100
+++ b/common/logwatch/http-error	Tue Sep 13 21:04:40 2016 +0100
@@ -1,17 +1,7 @@
-#!/usr/bin/perl -w
-#
-# $Id$
+#!/usr/bin/perl
 #
-# Logwatch service for http error logs
-# To be placed in 
-#	/etc/logwatch/scripts/http-error
-#
-# Processes all messages and summarizes them
-# Each message is given with a timestamp and RMS
-#
-########################################################
-##(C) 2006 by Jeremias Reith <jr@terragate.net>
-## Modified 2009 by Michael Baierl
+##########################################################################
+## Copyright (c) 2016 Logwatch
 ## Covered under the included MIT/X-Consortium License:
 ##    http://www.opensource.org/licenses/mit-license.php
 ## All modifications and contributions by other persons to
@@ -24,124 +14,103 @@
 ## contributions.  If you have made significant
 ## contributions to this script and want to claim
 ## copyright please contact logwatch-devel@lists.sourceforge.net.
-######################################################### 
+##########################################################################
 
+use diagnostics;
 use strict;
-use Logwatch ':dates';
-use Time::Local;
-use POSIX qw(strftime);
 
-my $date_format = '... %b %d %H:%M:%S %Y';
-my $filter = TimeFilter($date_format);
-my $detail = exists $ENV{'LOGWATCH_DETAIL_LEVEL'} ? $ENV{'LOGWATCH_DETAIL_LEVEL'} : 0;
+my $Detail = $ENV{'LOGWATCH_DETAIL_LEVEL'} || 0;
 
-# we do not use any Date:: package (or strptime) as they are probably not available
-my %month2num = ( Jan => 0, Feb => 1, Mar => 2, Apr => 3,
-		  May => 4, Jun => 5, Jul => 6, Aug => 7,
-		  Sep => 8, Oct => 9, Nov => 10, Dec => 11 );
+sub CustomizeErrorString {
+   my ($LogLevel, $ErrorCode, $Description) = @_;
+   # This function is only invoked when detail is set to 8 or 9.
+   # Here you would modify the Description.  Some Description strings
+   # may differ only on some printed parameters, and it is preferable
+   # to group them together.  Examples of these may be process numbers,
+   # IP addresses, port numbers, or file names.  The purpose of this
+   # function is to "collapse" these different messages into the same
+   # array entry.
 
-# array of message categories (we do not use a hash to keep the order)
-# first element: catorory name 
-# second element: matching regexp ($1 should contain the message)
-# third element: anonymous hash ref (stores message  counts)
-my @message_categories = (['Errors', qr/\[error\] (.*)$/o, {}],
-			  ['Warnings', qr/\[warn\] (.*)$/o, {}],
-			  ['Notices', qr/\[info\] (.*)$/o, {}]);
-
-# skipping categories depending on detail level
-pop(@message_categories) if $detail < 10;
-pop(@message_categories) if $detail < 5;
+   # For now, simply return the string.
+   return($Description);
+}
 
-# counting messages
-while(<>) {
-    my $line = $_;
-    # skipping messages that are not within the requested range
-    next unless $line =~ /^\[($filter)\]/o;
-    # skip PHP messages (have a separate script)
-    next if $line =~ / PHP (Warning|Fatal error|Notice):/o;
-    # skip ModSecurity messages
-    next if $line =~ / ModSecurity:/o; 
-    $1 =~ /(\w+) (\w+) (\d+) (\d+):(\d+):(\d+) (\d+)/;
-    my $time;
-
-    {
-	# timelocal is quite chatty
-	local $SIG{'__WARN__'}  = sub {};
-	$time = timelocal($6, $5, $4, $3, $month2num{$2}, $7-1900);
-    }
+my %LogMessages = ();
+my $MatchFilter = $ENV{'http_error_matchfilter'} || "";
+my $ReportFilter = $ENV{'http_error_reportfilter'} || "";
 
-    foreach my $cur_cat (@message_categories) {
-	if($line =~ /$cur_cat->[1]/) {
-	    my $msgs = $cur_cat->[2];
-	    $msgs->{$1} = {count => '0',
-			   first_occurrence => $time,
-			   sum => 0, 
-			   sqrsum => 0} unless exists $msgs->{$1};
-	    $msgs->{$1}->{'count'}++;
-	    # summing up timestamps and squares of timestamps
-	    # in order to calculate the rms
-	    # using first occurrence of message as offset in calculation to 
-	    # prevent an integer overflow
-	    $msgs->{$1}->{'sum'} += $time - $msgs->{$1}->{'first_occurrence'};
-	    $msgs->{$1}->{'sqrsum'} += ($time - $msgs->{$1}->{'first_occurrence'}) ** 2;
-	    last;
-	}
-    }
+while (defined(my $ThisLine = <STDIN>)) {
+   if (my ($LogLevel, $ErrorCode, $Description) =
+      ($ThisLine =~ /:(.*?)\].*(AH\d{5}): (.*)/) ) {
+   # $MatchFilter is a variable that is set by setting the
+   # $HTTP_Error_MatchFilter variable in the conf/services/http-error.conf
+   # file.  It is executed here, before any other matching statements.
+   eval $MatchFilter;
+   if ($@) {
+      print $@;
+      print "While processing MatchFilter:\n$MatchFilter\n";
+   }
+   # $ThisLine might have been reset (undef, or empty string) in $MatchFilter
+   next unless $ThisLine;
+
+      if (($Detail == 8) || ($Detail == 9)) {
+         $Description = CustomizeErrorString($LogLevel, $ErrorCode, $Description);
+      }
+      if (($Detail >= 1) || ($LogLevel =~ "emerg|alert|crit|error")) {
+         $LogMessages{$LogLevel}{$ErrorCode}{$Description}++;
+      }
+   }
 }
 
-
-# generating summary
-foreach my $cur_cat (@message_categories) {
-    # skipping non-requested message types
-    next unless keys %{$cur_cat->[2]};
-    my ($name, undef, $msgs) = @{$cur_cat};
-    print $name, ":\n";
-    my $last_count = 0;
-
-    # sorting messages by count
-    my @sorted_msgs = sort { $msgs->{$b}->{'count'} <=> $msgs->{$a}->{'count'} } keys %{$msgs};
-
-    foreach my $msg (@sorted_msgs) {
-	# grouping messages by number of occurrence
-	print "\n", $msgs->{$msg}->{'count'}, " times:\n" unless $last_count == $msgs->{$msg}->{'count'};   
-	my $rms = 0;
-
-
-	# printing timestamp
-        print '[';
-
-	if($msgs->{$msg}->{'count'} > 1) {
-	    # calculating rms
-	    $rms = int(sqrt(
-			   ($msgs->{$msg}->{'count'} * 
-			    $msgs->{$msg}->{'sqrsum'} - 
-			    $msgs->{$msg}->{'sum'}) / 
-			   ($msgs->{$msg}->{'count'} * 
-			    ($msgs->{$msg}->{'count'} - 1))));
-
-	    print strftime($date_format, localtime($msgs->{$msg}->{'first_occurrence'}+int($rms/2)));
-
-	    print ' +/-';
-	    
-	    # printing rms
-	    if($rms > 86400) {
-		print int($rms/86400) , ' day(s)';
-	    } elsif($rms > 3600) {
-		print int($rms/3600) , ' hour(s)';
-	    } elsif($rms > 60) {
-		print int($rms/60) , ' minute(s)';
-	    } else {
-		print $rms, ' seconds';
-	    }
-	} else {
-	    # we have got this message a single time
-	    print strftime($date_format, localtime($msgs->{$msg}->{'first_occurrence'}));
-	}
-	   
-	print '] ', $msg, "\n";	
-	$last_count = $msgs->{$msg}->{'count'};
-    }
-
-    print "\n";
+# $ReportFilter is a variable that is set by setting the
+# $HTTP_Error_ReportFilter variable in the conf/services/http-error.conf
+# file.  It is executed here, before any other printing statements.
+eval $ReportFilter;
+if ($@) {
+   print $@;
+   print "While processing ReportFilter:\n$ReportFilter\n";
 }
 
+if (keys %LogMessages) {
+   my $Count = 0;
+   foreach my $LogLevel (keys %LogMessages) {
+      printf("\nLevel %-6s", $LogLevel);
+      foreach my $ErrorCode (keys %{$LogMessages{$LogLevel}}) {
+         print "\n   Error Code: $ErrorCode" if $Detail >= 5;
+         foreach my $Description (keys %{$LogMessages{$LogLevel}{$ErrorCode}}) {
+            if ($Detail >= 9) {
+               print "\n      $Description:  ";
+               print "$LogMessages{$LogLevel}{$ErrorCode}{$Description} Time(s)";
+            }
+            $Count += $LogMessages{$LogLevel}{$ErrorCode}{$Description};
+         } # foreach $Description
+         if (($Detail >= 5) && ($Detail < 9)) { 
+            printf(":  %5d Time(s)", $Count);
+            $Count = 0;
+            if ($Detail >=6) {
+               print "\n      E.g.: ";
+               # print only first entry (index 0)
+               my $EG_string = (keys %{$LogMessages{$LogLevel}{$ErrorCode}})[0];
+               if (($Detail == 6) && (length($EG_string) > 66)) {
+                  printf ("%.62s ...", $EG_string);
+               } else {
+                  print $EG_string;
+               }
+            }
+         }
+      } # foreach $ErrorCode
+      if ($Detail < 5) { 
+         printf("%s%5d%s", ":  ", $Count, " Time(s)");
+         $Count = 0;
+      }
+   } # foreach $LogLevel
+} # if keys %LogMessages
+
+exit(0);
+
+# vi: shiftwidth=3 tabstop=3 syntax=perl et
+# Local Variables:
+# mode: perl
+# perl-indent-level: 3
+# indent-tabs-mode: nil
+# End:
\ No newline at end of file
--- a/common/logwatch/log-http-error.conf	Tue Sep 13 20:53:16 2016 +0100
+++ b/common/logwatch/log-http-error.conf	Tue Sep 13 21:04:40 2016 +0100
@@ -17,6 +17,8 @@
 
 
 # Keep only the lines in the proper date range...
-*ApplyHttpErrorDate
+#*ApplyHttpErrorDate
+*ApplyStdDate = "\[%a %b %d %H:%M:%S.\d{6} %Y\]"
+*RemoveHeaders = "\[\w{3} \w{3} \d{1,2} \d\d:\d\d:\d\d(\.\d*)? \d{4}\] "
 
 # vi: shiftwidth=3 tabstop=3 et
--- a/common/logwatch/services-http-error.conf	Tue Sep 13 20:53:16 2016 +0100
+++ b/common/logwatch/services-http-error.conf	Tue Sep 13 21:04:40 2016 +0100
@@ -1,15 +1,3 @@
-#
-# Service definition for http error log
-#
-# File to be placed in
-#	/etc/logwatch/conf/services/http-error.conf
-#
+# Add Apache 2.4-compliant log support
+LogFile = http-error-24
 
-Title = http errors
-
-# Which logfile group...
-LogFile = http-error
-
-# PHP notices should be fixed
-Detail = High
-
--- a/manifests/templates.pp	Tue Sep 13 20:53:16 2016 +0100
+++ b/manifests/templates.pp	Tue Sep 13 21:04:40 2016 +0100
@@ -280,8 +280,11 @@
 	file { '/etc/logwatch/conf/logfiles/http.conf':
 		content => 'LogFile = apache/access_*.log',
 	}
+	file { '/etc/logwatch/conf/logfiles/http-error-24.conf':
+		source => 'puppet:///common/logwatch/log-http-error.conf',
+	}
 	file { '/etc/logwatch/conf/logfiles/http-error.conf':
-		source => 'puppet:///common/logwatch/log-http-error.conf',
+		ensure=> absent,
 	}
 	file { '/etc/logwatch/conf/services/http-error.conf':
 		source => 'puppet:///common/logwatch/services-http-error.conf',
--- a/modules/postfix/manifests/init.pp	Tue Sep 13 20:53:16 2016 +0100
+++ b/modules/postfix/manifests/init.pp	Tue Sep 13 21:04:40 2016 +0100
@@ -75,10 +75,6 @@
   file { '/etc/postfix/helo_whitelist':
     source => 'puppet:///private/postfix/helo_whitelist',
   }
-  #Whitelisted IPs for greylisting process
-  file { '/etc/postfix/postscreen_access.cidr':
-    source => 'puppet:///modules/postfix/postscreen_access.cidr',
-  }
   #Private whitelisted IPs for greylisting process
   file { '/etc/postfix/postscreen_access_private.cidr':
     source => 'puppet:///private/postfix/postscreen_access_private.cidr',