Here is another useful administrative perl script. Note that it will
only work when patch E30 is applied, but it may be useful anyway.
Feel free to do with it what you want -- if someone wants to add it to
the Apache distribution (along with some form of E30, please add the
Apache copyright first.
.......Roy
----------------------cut here
#!/usr/public/bin/perl
# ==========================================================================
sub usage {
die <<"EndUsage";
USAGE: check_error_log [ error_log_name ]
This program reads the httpd error_log logfile (or whatever file is
given as the single argument) looking for failed requests (bad URLs)
which include a valid Referer (and thus can be tracked to the origin
of the reference). It assumes that the error_log has been enhanced
to include the referer and user-agent information.
The output is an HTML file that makes it easy to check references.
Created by Roy Fielding, 10 Apr 1995
EndUsage
}
if ($#ARGV > 0) { &usage; }
# ==========================================================================
# Get defaults
$Owner = "Roy Fielding <fielding@ics.uci.edu>";
@DoW = ('Sun','Mon','Tue','Wed','Thu','Fri','Sat');
@MoY = ('Jan','Feb','Mar','Apr','May','Jun',
'Jul','Aug','Sep','Oct','Nov','Dec');
$Updated = &wtime(time,''); # Format it as local time
if ($#ARGV == 0)
{
$ErrorLog = $ARGV[0];
if ($ErrorLog =~ /^-/) { &usage; }
}
else
{
$ErrorLog = '/dc/ud/www/httpd/logs/error_log'; # Server's Error Log
}
%pseen = ();
%unseen = ();
%cfgseen = ();
$UnusualHeader = "Unusual Error Messages";
$ConfigHeader = "Client Denied by Server Configuration";
$PermHeader = "File Permissions Deny Access";
$ExistHeader = "File Does Not Exist";
# ==========================================================================
# Start output
$| = 1; # set unbuffered
print "<HTML><HEAD>\n";
print "<TITLE>Collated Error Log Output</TITLE>\n";
print "</HEAD><BODY>\n";
print "<H1>Collated Error Log Output</H1>\n";
print "<EM>Last updated: $Updated</EM>\n";
print "<UL>\n";
print "<LI><A HREF=\"\#Unusual\">$UnusualHeader</A>\n";
print "<LI><A HREF=\"\#Config\">$ConfigHeader</A>\n";
print "<LI><A HREF=\"\#Perm\">$PermHeader</A>\n";
print "<LI><A HREF=\"\#Exist\">$ExistHeader</A>\n";
print "</UL>\n";
# ==========================================================================
print "<HR>\n";
print "<H2><A NAME=\"Unusual\">$UnusualHeader</A></H2>\n";
print "<PRE>\n";
&process_log;
print "</PRE>\n";
# ==========================================================================
print "<HR>\n";
print "<H2><A NAME=\"Config\">$ConfigHeader</A></H2>\n";
print "<PRE>\n";
foreach $pfile ( sort keys(cfgseen) )
{
printf ("%-70s |%s\n", $pfile, $cfgseen{$pfile});
}
print "</PRE>\n";
# ==========================================================================
print "<HR>\n";
print "<H2><A NAME=\"Perm\">$PermHeader</A></H2>\n";
print "<PRE>\n";
foreach $pfile ( sort keys(pseen) )
{
printf ("%-70s |%s\n", $pfile, $pseen{$pfile});
}
print "</PRE>\n";
# ==========================================================================
print "<HR>\n";
print "<H2><A NAME=\"Exist\">$ExistHeader</A></H2>\n";
foreach $pfile ( sort keys(unseen) )
{
print $pfile, "\n";
print "<UL>\n";
foreach $exref ( sort split(/>/, $unseen{$pfile}) )
{
$ref = $exref;
$ref =~ s/ .*//;
print "<LI><A HREF=\"$ref\">$exref</A>\n";
}
print "</UL>\n";
}
# ==========================================================================
print "<HR>\n";
print "<ADDRESS>\n";
print "$Owner\n";
print "</ADDRESS>\n";
print "</BODY></HTML>\n";
exit(0);
# ==========================================================================
# ==========================================================================
# Iterate through the logfile
sub process_log
{
local($in_entry) = 0;
local($entry) = '';
local($ua) = '';
local($ref) = '';
local($saveline) = '';
open(LOG, $ErrorLog) || die "$!: Failed to open $ErrorLog, stopped";
LINE: while (<LOG>)
{
s/\000//g;
s/\&/\&/g;
s/</\</g;
s/>/\>/g;
s/%7E/~/i;
$saveline = $_;
if ($in_entry)
{
if (/^\s+User-Agent: (.*)/) { $ua = $1; next LINE; }
elsif (/^\s+Referer: (.*)/) { $ref = $1; next LINE; }
else
{
$in_entry = 0;
&handle_entry($entry, $ua, $ref);
}
}
$in_entry = 1;
$entry = $saveline;
$ua = '';
$ref = '';
}
&handle_entry($entry, $ua, $ref);
close(LOG);
}
# ==========================================================================
sub handle_entry
{
local($_, $ua, $ref) = @_;
local($reason, $date, $file, $host);
study;
return if (/ timed out/);
return if (/ caught SIGHUP/);
return if (/ successful restart/);
return if (/ killing CGI/);
if (/ httpd: access to /)
{
($date, $file, $host, $reason) =
/^\[([^\]]*)\] httpd: access to (\S+) failed for ([^, ]+), reason: (.*)/;
if ($reason =~ /^file permissions/)
{
$pseen{$file} = $host;
return;
}
if ($reason =~ /^file does not exist/)
{
return if (!$ref); # Don't waste time if no Referer
if ($oldrefs = $unseen{$file})
{
if ($oldrefs !~ /$ref/)
{
$oldrefs .= '>'. $ref ." [$ua]";
$unseen{$file} = $oldrefs;
}
return;
}
$unseen{$file} = $ref ." [$ua]";
return;
}
if ($reason =~ /^client denied by server config/)
{
$cfgseen{$file} = $host;
return;
}
}
print; # This is for unknown/unusual messages
}
# ---------------------------------------------------------------------------
# The following routine is from the wwwdates.pl package of the
# libwww-perl library <http://www.ics.uci.edu/WebSoft/libwww-perl/>.
# ===========================================================================
# wtime() is a modified version of Perl 4.036's ctime.pl
# library by Waldemar Kebsch <kebsch.pad@nixpbe.UUCP> and
# Marion Hakanson <hakanson@cse.ogi.edu>.
#
# wtime returns a time string in the format "Wkd, Dy Mon Year HH:MM:SS Zone"
# with no newline appended.
#
# USAGE:
#
# &wtime(time,''); -- returns the local time with no timezone appended
# As in "Wed, 15 Dec 1993 23:59:59 "
#
# &wtime(time,'GMT'); -- returns GMT time
# As in "Thu, 16 Dec 1993 07:59:59 GMT"
#
sub wtime
{
local($time, $tz) = @_;
local($[.) = 0;
local($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst);
# Use local time if tz is anything other than 'GMT'
($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) =
($tz eq 'GMT') ? gmtime($time) : localtime($time);
$year += 1900;
sprintf("%s, %02d %s %04d %02d:%02d:%02d %s", $DoW[$wday],
$mday, $MoY[$mon], $year, $hour, $min, $sec, $tz);
}
1;
only work when patch E30 is applied, but it may be useful anyway.
Feel free to do with it what you want -- if someone wants to add it to
the Apache distribution (along with some form of E30, please add the
Apache copyright first.
.......Roy
----------------------cut here
#!/usr/public/bin/perl
# ==========================================================================
sub usage {
die <<"EndUsage";
USAGE: check_error_log [ error_log_name ]
This program reads the httpd error_log logfile (or whatever file is
given as the single argument) looking for failed requests (bad URLs)
which include a valid Referer (and thus can be tracked to the origin
of the reference). It assumes that the error_log has been enhanced
to include the referer and user-agent information.
The output is an HTML file that makes it easy to check references.
Created by Roy Fielding, 10 Apr 1995
EndUsage
}
if ($#ARGV > 0) { &usage; }
# ==========================================================================
# Get defaults
$Owner = "Roy Fielding <fielding@ics.uci.edu>";
@DoW = ('Sun','Mon','Tue','Wed','Thu','Fri','Sat');
@MoY = ('Jan','Feb','Mar','Apr','May','Jun',
'Jul','Aug','Sep','Oct','Nov','Dec');
$Updated = &wtime(time,''); # Format it as local time
if ($#ARGV == 0)
{
$ErrorLog = $ARGV[0];
if ($ErrorLog =~ /^-/) { &usage; }
}
else
{
$ErrorLog = '/dc/ud/www/httpd/logs/error_log'; # Server's Error Log
}
%pseen = ();
%unseen = ();
%cfgseen = ();
$UnusualHeader = "Unusual Error Messages";
$ConfigHeader = "Client Denied by Server Configuration";
$PermHeader = "File Permissions Deny Access";
$ExistHeader = "File Does Not Exist";
# ==========================================================================
# Start output
$| = 1; # set unbuffered
print "<HTML><HEAD>\n";
print "<TITLE>Collated Error Log Output</TITLE>\n";
print "</HEAD><BODY>\n";
print "<H1>Collated Error Log Output</H1>\n";
print "<EM>Last updated: $Updated</EM>\n";
print "<UL>\n";
print "<LI><A HREF=\"\#Unusual\">$UnusualHeader</A>\n";
print "<LI><A HREF=\"\#Config\">$ConfigHeader</A>\n";
print "<LI><A HREF=\"\#Perm\">$PermHeader</A>\n";
print "<LI><A HREF=\"\#Exist\">$ExistHeader</A>\n";
print "</UL>\n";
# ==========================================================================
print "<HR>\n";
print "<H2><A NAME=\"Unusual\">$UnusualHeader</A></H2>\n";
print "<PRE>\n";
&process_log;
print "</PRE>\n";
# ==========================================================================
print "<HR>\n";
print "<H2><A NAME=\"Config\">$ConfigHeader</A></H2>\n";
print "<PRE>\n";
foreach $pfile ( sort keys(cfgseen) )
{
printf ("%-70s |%s\n", $pfile, $cfgseen{$pfile});
}
print "</PRE>\n";
# ==========================================================================
print "<HR>\n";
print "<H2><A NAME=\"Perm\">$PermHeader</A></H2>\n";
print "<PRE>\n";
foreach $pfile ( sort keys(pseen) )
{
printf ("%-70s |%s\n", $pfile, $pseen{$pfile});
}
print "</PRE>\n";
# ==========================================================================
print "<HR>\n";
print "<H2><A NAME=\"Exist\">$ExistHeader</A></H2>\n";
foreach $pfile ( sort keys(unseen) )
{
print $pfile, "\n";
print "<UL>\n";
foreach $exref ( sort split(/>/, $unseen{$pfile}) )
{
$ref = $exref;
$ref =~ s/ .*//;
print "<LI><A HREF=\"$ref\">$exref</A>\n";
}
print "</UL>\n";
}
# ==========================================================================
print "<HR>\n";
print "<ADDRESS>\n";
print "$Owner\n";
print "</ADDRESS>\n";
print "</BODY></HTML>\n";
exit(0);
# ==========================================================================
# ==========================================================================
# Iterate through the logfile
sub process_log
{
local($in_entry) = 0;
local($entry) = '';
local($ua) = '';
local($ref) = '';
local($saveline) = '';
open(LOG, $ErrorLog) || die "$!: Failed to open $ErrorLog, stopped";
LINE: while (<LOG>)
{
s/\000//g;
s/\&/\&/g;
s/</\</g;
s/>/\>/g;
s/%7E/~/i;
$saveline = $_;
if ($in_entry)
{
if (/^\s+User-Agent: (.*)/) { $ua = $1; next LINE; }
elsif (/^\s+Referer: (.*)/) { $ref = $1; next LINE; }
else
{
$in_entry = 0;
&handle_entry($entry, $ua, $ref);
}
}
$in_entry = 1;
$entry = $saveline;
$ua = '';
$ref = '';
}
&handle_entry($entry, $ua, $ref);
close(LOG);
}
# ==========================================================================
sub handle_entry
{
local($_, $ua, $ref) = @_;
local($reason, $date, $file, $host);
study;
return if (/ timed out/);
return if (/ caught SIGHUP/);
return if (/ successful restart/);
return if (/ killing CGI/);
if (/ httpd: access to /)
{
($date, $file, $host, $reason) =
/^\[([^\]]*)\] httpd: access to (\S+) failed for ([^, ]+), reason: (.*)/;
if ($reason =~ /^file permissions/)
{
$pseen{$file} = $host;
return;
}
if ($reason =~ /^file does not exist/)
{
return if (!$ref); # Don't waste time if no Referer
if ($oldrefs = $unseen{$file})
{
if ($oldrefs !~ /$ref/)
{
$oldrefs .= '>'. $ref ." [$ua]";
$unseen{$file} = $oldrefs;
}
return;
}
$unseen{$file} = $ref ." [$ua]";
return;
}
if ($reason =~ /^client denied by server config/)
{
$cfgseen{$file} = $host;
return;
}
}
print; # This is for unknown/unusual messages
}
# ---------------------------------------------------------------------------
# The following routine is from the wwwdates.pl package of the
# libwww-perl library <http://www.ics.uci.edu/WebSoft/libwww-perl/>.
# ===========================================================================
# wtime() is a modified version of Perl 4.036's ctime.pl
# library by Waldemar Kebsch <kebsch.pad@nixpbe.UUCP> and
# Marion Hakanson <hakanson@cse.ogi.edu>.
#
# wtime returns a time string in the format "Wkd, Dy Mon Year HH:MM:SS Zone"
# with no newline appended.
#
# USAGE:
#
# &wtime(time,''); -- returns the local time with no timezone appended
# As in "Wed, 15 Dec 1993 23:59:59 "
#
# &wtime(time,'GMT'); -- returns GMT time
# As in "Thu, 16 Dec 1993 07:59:59 GMT"
#
sub wtime
{
local($time, $tz) = @_;
local($[.) = 0;
local($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst);
# Use local time if tz is anything other than 'GMT'
($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) =
($tz eq 'GMT') ? gmtime($time) : localtime($time);
$year += 1900;
sprintf("%s, %02d %s %04d %02d:%02d:%02d %s", $DoW[$wday],
$mday, $MoY[$mon], $year, $hour, $min, $sec, $tz);
}
1;