Liam Quinn <[EMAIL PROTECTED]> writes: > LWP::RobotUA won't parse a robots.txt file if the file does not contain > "Disallow". The check for "Disallow" is case sensitive, but according to > the robot exclusion standard, field names are case insensitive. This > causes LWP::RobotUA to ignore some robots.txt files that it should parse. > > Attached is a patch that makes the check for "Disallow" case insensitive. > The patch is against libwww-perl 5.76 (RobotUA.pm 1.23).
Thanks! Applied as: Index: lib/LWP/RobotUA.pm =================================================================== RCS file: /cvsroot/libwww-perl/lwp5/lib/LWP/RobotUA.pm,v retrieving revision 1.23 retrieving revision 1.24 diff -u -p -r1.23 -r1.24 --- lib/LWP/RobotUA.pm 24 Oct 2003 11:13:03 -0000 1.23 +++ lib/LWP/RobotUA.pm 6 Apr 2004 11:02:50 -0000 1.24 @@ -1,10 +1,10 @@ package LWP::RobotUA; -# $Id: RobotUA.pm,v 1.23 2003/10/24 11:13:03 gisle Exp $ +# $Id: RobotUA.pm,v 1.24 2004/04/06 11:02:50 gisle Exp $ require LWP::UserAgent; @ISA = qw(LWP::UserAgent); -$VERSION = sprintf("%d.%02d", q$Revision: 1.23 $ =~ /(\d+)\.(\d+)/); +$VERSION = sprintf("%d.%02d", q$Revision: 1.24 $ =~ /(\d+)\.(\d+)/); require WWW::RobotRules; require HTTP::Request; @@ -126,7 +126,7 @@ sub simple_request my $fresh_until = $robot_res->fresh_until; if ($robot_res->is_success) { my $c = $robot_res->content; - if ($robot_res->content_type =~ m,^text/, && $c =~ /Disallow/) { + if ($robot_res->content_type =~ m,^text/, && $c =~ /^Disallow\s*:/mi) { LWP::Debug::debug("Parsing robot rules"); $self->{'rules'}->parse($robot_url, $c, $fresh_until); } > > -- > Liam Quinn > > > > --- LWP/RobotUA.pm.orig 2003-10-24 07:13:03.000000000 -0400 > +++ LWP/RobotUA.pm 2004-04-03 17:59:04.000000000 -0500 > @@ -126,7 +126,7 @@ > my $fresh_until = $robot_res->fresh_until; > if ($robot_res->is_success) { > my $c = $robot_res->content; > - if ($robot_res->content_type =~ m,^text/, && $c =~ /Disallow/) { > + if ($robot_res->content_type =~ m,^text/, && $c =~ /Disallow/i) { > LWP::Debug::debug("Parsing robot rules"); > $self->{'rules'}->parse($robot_url, $c, $fresh_until); > }