Re: [PATCH] LWP::RobotUA case-sensitive check for "Disallow"

[Gisle Aas]

Liam Quinn <[EMAIL PROTECTED]> writes:

> LWP::RobotUA won't parse a robots.txt file if the file does not contain
> "Disallow".  The check for "Disallow" is case sensitive, but according to
> the robot exclusion standard, field names are case insensitive.  This
> causes LWP::RobotUA to ignore some robots.txt files that it should parse.
> 
> Attached is a patch that makes the check for "Disallow" case insensitive.  
> The patch is against libwww-perl 5.76 (RobotUA.pm 1.23).

Thanks! Applied as:

Index: lib/LWP/RobotUA.pm
===================================================================
RCS file: /cvsroot/libwww-perl/lwp5/lib/LWP/RobotUA.pm,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -p -r1.23 -r1.24
--- lib/LWP/RobotUA.pm  24 Oct 2003 11:13:03 -0000      1.23
+++ lib/LWP/RobotUA.pm  6 Apr 2004 11:02:50 -0000       1.24
@@ -1,10 +1,10 @@
 package LWP::RobotUA;
 
-# $Id: RobotUA.pm,v 1.23 2003/10/24 11:13:03 gisle Exp $
+# $Id: RobotUA.pm,v 1.24 2004/04/06 11:02:50 gisle Exp $
 
 require LWP::UserAgent;
 @ISA = qw(LWP::UserAgent);
-$VERSION = sprintf("%d.%02d", q$Revision: 1.23 $ =~ /(\d+)\.(\d+)/);
+$VERSION = sprintf("%d.%02d", q$Revision: 1.24 $ =~ /(\d+)\.(\d+)/);
 
 require WWW::RobotRules;
 require HTTP::Request;
@@ -126,7 +126,7 @@ sub simple_request
        my $fresh_until = $robot_res->fresh_until;
        if ($robot_res->is_success) {
            my $c = $robot_res->content;
-           if ($robot_res->content_type =~ m,^text/, && $c =~ /Disallow/) {
+           if ($robot_res->content_type =~ m,^text/, && $c =~ /^Disallow\s*:/mi) {
                LWP::Debug::debug("Parsing robot rules");
                $self->{'rules'}->parse($robot_url, $c, $fresh_until);
            }

> 
> -- 
> Liam Quinn
> 
> 
> 
> --- LWP/RobotUA.pm.orig       2003-10-24 07:13:03.000000000 -0400
> +++ LWP/RobotUA.pm    2004-04-03 17:59:04.000000000 -0500
> @@ -126,7 +126,7 @@
>       my $fresh_until = $robot_res->fresh_until;
>       if ($robot_res->is_success) {
>           my $c = $robot_res->content;
> -         if ($robot_res->content_type =~ m,^text/, && $c =~ /Disallow/) {
> +         if ($robot_res->content_type =~ m,^text/, && $c =~ /Disallow/i) {
>               LWP::Debug::debug("Parsing robot rules");
>               $self->{'rules'}->parse($robot_url, $c, $fresh_until);
>           }