Gisle Aas wrote:
> <[EMAIL PROTECTED]> writes:
> 
>> And here's a smaller patch - only eleven new lines of code - which
>> should have the same net effect.
> 
> This patch looks good.  I'll apply it.  Can you provide an update to
> t/robot/rules.t as well?

I've added a "warn" line in the case where a record separation is assumed... see
http://www.geocities.com/mvaneerde/RobotRules.patch-4.txt

rules.t patch:
http://www.geocities.com/mvaneerde/rules-patch.txt

make test runs fine

-- 
Matthew.van.Eerde (at) hbinc.com               805.964.4554 x902
Hispanic Business Inc./HireDiversity.com       Software Engineer
--- RobotRules.pm       2005-09-21 10:14:03.000000000 -0700
+++ RobotRules.smallchanges.pm  2005-09-21 10:53:47.000000000 -0700
@@ -34,6 +34,7 @@
     my $ua;
     my $is_me = 0;             # 1 iff this record is for me
     my $is_anon = 0;           # 1 iff this record is for *
+    my $seen_disallow = 0;      # watch for missing record separators
     my @me_disallowed = ();    # rules disallowed for me
     my @anon_disallowed = ();  # rules disallowed for *
 
@@ -53,10 +54,19 @@
        if (/^\s*$/) {      # blank line
            last if $is_me; # That was our record. No need to read the rest.
            $is_anon = 0;
+           $seen_disallow = 0;
        }
         elsif (/^\s*User-Agent\s*:\s*(.*)/i) {
            $ua = $1;
            $ua =~ s/\s+$//;
+
+           if ($seen_disallow) {
+               # malformed robots.txt - should be a newline here
+               warn "RobotRules <$robot_txt_uri>: User-agent after Disallow in 
same record\n" if $^W;
+               # treat as start of a new record
+               $seen_disallow = 0;
+               last if $is_me; # That was our record. No need to read the rest.
+               $is_anon = 0;
+           }
+
            if ($is_me) {
                # This record already had a User-agent that
                # we matched, so just continue.
@@ -75,6 +85,7 @@
            }
            my $disallow = $1;
            $disallow =~ s/\s+$//;
+           $seen_disallow = 1;
            if (length $disallow) {
                my $ignore;
                eval {
@@ -372,6 +383,14 @@
 value is '*', the record describes the default access policy for any
 robot that has not not matched any of the other records.
 
+The I<User-Agent> fields must occur before the I<Disallow> fields.  If a
+record contains a I<User-Agent> field after a I<Disallow> field, that
+constitutes a malformed record.  This parser will assume that a blank
+line should have been placed before that I<User-Agent> field, and will
+break the record into two.  All the fields before the I<User-Agent> field
+will constitute a record, and the I<User-Agent> field will be the first
+field in a new record.
+
 =item Disallow
 
 The value of this field specifies a partial URL that is not to be
@@ -406,6 +425,22 @@
   User-agent: *
   Disallow: /
 
+This is an example of a malformed robots.txt file.
+
+  # robots.txt for ancientcastle.example.com
+  # I've locked myself away.
+  User-agent: *
+  Disallow: /
+  # The castle is your home now, so you can go anywhere you like.
+  User-agent: Belle
+  Disallow: /west-wing/ # except the west wing!
+  # It's good to be the Prince...
+  User-agent: Beast
+  Disallow: 
+
+This file is missing the required blank lines between records.
+However, the intention is clear.
+
 =head1 SEE ALSO
 
 L<LWP::RobotUA>, L<WWW::RobotRules::AnyDBM_File>
--- rules.t     2005-09-21 11:23:19.000000000 -0700
+++ rules-new.t 2005-09-21 11:23:26.000000000 -0700
@@ -15,7 +15,7 @@
 use Carp;
 use strict;
 
-print "1..38\n"; # for Test::Harness
+print "1..50\n"; # for Test::Harness
 
 # We test a number of different /robots.txt files,
 #
@@ -64,6 +64,31 @@
 Disallow: http://bar/
 EOM
 
+my $content5 = <<EOM;
+# I've locked myself away
+User-agent: *
+Disallow: /
+# The castle is your home now, so you can go anywhere you like.
+User-agent: Belle
+Disallow: /west-wing/ # except the west wing!
+# It's good to be the Prince...
+User-agent: Beast
+Disallow: 
+EOM
+
+# same thing backwards
+my $content6 = <<EOM;
+# It's good to be the Prince...
+User-agent: Beast
+Disallow: 
+# The castle is your home now, so you can go anywhere you like.
+User-agent: Belle
+Disallow: /west-wing/ # except the west wing!
+# I've locked myself away
+User-agent: *
+Disallow: /
+EOM
+
 # and a number of different robots:
 
 my @tests1 = (
@@ -147,6 +172,36 @@
            38 => "http://bar/"; => 1,
           ],
 
+          [$content5, 'Villager/1.0' =>
+           39 => 'http://foo/west-wing/' => 0,
+           40 => 'http://foo/' => 0,
+          ],
+
+          [$content5, 'Belle/2.0' =>
+           41 => 'http://foo/west-wing/' => 0,
+           42 => 'http://foo/' => 1,
+          ],
+
+          [$content5, 'Beast/3.0' =>
+           43 => 'http://foo/west-wing/' => 1,
+           44 => 'http://foo/' => 1,
+          ],
+
+          [$content6, 'Villager/1.0' =>
+           45 => 'http://foo/west-wing/' => 0,
+           46 => 'http://foo/' => 0,
+          ],
+
+          [$content6, 'Belle/2.0' =>
+           47 => 'http://foo/west-wing/' => 0,
+           48 => 'http://foo/' => 1,
+          ],
+
+          [$content6, 'Beast/3.0' =>
+           49 => 'http://foo/west-wing/' => 1,
+           50 => 'http://foo/' => 1,
+          ],
+
           # when adding tests, remember to increase
           # the maximum at the top
 

Reply via email to