Gisle Aas wrote:
> <[EMAIL PROTECTED]> writes:
>
>> And here's a smaller patch - only eleven new lines of code - which
>> should have the same net effect.
>
> This patch looks good. I'll apply it. Can you provide an update to
> t/robot/rules.t as well?
I've added a "warn" line in the case where a record separation is assumed... see
http://www.geocities.com/mvaneerde/RobotRules.patch-4.txt
rules.t patch:
http://www.geocities.com/mvaneerde/rules-patch.txt
make test runs fine
--
Matthew.van.Eerde (at) hbinc.com 805.964.4554 x902
Hispanic Business Inc./HireDiversity.com Software Engineer
--- RobotRules.pm 2005-09-21 10:14:03.000000000 -0700
+++ RobotRules.smallchanges.pm 2005-09-21 10:53:47.000000000 -0700
@@ -34,6 +34,7 @@
my $ua;
my $is_me = 0; # 1 iff this record is for me
my $is_anon = 0; # 1 iff this record is for *
+ my $seen_disallow = 0; # watch for missing record separators
my @me_disallowed = (); # rules disallowed for me
my @anon_disallowed = (); # rules disallowed for *
@@ -53,10 +54,19 @@
if (/^\s*$/) { # blank line
last if $is_me; # That was our record. No need to read the rest.
$is_anon = 0;
+ $seen_disallow = 0;
}
elsif (/^\s*User-Agent\s*:\s*(.*)/i) {
$ua = $1;
$ua =~ s/\s+$//;
+
+ if ($seen_disallow) {
+ # malformed robots.txt - should be a newline here
+ warn "RobotRules <$robot_txt_uri>: User-agent after Disallow in
same record\n" if $^W;
+ # treat as start of a new record
+ $seen_disallow = 0;
+ last if $is_me; # That was our record. No need to read the rest.
+ $is_anon = 0;
+ }
+
if ($is_me) {
# This record already had a User-agent that
# we matched, so just continue.
@@ -75,6 +85,7 @@
}
my $disallow = $1;
$disallow =~ s/\s+$//;
+ $seen_disallow = 1;
if (length $disallow) {
my $ignore;
eval {
@@ -372,6 +383,14 @@
value is '*', the record describes the default access policy for any
robot that has not not matched any of the other records.
+The I<User-Agent> fields must occur before the I<Disallow> fields. If a
+record contains a I<User-Agent> field after a I<Disallow> field, that
+constitutes a malformed record. This parser will assume that a blank
+line should have been placed before that I<User-Agent> field, and will
+break the record into two. All the fields before the I<User-Agent> field
+will constitute a record, and the I<User-Agent> field will be the first
+field in a new record.
+
=item Disallow
The value of this field specifies a partial URL that is not to be
@@ -406,6 +425,22 @@
User-agent: *
Disallow: /
+This is an example of a malformed robots.txt file.
+
+ # robots.txt for ancientcastle.example.com
+ # I've locked myself away.
+ User-agent: *
+ Disallow: /
+ # The castle is your home now, so you can go anywhere you like.
+ User-agent: Belle
+ Disallow: /west-wing/ # except the west wing!
+ # It's good to be the Prince...
+ User-agent: Beast
+ Disallow:
+
+This file is missing the required blank lines between records.
+However, the intention is clear.
+
=head1 SEE ALSO
L<LWP::RobotUA>, L<WWW::RobotRules::AnyDBM_File>
--- rules.t 2005-09-21 11:23:19.000000000 -0700
+++ rules-new.t 2005-09-21 11:23:26.000000000 -0700
@@ -15,7 +15,7 @@
use Carp;
use strict;
-print "1..38\n"; # for Test::Harness
+print "1..50\n"; # for Test::Harness
# We test a number of different /robots.txt files,
#
@@ -64,6 +64,31 @@
Disallow: http://bar/
EOM
+my $content5 = <<EOM;
+# I've locked myself away
+User-agent: *
+Disallow: /
+# The castle is your home now, so you can go anywhere you like.
+User-agent: Belle
+Disallow: /west-wing/ # except the west wing!
+# It's good to be the Prince...
+User-agent: Beast
+Disallow:
+EOM
+
+# same thing backwards
+my $content6 = <<EOM;
+# It's good to be the Prince...
+User-agent: Beast
+Disallow:
+# The castle is your home now, so you can go anywhere you like.
+User-agent: Belle
+Disallow: /west-wing/ # except the west wing!
+# I've locked myself away
+User-agent: *
+Disallow: /
+EOM
+
# and a number of different robots:
my @tests1 = (
@@ -147,6 +172,36 @@
38 => "http://bar/" => 1,
],
+ [$content5, 'Villager/1.0' =>
+ 39 => 'http://foo/west-wing/' => 0,
+ 40 => 'http://foo/' => 0,
+ ],
+
+ [$content5, 'Belle/2.0' =>
+ 41 => 'http://foo/west-wing/' => 0,
+ 42 => 'http://foo/' => 1,
+ ],
+
+ [$content5, 'Beast/3.0' =>
+ 43 => 'http://foo/west-wing/' => 1,
+ 44 => 'http://foo/' => 1,
+ ],
+
+ [$content6, 'Villager/1.0' =>
+ 45 => 'http://foo/west-wing/' => 0,
+ 46 => 'http://foo/' => 0,
+ ],
+
+ [$content6, 'Belle/2.0' =>
+ 47 => 'http://foo/west-wing/' => 0,
+ 48 => 'http://foo/' => 1,
+ ],
+
+ [$content6, 'Beast/3.0' =>
+ 49 => 'http://foo/west-wing/' => 1,
+ 50 => 'http://foo/' => 1,
+ ],
+
# when adding tests, remember to increase
# the maximum at the top