diff -c3p htdig_orig/htdig/Retriever.cc htdig/htdig/Retriever.cc
*** htdig_orig/htdig/Retriever.cc	Sun Jun 29 01:19:30 2003
--- htdig/htdig/Retriever.cc	Sun Jun 29 01:20:28 2003
*************** int Retriever::IsValidURL(const String &
*** 1080,1086 ****
  	//
  	URL testURL((char *) url);
  	Server *server = (Server *) servers[testURL.signature()];
! 	if (server && server->IsDisallowed(url) != 0)
  	{
  		if (debug > 2)
  			cout << endl << "   Rejected: forbidden by server robots.txt!";
--- 1080,1086 ----
  	//
  	URL testURL((char *) url);
  	Server *server = (Server *) servers[testURL.signature()];
! 	if (server && server->IsDisallowed(testURL.path()) != 0)
  	{
  		if (debug > 2)
  			cout << endl << "   Rejected: forbidden by server robots.txt!";
diff -c3p htdig_orig/htdig/Server.cc htdig/htdig/Server.cc
*** htdig_orig/htdig/Server.cc	Sun Jun 29 01:19:19 2003
--- htdig/htdig/Server.cc	Sun Jun 29 01:20:04 2003
*************** void Server::robotstxt(Document &doc)
*** 316,324 ****
  	    if (*rest)
  	    {
  		if (pattern.length())
! 		    pattern << '|' << rest;
  		else
! 		    pattern = rest;
  	    }
  	}
  	//
--- 316,324 ----
  	    if (*rest)
  	    {
  		if (pattern.length())
! 		    pattern << '|' << '^' << rest;
  		else
! 		    pattern << '^' << rest;
  	    }
  	}
  	//