Author: lewismc
Date: Fri May 11 22:29:23 2012
New Revision: 1337401
URL: http://svn.apache.org/viewvc?rev=1337401&view=rev
Log:
commit to address ANY23-86.
Added:
incubator/any23/trunk/plugins/basic-crawler/src/test/resources/
incubator/any23/trunk/plugins/basic-crawler/src/test/resources/log4j.properties
Modified:
incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/cli/CrawlerTest.java
incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/plugin/crawler/SiteCrawlerTest.java
Modified:
incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/cli/CrawlerTest.java
URL:
http://svn.apache.org/viewvc/incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/cli/CrawlerTest.java?rev=1337401&r1=1337400&r2=1337401&view=diff
==============================================================================
---
incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/cli/CrawlerTest.java
(original)
+++
incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/cli/CrawlerTest.java
Fri May 11 22:29:23 2012
@@ -51,7 +51,7 @@ public class CrawlerTest extends Any23On
final File outFile = File.createTempFile("crawler-test", ".nq");
outFile.delete();
- logger.debug( "Outfile: " + outFile.getAbsolutePath() );
+ logger.info( "Outfile: " + outFile.getAbsolutePath() );
final Future<?> future = Executors.newSingleThreadExecutor().submit(
new Runnable() {
Modified:
incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/plugin/crawler/SiteCrawlerTest.java
URL:
http://svn.apache.org/viewvc/incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/plugin/crawler/SiteCrawlerTest.java?rev=1337401&r1=1337400&r2=1337401&view=diff
==============================================================================
---
incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/plugin/crawler/SiteCrawlerTest.java
(original)
+++
incubator/any23/trunk/plugins/basic-crawler/src/test/java/org/apache/any23/plugin/crawler/SiteCrawlerTest.java
Fri May 11 22:29:23 2012
@@ -28,6 +28,7 @@ import java.io.File;
import java.net.URL;
import java.util.HashSet;
import java.util.Set;
+import java.util.Iterator;
/**
* Test case for {@link SiteCrawler}.
@@ -52,13 +53,19 @@ public class SiteCrawlerTest extends Any
final SiteCrawler controller = new SiteCrawler(tmpFile);
controller.setMaxPages(100);
+ logger.info("Crawler4j: Setting max num of pages to: " +
controller.getMaxPages());
controller.setPolitenessDelay(500);
+ logger.info("Crawler4j: Setting Politeness delay to: " +
controller.getPolitenessDelay() + "ms");
final Set<String> distinctPages = new HashSet<String>();
controller.addListener(new CrawlerListener() {
@Override
public void visitedPage(Page page) {
distinctPages.add( page.getWebURL().getURL() );
+ Iterator it = distinctPages.iterator();
+ while (it.hasNext()) {
+ logger.info("Crawler4j: Fetching page - " + it.next());
+ }
}
});
@@ -69,7 +76,7 @@ public class SiteCrawlerTest extends Any
}
controller.stop();
- logger.debug("Crawled pages: " + distinctPages.size());
+ logger.info("Distinct pages: " + distinctPages.size());
Assert.assertTrue("Expected some page crawled.", distinctPages.size()
> 0);
}
Added:
incubator/any23/trunk/plugins/basic-crawler/src/test/resources/log4j.properties
URL:
http://svn.apache.org/viewvc/incubator/any23/trunk/plugins/basic-crawler/src/test/resources/log4j.properties?rev=1337401&view=auto
==============================================================================
---
incubator/any23/trunk/plugins/basic-crawler/src/test/resources/log4j.properties
(added)
+++
incubator/any23/trunk/plugins/basic-crawler/src/test/resources/log4j.properties
Fri May 11 22:29:23 2012
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootCategory=DEBUG, stdout
+
+log4j.appender.stdout.Threshold=DEBUG
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+log4j.appender.stdout.layout.ConversionPattern=%5p [%t] %m%n
