Author: thorsten
Date: Mon Dec 21 14:12:43 2009
New Revision: 892828
URL: http://svn.apache.org/viewvc?rev=892828&view=rev
Log:
DROIDS-68 HandlerFactory fails with multithreaded implementation
Reporter/patch: Javier Puerto
Thanks Javier
Added:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ReportHandler.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java
Modified:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerDroid.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
Modified:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java?rev=892828&r1=892827&r2=892828&view=diff
==============================================================================
---
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java
(original)
+++
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingDroid.java
Mon Dec 21 14:12:43 2009
@@ -18,28 +18,25 @@
import java.net.URI;
import java.net.URISyntaxException;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.concurrent.TimeUnit;
import org.apache.droids.AbstractDroid;
import org.apache.droids.LinkTask;
-import org.apache.droids.api.*;
+import org.apache.droids.api.Link;
+import org.apache.droids.api.TaskMaster;
+import org.apache.droids.api.TaskQueue;
+import org.apache.droids.api.Worker;
import org.apache.droids.exception.InvalidTaskException;
-import org.apache.droids.helper.factories.HandlerFactory;
import org.apache.droids.helper.factories.ParserFactory;
import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
-import org.apache.droids.impl.MultiThreadedTaskMaster;
-import org.apache.droids.impl.SimpleTaskQueue;
-public class CrawlingDroid extends AbstractDroid<Link>
+public abstract class CrawlingDroid extends AbstractDroid<Link>
{
private Collection<String> initialLocations;
ProtocolFactory protocolFactory;
ParserFactory parserFactory;
- HandlerFactory handlerFactory;
URLFiltersFactory filtersFactory;
public CrawlingDroid( TaskQueue<Link> queue, TaskMaster<Link> taskMaster )
@@ -76,9 +73,7 @@
log.info( "FINISHED!!!" );
}
- public CrawlingWorker getNewWorker() {
- return new CrawlingWorker( this );
- }
+ public abstract Worker<Link> getNewWorker();
public ProtocolFactory getProtocolFactory() {
return protocolFactory;
@@ -96,14 +91,6 @@
this.parserFactory = parserFactory;
}
- public HandlerFactory getHandlerFactory() {
- return handlerFactory;
- }
-
- public void setHandlerFactory(HandlerFactory handlerFactory) {
- this.handlerFactory = handlerFactory;
- }
-
public URLFiltersFactory getFiltersFactory() {
return filtersFactory;
}
@@ -111,25 +98,5 @@
public void setFiltersFactory(URLFiltersFactory filtersFactory) {
this.filtersFactory = filtersFactory;
}
-
- //------------------------------------------------------------------
- //------------------------------------------------------------------
-
- public static void main( String[] args ) throws Exception
- {
- MultiThreadedTaskMaster<Link> taskMaster = new
MultiThreadedTaskMaster<Link>();
- taskMaster.setMaxThreads( 3 );
-
- TaskQueue<Link> queue = new SimpleTaskQueue<Link>();
-
- Collection<String> locations = new ArrayList<String>();
- locations.add( args[0] );
-
- CrawlingDroid simple = new CrawlingDroid( queue, taskMaster );
- simple.setInitialLocations( locations );
- simple.init();
- simple.start(); // TODO? perhaps start internally calls init()?
- simple.getTaskMaster().awaitTermination(0, TimeUnit.SECONDS);
- }
}
Modified:
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java?rev=892828&r1=892827&r2=892828&view=diff
==============================================================================
---
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
(original)
+++
incubator/droids/trunk/droids-core/src/main/java/org/apache/droids/robot/crawler/CrawlingWorker.java
Mon Dec 21 14:12:43 2009
@@ -31,11 +31,13 @@
import org.apache.droids.api.Worker;
import org.apache.droids.exception.DroidsException;
import org.apache.droids.helper.Loggable;
+import org.apache.droids.helper.factories.HandlerFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
public class CrawlingWorker extends Loggable implements Worker<Link>
{
private final CrawlingDroid droid;
+ HandlerFactory handlerFactory;
public CrawlingWorker( CrawlingDroid droid )
{
@@ -93,7 +95,7 @@
protected void handle(ContentEntity entity, Link link)
throws DroidsException, IOException
{
- droid.getHandlerFactory().handle(link.getURI(), entity);
+ getHandlerFactory().handle(link.getURI(), entity);
}
protected Collection<Link> getFilteredOutlinks( Parse parse )
@@ -110,5 +112,13 @@
}
return filtered.values();
}
+
+ public HandlerFactory getHandlerFactory() {
+ return handlerFactory;
+ }
+
+ public void setHandlerFactory(HandlerFactory handlerFactory) {
+ this.handlerFactory = handlerFactory;
+ }
}
Modified:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java?rev=892828&r1=892827&r2=892828&view=diff
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java
(original)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/DroidsFactory.java
Mon Dec 21 14:12:43 2009
@@ -32,15 +32,20 @@
import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.helper.factories.URLFiltersFactory;
import org.apache.droids.impl.DefaultTaskExceptionHandler;
+import org.apache.droids.impl.ExceptionCrawlingDroid;
+import org.apache.droids.impl.ReportCrawlingDroid;
+import org.apache.droids.impl.SaveCrawlingDroid;
import org.apache.droids.impl.SequentialTaskMaster;
import org.apache.droids.impl.SimpleTaskQueue;
+import org.apache.droids.impl.SimpleTaskQueueWithHistory;
+import org.apache.droids.impl.SysoutCrawlingDroid;
import org.apache.droids.parse.html.HtmlParser;
import org.apache.droids.protocol.http.HttpProtocol;
import org.apache.droids.robot.crawler.CrawlingDroid;
public class DroidsFactory
{
-
+
public static ParserFactory createDefaultParserFactory() {
ParserFactory parserFactory = new ParserFactory();
HtmlParser htmlParser = new HtmlParser();
@@ -85,27 +90,77 @@
return handlerFactory;
}
- public static Droid<Link> createSimpleCrawlingDroid(
- String targetURI, Handler testHandler) {
+ public static Droid<Link> createSimpleSaveCrawlingDroid(
+ String targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory = createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
+
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ SimpleTaskQueueWithHistory<Link> simpleQueue = new
SimpleTaskQueueWithHistory<Link>();
+
+ SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
+ taskMaster.setDelayTimer( simpleDelayTimer );
+ taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
+
+ CrawlingDroid crawler = new SaveCrawlingDroid( simpleQueue, taskMaster );
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add( targetURI );
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+ public static Droid<Link> createSimpleReportCrawlingDroid(
+ String targetURI) {
ParserFactory parserFactory = createDefaultParserFactory();
ProtocolFactory protocolFactory = createDefaultProtocolFactory();
URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
- HandlerFactory handlerFactory = createDefaultHandlerFactory(testHandler);
SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
simpleDelayTimer.setDelayMillis(100);
- SimpleTaskQueue<Link> simpleQueue = new SimpleTaskQueue<Link>();
+ SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
+ taskMaster.setDelayTimer( simpleDelayTimer );
+ taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
+
+ SimpleTaskQueue<Link> queue = new SimpleTaskQueue<Link>();
+
+ CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
+ crawler.setFiltersFactory(filtersFactory);
+ crawler.setParserFactory(parserFactory);
+ crawler.setProtocolFactory(protocolFactory);
+
+ Collection<String> initialLocations = new ArrayList<String>();
+ initialLocations.add( targetURI );
+ crawler.setInitialLocations(initialLocations);
+ return crawler;
+ }
+
+ public static Droid<Link> createSimpleExceptionCrawlingDroid(
+ String targetURI) {
+ ParserFactory parserFactory = createDefaultParserFactory();
+ ProtocolFactory protocolFactory = createDefaultProtocolFactory();
+ URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();
+ SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
+ simpleDelayTimer.setDelayMillis(100);
+
+ SimpleTaskQueue<Link> queue = new SimpleTaskQueue<Link>();
+
SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
taskMaster.setDelayTimer( simpleDelayTimer );
taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
- CrawlingDroid crawler = new CrawlingDroid( simpleQueue, taskMaster );
+ CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
crawler.setFiltersFactory(filtersFactory);
crawler.setParserFactory(parserFactory);
crawler.setProtocolFactory(protocolFactory);
- crawler.setHandlerFactory(handlerFactory);
Collection<String> initialLocations = new ArrayList<String>();
initialLocations.add( targetURI );
Modified:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerDroid.java?rev=892828&r1=892827&r2=892828&view=diff
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerDroid.java
(original)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerDroid.java
Mon Dec 21 14:12:43 2009
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.droids.examples;
-
-import org.apache.droids.api.Link;
-import org.apache.droids.api.TaskMaster;
-import org.apache.droids.api.TaskQueue;
-import org.apache.droids.robot.crawler.CrawlingDroid;
-
-public class IndexerDroid extends CrawlingDroid {
-
- public IndexerDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster) {
- super(queue, taskMaster);
- }
-
- private String updateUrl;
-
- public String getUpdateUrl() {
- return updateUrl;
- }
-
- public void setUpdateUrl(String updateUrl) {
- this.updateUrl = updateUrl;
- }
-}
Modified:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java?rev=892828&r1=892827&r2=892828&view=diff
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java
(original)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/IndexerWorker.java
Mon Dec 21 14:12:43 2009
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.droids.examples;
-
-import org.apache.droids.api.ContentEntity;
-import org.apache.droids.api.Handler;
-import org.apache.droids.api.Link;
-import org.apache.droids.robot.crawler.CrawlingWorker;
-
-public class IndexerWorker extends CrawlingWorker {
-
- final Handler solrHandler;
-
- public IndexerWorker(IndexerDroid droid, Handler handler) {
- super(droid);
-
- solrHandler = handler;
- }
-
-
- @Override
- protected void handle(ContentEntity entity, Link link)
- {
- // TODO -- something different...
- }
-}
Modified:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java?rev=892828&r1=892827&r2=892828&view=diff
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java
(original)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/SimpleRuntime.java
Mon Dec 21 14:12:43 2009
@@ -33,6 +33,7 @@
import org.apache.droids.impl.DefaultTaskExceptionHandler;
import org.apache.droids.impl.SequentialTaskMaster;
import org.apache.droids.impl.SimpleTaskQueue;
+import org.apache.droids.impl.SysoutCrawlingDroid;
import org.apache.droids.net.RegexURLFilter;
import org.apache.droids.parse.html.HtmlParser;
import org.apache.droids.protocol.http.DroidsHttpClient;
@@ -131,11 +132,10 @@
taskMaster.setDelayTimer( simpleDelayTimer );
taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
- CrawlingDroid helloCrawler = new CrawlingDroid( simpleQueue, taskMaster );
+ CrawlingDroid helloCrawler = new SysoutCrawlingDroid( simpleQueue,
taskMaster );
helloCrawler.setFiltersFactory(filtersFactory);
helloCrawler.setParserFactory(parserFactory);
helloCrawler.setProtocolFactory(protocolFactory);
- helloCrawler.setHandlerFactory(handlerFactory);
Collection<String> initialLocations = new ArrayList<String>();
initialLocations.add( targetURL );
Added:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ExceptionReportHandler.java?rev=892828&view=auto
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
(added)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ExceptionReportHandler.java
Mon Dec 21 14:12:43 2009
@@ -0,0 +1,20 @@
+package org.apache.droids.examples.handler;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.droids.api.ContentEntity;
+import org.apache.droids.exception.DroidsException;
+
+public class ExceptionReportHandler extends ReportHandler {
+
+ @Override
+ public void handle(URI uri, ContentEntity entity) throws IOException,
+ DroidsException {
+ super.handle(uri, entity);
+ if (uri.getPath().equals("/page3_html")) {
+ throw new RuntimeException("Oppsie!!!");
+ }
+ }
+
+}
Added:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ReportHandler.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ReportHandler.java?rev=892828&view=auto
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ReportHandler.java
(added)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/examples/handler/ReportHandler.java
Mon Dec 21 14:12:43 2009
@@ -0,0 +1,34 @@
+package org.apache.droids.examples.handler;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.droids.api.ContentEntity;
+import org.apache.droids.api.Handler;
+import org.apache.droids.exception.DroidsException;
+
+public class ReportHandler implements Handler {
+
+ private static Set<String> report;
+
+ static {
+ report = Collections.synchronizedSet(new HashSet<String>());
+ }
+
+ @Override
+ public void handle(URI uri, ContentEntity entity) throws IOException,
+ DroidsException {
+ report.add(uri.toString());
+ }
+
+ public static Set<String> getReport() {
+ return report;
+ }
+
+ public static void recycle() {
+ report.clear();
+ }
+}
Added:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java?rev=892828&view=auto
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java
(added)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ExceptionCrawlingDroid.java
Mon Dec 21 14:12:43 2009
@@ -0,0 +1,30 @@
+package org.apache.droids.impl;
+
+import java.net.URI;
+
+import org.apache.droids.DroidsFactory;
+import org.apache.droids.api.ContentEntity;
+import org.apache.droids.api.Handler;
+import org.apache.droids.api.Link;
+import org.apache.droids.api.TaskMaster;
+import org.apache.droids.api.TaskQueue;
+import org.apache.droids.api.Worker;
+import org.apache.droids.examples.handler.ExceptionReportHandler;
+import org.apache.droids.robot.crawler.CrawlingDroid;
+import org.apache.droids.robot.crawler.CrawlingWorker;
+
+public class ExceptionCrawlingDroid extends CrawlingDroid {
+
+ public ExceptionCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link>
taskMaster) {
+ super(queue, taskMaster);
+ }
+
+ @Override
+ public Worker<Link> getNewWorker() {
+ final CrawlingWorker worker = new CrawlingWorker( this );
+ Handler testHandler = new ExceptionReportHandler();
+
worker.setHandlerFactory(DroidsFactory.createDefaultHandlerFactory(testHandler));
+ return worker;
+ }
+
+}
Added:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java?rev=892828&view=auto
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java
(added)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/ReportCrawlingDroid.java
Mon Dec 21 14:12:43 2009
@@ -0,0 +1,27 @@
+package org.apache.droids.impl;
+
+import org.apache.droids.DroidsFactory;
+import org.apache.droids.api.Link;
+import org.apache.droids.api.TaskMaster;
+import org.apache.droids.api.TaskQueue;
+import org.apache.droids.api.Worker;
+import org.apache.droids.examples.handler.ReportHandler;
+import org.apache.droids.handle.Save;
+import org.apache.droids.handle.Sysout;
+import org.apache.droids.robot.crawler.CrawlingDroid;
+import org.apache.droids.robot.crawler.CrawlingWorker;
+
+public class ReportCrawlingDroid extends CrawlingDroid {
+
+ public ReportCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link>
taskMaster) {
+ super(queue, taskMaster);
+ }
+
+ @Override
+ public Worker<Link> getNewWorker() {
+ final CrawlingWorker worker = new CrawlingWorker( this );
+ worker.setHandlerFactory(DroidsFactory.createDefaultHandlerFactory(new
ReportHandler()));
+ return worker;
+ }
+
+}
Added:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java?rev=892828&view=auto
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java
(added)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SaveCrawlingDroid.java
Mon Dec 21 14:12:43 2009
@@ -0,0 +1,25 @@
+package org.apache.droids.impl;
+
+import org.apache.droids.DroidsFactory;
+import org.apache.droids.api.Link;
+import org.apache.droids.api.TaskMaster;
+import org.apache.droids.api.TaskQueue;
+import org.apache.droids.api.Worker;
+import org.apache.droids.handle.Save;
+import org.apache.droids.robot.crawler.CrawlingDroid;
+import org.apache.droids.robot.crawler.CrawlingWorker;
+
+public class SaveCrawlingDroid extends CrawlingDroid {
+
+ public SaveCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link> taskMaster)
{
+ super(queue, taskMaster);
+ }
+
+ @Override
+ public Worker<Link> getNewWorker() {
+ final CrawlingWorker worker = new CrawlingWorker( this );
+ worker.setHandlerFactory(DroidsFactory.createDefaultHandlerFactory(new
Save()));
+ return worker;
+ }
+
+}
Added:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java?rev=892828&view=auto
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java
(added)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/SysoutCrawlingDroid.java
Mon Dec 21 14:12:43 2009
@@ -0,0 +1,26 @@
+package org.apache.droids.impl;
+
+import org.apache.droids.DroidsFactory;
+import org.apache.droids.api.Link;
+import org.apache.droids.api.TaskMaster;
+import org.apache.droids.api.TaskQueue;
+import org.apache.droids.api.Worker;
+import org.apache.droids.handle.Save;
+import org.apache.droids.handle.Sysout;
+import org.apache.droids.robot.crawler.CrawlingDroid;
+import org.apache.droids.robot.crawler.CrawlingWorker;
+
+public class SysoutCrawlingDroid extends CrawlingDroid {
+
+ public SysoutCrawlingDroid(TaskQueue<Link> queue, TaskMaster<Link>
taskMaster) {
+ super(queue, taskMaster);
+ }
+
+ @Override
+ public Worker<Link> getNewWorker() {
+ final CrawlingWorker worker = new CrawlingWorker( this );
+ worker.setHandlerFactory(DroidsFactory.createDefaultHandlerFactory(new
Sysout()));
+ return worker;
+ }
+
+}
Modified:
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java?rev=892828&r1=892827&r2=892828&view=diff
==============================================================================
---
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
(original)
+++
incubator/droids/trunk/droids-core/src/test/java/org/apache/droids/impl/TestSimpleDroid.java
Mon Dec 21 14:12:43 2009
@@ -17,20 +17,16 @@
package org.apache.droids.impl;
import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.TimeUnit;
import junit.framework.Assert;
import org.apache.droids.DroidsFactory;
-import org.apache.droids.api.ContentEntity;
import org.apache.droids.api.Droid;
-import org.apache.droids.api.Handler;
import org.apache.droids.api.Link;
import org.apache.droids.api.TaskExceptionHandler;
import org.apache.droids.api.TaskExceptionResult;
+import org.apache.droids.examples.handler.ReportHandler;
import org.apache.droids.localserver.LocalHttpServer;
import org.apache.droids.localserver.ResourceHandler;
import org.junit.After;
@@ -61,31 +57,22 @@
String baseURI = "http:/" + this.testserver.getServiceAddress();
String targetURI = baseURI + "/start_html";
- final List<URI> visitedLinks = new ArrayList<URI>();
-
- Handler testHandler = new Handler() {
-
- public void handle(URI uri, ContentEntity entity) {
- visitedLinks.add(uri);
- }
-
- };
- Droid<Link> droid = DroidsFactory.createSimpleCrawlingDroid(
- targetURI,
- testHandler);
+ Droid<Link> droid =
DroidsFactory.createSimpleReportCrawlingDroid(targetURI);
droid.init();
droid.start();
droid.getTaskMaster().awaitTermination(30, TimeUnit.SECONDS);
- Assert.assertFalse(visitedLinks.isEmpty());
- Assert.assertEquals(5, visitedLinks.size());
- Assert.assertEquals(new URI(baseURI + "/start_html"), visitedLinks.get(0));
- Assert.assertEquals(new URI(baseURI + "/page1_html"), visitedLinks.get(1));
- Assert.assertEquals(new URI(baseURI + "/page2_html"), visitedLinks.get(2));
- Assert.assertEquals(new URI(baseURI + "/page3_html"), visitedLinks.get(3));
- Assert.assertEquals(new URI(baseURI + "/page4_html"), visitedLinks.get(4));
+ Assert.assertFalse(ReportHandler.getReport().isEmpty());
+ Assert.assertEquals(5, ReportHandler.getReport().size());
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/start_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page1_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page2_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page3_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page4_html"));
+
+ ReportHandler.recycle();
}
@Test
@@ -97,22 +84,8 @@
String baseURI = "http:/" + this.testserver.getServiceAddress();
String targetURI = baseURI + "/start_html";
- final List<URI> visitedLinks = new ArrayList<URI>();
-
- Handler testHandler = new Handler() {
-
- public void handle(URI uri, ContentEntity entity) {
- visitedLinks.add(uri);
- if (uri.getPath().equals("/page3_html")) {
- throw new RuntimeException("Oppsie!!!");
- }
- }
-
- };
-
- Droid<Link> droid = DroidsFactory.createSimpleCrawlingDroid(
- targetURI,
- testHandler);
+ Droid<Link> droid = DroidsFactory.createSimpleExceptionCrawlingDroid(
+ targetURI);
SequentialTaskMaster<Link> taskMaster = (SequentialTaskMaster<Link>)
droid.getTaskMaster();
taskMaster.setExceptionHandler(new TaskExceptionHandler() {
@@ -130,12 +103,14 @@
droid.start();
droid.getTaskMaster().awaitTermination(30, TimeUnit.SECONDS);
- Assert.assertFalse(visitedLinks.isEmpty());
- Assert.assertEquals(4, visitedLinks.size());
- Assert.assertEquals(new URI(baseURI + "/start_html"), visitedLinks.get(0));
- Assert.assertEquals(new URI(baseURI + "/page1_html"), visitedLinks.get(1));
- Assert.assertEquals(new URI(baseURI + "/page2_html"), visitedLinks.get(2));
- Assert.assertEquals(new URI(baseURI + "/page3_html"), visitedLinks.get(3));
+ Assert.assertFalse(ReportHandler.getReport().isEmpty());
+ Assert.assertEquals(4, ReportHandler.getReport().size());
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/start_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page1_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page2_html"));
+ Assert.assertTrue(ReportHandler.getReport().contains(baseURI +
"/page3_html"));
+
+ ReportHandler.recycle();
}
}
