Author: tobr
Date: Mon Jan 7 13:07:00 2013
New Revision: 1429788
URL: http://svn.apache.org/viewvc?rev=1429788&view=rev
Log:
removed Protocol
added Fetcher
Added:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java
- copied, changed from r1429330,
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java
Removed:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/exception/ProtocolNotFoundException.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/helper/factories/ProtocolFactory.java
Modified:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java
Modified:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java?rev=1429788&r1=1429787&r2=1429788&view=diff
==============================================================================
---
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
(original)
+++
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/AbstractDroid.java
Mon Jan 7 13:07:00 2013
@@ -19,7 +19,6 @@ package org.apache.droids.core;
import org.apache.droids.helper.factories.FilterFactory;
import org.apache.droids.helper.factories.HandlerFactory;
import org.apache.droids.helper.factories.ParserFactory;
-import org.apache.droids.helper.factories.ProtocolFactory;
import org.apache.droids.filter.Filter;
import org.apache.droids.parse.Parser;
import org.slf4j.Logger;
@@ -35,7 +34,7 @@ import java.util.concurrent.TimeUnit;
public abstract class AbstractDroid<T extends Task> implements Droid<T> {
protected final Queue<T> queue;
protected final TaskMaster<T> taskMaster;
- protected ProtocolFactory protocolFactory;
+ protected Fetcher fetcher;
protected ParserFactory parserFactory;
protected FilterFactory filterFactory;
protected HandlerFactory handlerFactory;
@@ -49,7 +48,6 @@ public abstract class AbstractDroid<T ex
public AbstractDroid(Queue<T> queue, TaskMaster<T> taskMaster) {
this.queue = queue;
this.taskMaster = taskMaster;
- this.protocolFactory = new ProtocolFactory();
this.parserFactory = new ParserFactory();
this.filterFactory = new FilterFactory();
this.handlerFactory = new HandlerFactory();
@@ -66,11 +64,26 @@ public abstract class AbstractDroid<T ex
}
@Override
+ public void finished() {
+ logger.info("FINISHED!!!");
+ }
+
+
+ @Override
public void add(T task) {
queue.add(task);
}
@Override
+ public void load(T task) throws DroidsException, IOException {
+ if (this.fetcher == null) {
+ throw new DroidsException("Fetcher not set");
+ } else {
+ this.fetcher.fetch(task);
+ }
+ }
+
+ @Override
public void parse(T task) throws DroidsException, IOException {
this.parserFactory.parse(task);
}
@@ -85,18 +98,16 @@ public abstract class AbstractDroid<T ex
return this.filterFactory.filter(task);
}
+ public void setFetcher(Fetcher fetcher) {
+ this.fetcher = fetcher;
+ }
+
public void addParsers(Parser... parsers) {
for (Parser parser : parsers) {
this.parserFactory.addParser(parser);
}
}
- public void addProtocols(Protocol... protocols) {
- for (Protocol protocol : protocols) {
- this.protocolFactory.addProtocol(protocol);
- }
- }
-
public void addHandlers(Handler... handlers) {
for (Handler handler : handlers) {
this.handlerFactory.addHandler(handler);
Modified:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
URL:
http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java?rev=1429788&r1=1429787&r2=1429788&view=diff
==============================================================================
---
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
(original)
+++
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/ContentEntity.java
Mon Jan 7 13:07:00 2013
@@ -1,5 +1,6 @@
package org.apache.droids.core;
+import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
@@ -14,6 +15,7 @@ public class ContentEntity {
public final static String CONTENT = "content";
public final static String MIME_TYPE = "mime";
public final static String CONTENT_LENGTH = "content-length";
+ public final static String LINKS = "links";
public ContentEntity() {
this.data = new HashMap<String, Object>();
@@ -27,6 +29,22 @@ public class ContentEntity {
this.data.put(key, value);
}
+ public void setContent(InputStream in) {
+ this.put(CONTENT, in);
+ }
+
+ public InputStream getContent() throws DroidsException {
+ if (this.getValue(CONTENT) != null) {
+ if (this.getValue(CONTENT) instanceof InputStream) {
+ return (InputStream)this.getValue(CONTENT);
+ } else {
+ throw new DroidsException("wrong type of content");
+ }
+ } else {
+ throw new NullPointerException();
+ }
+ }
+
public Map<String, Object> getData() {
return data;
}
Modified:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
URL:
http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java?rev=1429788&r1=1429787&r2=1429788&view=diff
==============================================================================
---
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
(original)
+++
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Droid.java
Mon Jan 7 13:07:00 2013
@@ -34,7 +34,8 @@ import java.io.IOException;
public interface Droid<T extends Task> {
/**
- * Invoke an instance of the worker used in the droid
+ * Start the droid with checking the preconditions and
+ * initializing the taskmaster with the initial tasks
*/
public void start();
@@ -56,23 +57,30 @@ public interface Droid<T extends Task> {
public Worker<T> getNewWorker();
/**
- * Parse the task
+ * Load the content of the task using the fetcher.
*
- * @param task the task
+ * @param task the task for loading the content
+ */
+ public void load(T task) throws DroidsException, IOException;
+
+ /**
+ * Parse the task with the defined Parsers.
+ *
+ * @param task the task to parse
*/
public void parse(T task) throws DroidsException, IOException;
/**
* Handle the task.
*
- * @param task
+ * @param task the task to handle
*/
public void handle(T task) throws DroidsException, IOException;
/**
* Filter the task.
*
- * @param task
+ * @param task the task to filter
* @return the task or null, if the task is not valid
*/
public T filter(T task);
Copied:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java
(from r1429330,
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java)
URL:
http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java?p2=incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java&p1=incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java&r1=1429330&r2=1429788&rev=1429788&view=diff
==============================================================================
---
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Protocol.java
(original)
+++
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/core/Fetcher.java
Mon Jan 7 13:07:00 2013
@@ -17,7 +17,6 @@
package org.apache.droids.core;
import java.io.IOException;
-import java.net.URI;
/**
@@ -26,25 +25,23 @@ import java.net.URI;
*
* @version 1.0
*/
-public interface Protocol {
+public interface Fetcher<T extends Task> {
/**
* Some protocols (like http) offer a mechanism to evaluate whether the
client
* can request a given url (in http this is the robots.txt configuration)
*
- * @param url the url to evaluate
- * @return true if we can request the url. false if we are forbidden.
- * @throws MalformedURLException
+ * @param task the task to evaluate
+ * @return true if we can request the task. false if we are forbidden.
+ * @throws IOException
*/
- boolean isAllowed(URI url) throws IOException;
+ public boolean isAllowed(T task) throws IOException;
/**
* Return the content entity represent of the url
*
- * @param url url of the stream we want to open
- * @return the content of the given url
+ * @param task the task we want to retrieve.
* @throws IOException
*/
- Task load(URI uri) throws IOException;
+ public void fetch(T task) throws IOException;
- String scheme();
}
\ No newline at end of file
Modified:
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java
URL:
http://svn.apache.org/viewvc/incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java?rev=1429788&r1=1429787&r2=1429788&view=diff
==============================================================================
---
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java
(original)
+++
incubator/droids/branches/0.2.x-cleanup/droids-core/src/main/java/org/apache/droids/handle/SysoutHandler.java
Mon Jan 7 13:07:00 2013
@@ -23,6 +23,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
+import java.util.Set;
/**
* Handler that write the stream to the sysout. Mostly added for debugging
@@ -31,8 +32,18 @@ import java.io.*;
* @version 1.0
*/
public class SysoutHandler extends WriterHandler implements Handler {
+ private Set<String> attributes;
private static final Logger logger =
LoggerFactory.getLogger(SysoutHandler.class);
+ public SysoutHandler() {
+ super();
+ }
+
+ public SysoutHandler(Set<String> attributes) {
+ super();
+ this.attributes = attributes;
+ }
+
private static void writeOutput(InputStream stream) throws IOException {
Reader reader = new InputStreamReader(stream);
Writer output = new OutputStreamWriter(System.out);
@@ -42,16 +53,18 @@ public class SysoutHandler extends Write
@Override
public void handle(Task task) throws IOException, DroidsException {
for (String key : task.getContentEntity().getData().keySet()) {
- if (task.getContentEntity().getValue(key) instanceof InputStream) {
- InputStream instream = (InputStream)
task.getContentEntity().getValue(key);
- try {
- logger.info(key + ": ");
- writeOutput(instream);
- } finally {
- instream.close();
+ if (attributes.contains(key)) {
+ if (task.getContentEntity().getValue(key) instanceof
InputStream) {
+ InputStream instream = (InputStream)
task.getContentEntity().getValue(key);
+ try {
+ logger.info(key + ": ");
+ writeOutput(instream);
+ } finally {
+ instream.close();
+ }
+ } else {
+ logger.info(key + ": " +
task.getContentEntity().getValue(key).toString());
}
- } else {
- logger.info(key + ": " +
task.getContentEntity().getValue(key).toString());
}
}
}
