[gate-cvs] SF.net SVN: gate:[18954] mimir/trunk/mimir-client/src/gate/mimir/index

[ian_roberts]

Revision: 18954
          http://sourceforge.net/p/gate/code/18954
Author:   ian_roberts
Date:     2015-10-20 21:13:20 +0000 (Tue, 20 Oct 2015)
Log Message:
-----------
Added batching support to Mimir indexing PR.  Also improved the test in 
MimirConnector for checking whether there are any outstanding documents to 
send, so we don't make un-necessary HTTP connections to Mimir.

Modified Paths:
--------------
    mimir/trunk/mimir-client/src/gate/mimir/index/MimirConnector.java
    mimir/trunk/mimir-client/src/gate/mimir/index/MimirIndexingPR.java

Modified: mimir/trunk/mimir-client/src/gate/mimir/index/MimirConnector.java
===================================================================
--- mimir/trunk/mimir-client/src/gate/mimir/index/MimirConnector.java   
2015-10-18 11:59:01 UTC (rev 18953)
+++ mimir/trunk/mimir-client/src/gate/mimir/index/MimirConnector.java   
2015-10-20 21:13:20 UTC (rev 18954)
@@ -84,6 +84,11 @@
   private int connectionInterval = -1;
   
   /**
+   * How many documents have been buffered since the last connection.
+   */
+  private int docsSinceLastConnection = 0;
+
+  /**
    * The URL of the mimir index that is to receive the document.  This would 
    * typically be of the form 
    * <code>http://server:port/mimir/&lt;index UUID&gt;/</code>.
@@ -145,6 +150,7 @@
     synchronized(this) {
       if(doc != null){
         objectOutputStream.writeUnshared(doc);
+        docsSinceLastConnection++;
       }
       if(byteBuffer.size() > BYTE_BUFFER_SIZE) {
         writeBuffer(); // this will also empty (reset) the buffer
@@ -170,7 +176,7 @@
    * @throws IOException 
    */
   protected synchronized void writeBuffer() throws IOException {
-    if(byteBuffer.size() > 0) {
+    if(docsSinceLastConnection > 0) {
       // first phase - call the indexUrl action to find out where to post the
       // data
       StringBuilder indexURLString = new 
StringBuilder(indexURL.toExternalForm());
@@ -194,6 +200,7 @@
       } finally {
         byteBuffer.reset();
         objectOutputStream = new ObjectOutputStream(byteBuffer);
+        docsSinceLastConnection = 0;
       }
     }
     

Modified: mimir/trunk/mimir-client/src/gate/mimir/index/MimirIndexingPR.java
===================================================================
--- mimir/trunk/mimir-client/src/gate/mimir/index/MimirIndexingPR.java  
2015-10-18 11:59:01 UTC (rev 18953)
+++ mimir/trunk/mimir-client/src/gate/mimir/index/MimirIndexingPR.java  
2015-10-20 21:13:20 UTC (rev 18954)
@@ -23,6 +23,8 @@
 import gate.mimir.tool.WebUtils;
 import gate.util.GateRuntimeException;
 
+import org.apache.log4j.Logger;
+
 import java.net.URL;
 
 
@@ -35,12 +37,16 @@
 
   private static final long serialVersionUID = 3291873032301133998L;
 
+  private static final Logger log = Logger.getLogger(MimirIndexingPR.class);
+
   private URL mimirIndexUrl;
   
   private String mimirUsername;
   
   private String mimirPassword;
 
+  private Integer connectionInterval;
+
   protected MimirConnector mimirConnector;
   
   
@@ -64,8 +70,7 @@
   @RunTime
   public void setMimirUsername(String mimirUsername) {
     this.mimirUsername = mimirUsername;
-    // invalidate the connector
-    this.mimirConnector = null;
+    closeConnector();
   }
 
   public String getMimirPassword() {
@@ -77,17 +82,36 @@
   @RunTime
   public void setMimirPassword(String mimirPassword) {
     this.mimirPassword = mimirPassword;
-    // invalidate the connector    
-    this.mimirConnector = null;
+    closeConnector();
   }
 
+  public Integer getConnectionInterval() {
+    return connectionInterval;
+  }
+
+  @CreoleParameter(comment="Interval between connections to the Mímir server.  
-1 causes each document "
+      + "to be sent immediately, positive values cause documents to be 
buffered and sent to the server in "
+      + "batches, which is generally much more efficient, particularly when 
the documents are short.",
+      defaultValue = "1000")
+  @Optional
+  @RunTime
+  public void setConnectionInterval(Integer connectionInterval) {
+    this.connectionInterval = connectionInterval;
+    closeConnector();
+  }
+
   @Override
   public void cleanup() {
-    try {
-      mimirConnector.close();
-    } catch(Exception e) {
-      throw new GateRuntimeException("Execption while closing Mímir 
connector", 
-          e);
+    closeConnector();
+  }
+
+  protected void closeConnector() {
+    if(mimirConnector != null) {
+      try {
+        mimirConnector.close();
+      } catch(Exception e) {
+        log.warn("Execption while closing Mímir connector", e);
+      }
     }
     mimirConnector = null;
   }
@@ -103,6 +127,9 @@
         } else {
           mimirConnector = new MimirConnector(mimirIndexUrl);  
         }
+        if(connectionInterval != null) {
+          mimirConnector.setConnectionInterval(connectionInterval.intValue());
+        }
       }
       mimirConnector.sendToMimir(getDocument(), null);
     } catch(Exception e) {

This was sent by the SourceForge.net collaborative development platform, the 
world's largest Open Source development site.


------------------------------------------------------------------------------
_______________________________________________
GATE-cvs mailing list
GATE-cvs@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/gate-cvs