Revision: 14754
http://gate.svn.sourceforge.net/gate/?rev=14754&view=rev
Author: valyt
Date: 2011-12-13 12:08:02 +0000 (Tue, 13 Dec 2011)
Log Message:
-----------
Remote Query Runner:
- logic bug fixed: on the remote side the document IDs and document scores are
always in ranked order.
- added caching for document IDs and document scores on the remote side
Modified Paths:
--------------
mimir/trunk/mimir-client/src/gate/mimir/search/RemoteQueryRunner.java
mimir/trunk/mimir-web/grails-app/controllers/gate/mimir/web/SearchController.groovy
Modified: mimir/trunk/mimir-client/src/gate/mimir/search/RemoteQueryRunner.java
===================================================================
--- mimir/trunk/mimir-client/src/gate/mimir/search/RemoteQueryRunner.java
2011-12-13 02:17:44 UTC (rev 14753)
+++ mimir/trunk/mimir-client/src/gate/mimir/search/RemoteQueryRunner.java
2011-12-13 12:08:02 UTC (rev 14754)
@@ -19,10 +19,14 @@
import gate.mimir.search.query.Binding;
import gate.mimir.tool.WebUtils;
import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
+import it.unimi.dsi.fastutil.doubles.DoubleList;
import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
import java.io.IOException;
import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.List;
@@ -46,12 +50,12 @@
protected static final String ACTION_DOC_COUNT_BIN = "documentsCountBin";
- protected static final String ACTION_DOC_ID_BIN = "documentIdBin";
+ protected static final String ACTION_DOC_IDS_BIN = "documentIdsBin";
+ protected static final String ACTION_DOC_SCORES_BIN = "documentsScoresBin";
+
protected static final String ACTION_DOC_HITS_BIN = "documentHitsBin";
- protected static final String ACTION_DOC_SCORES_BIN = "documentsScoresBin";
-
protected static final String ACTION_DOC_DATA_BIN = "documentDataBin";
protected static final String ACTION_RENDER_DOCUMENT = "renderDocument";
@@ -89,24 +93,8 @@
Thread.sleep(500);
} else {
// remote side has finished enumerating all documents
- // download all the scores in one go.
- double[] docScores = (double[]) webUtils.getObject(
- getActionBaseUrl(ACTION_DOC_SCORES_BIN), "queryId",
- URLEncoder.encode(queryId, "UTF-8"));
- if(docScores != null && docScores.length > 0) {
- documentScores = new DoubleArrayList(docScores);
- if(documentScores.size() != newDocumentsCount) {
- // malfunction
- exceptionInBackgroundThread = new RuntimeException(
- "Incorrect number of document scores from remote side: " +
- "was expecting " + newDocumentsCount + " but got " +
- documentScores.size() +"!");
- return;
- }
- } else {
- // we're not scoring
- documentScores = null;
- }
+ // download the first block of IDs and scores
+ downloadDocIdScores(0);
// ...and we're done!
documentsCount = newDocumentsCount;
}
@@ -124,20 +112,6 @@
exceptionInBackgroundThread = e;
return;
}
- } catch(ClassNotFoundException e) {
- if(failuresAllowed > 0) {
- failuresAllowed --;
- logger.error("Exception while obtaining remote document data (will
retry)", e);
- try {
- Thread.sleep(100);
- } catch(InterruptedException e1) {
- Thread.currentThread().interrupt();
- }
- } else {
- logger.error("Exception while obtaining remote document data.", e);
- exceptionInBackgroundThread = e;
- return;
- }
} catch(InterruptedException e) {
Thread.currentThread().interrupt();
logger.warn("Interrupted while waiting", e);
@@ -146,6 +120,11 @@
}
}
+ /**
+ * The size of the document block (the number of documents for which the IDs
+ * are downloaded in one operation.
+ */
+ private int docBlockSize = 1000;
/**
* A cache of MG4J {@link Document}s used for returning the hit text.
@@ -173,14 +152,14 @@
*/
private volatile int documentsCount;
- private volatile boolean closed;
-
/**
* The current number of documents. After all documents have been retrieved,
* this value is identical to {@link #documentsCount}.
*/
- private int currentDocumentsCount;
+ private volatile int currentDocumentsCount;
+ private volatile boolean closed;
+
/**
* Shared Logger
*/
@@ -193,15 +172,18 @@
* the job of any of the interactive methods to report it.
*/
private Exception exceptionInBackgroundThread;
+
+ /**
+ * The document IDs in ranking order. If ranking is not preformed, then the
+ * document IDs are in the order they are returned by the index.
+ */
+ protected IntList documentIds;
/**
- * If scoring is enabled ({@link #scorer} is not <code>null</code>), this
list
- * contains the scores for the documents found to contain hits. This list is
- * aligned to {@link #documentIds}.
+ * The document scores. This list is aligned to {@link #documentIds}.
*/
- protected DoubleArrayList documentScores;
+ protected DoubleList documentScores;
-
public RemoteQueryRunner(String remoteUrl, String queryString,
Executor threadSource, WebUtils webUtils) throws IOException {
this.remoteUrl = remoteUrl.endsWith("/") ? remoteUrl : (remoteUrl + "/");
@@ -220,8 +202,10 @@
"an unknown object type!").initCause(e);
}
- // init the cache
- documentCache = new Int2ObjectLinkedOpenHashMap<DocumentData>();
+ // init the caches
+ this.documentIds = new IntArrayList();
+ this.documentScores = new DoubleArrayList();
+ this.documentCache = new Int2ObjectLinkedOpenHashMap<DocumentData>();
// start the background action
documentsCount = -1;
@@ -245,9 +229,8 @@
throw (IOException)new IOException(
"Problem communicating with the remote index", e);
}
-
//an example URL looks like this:
-
//http://localhost:8080/mimir/bf25398f-f087-4224-bfa6-c2ef00399c04/search/hitCountBin?queryId=c4da799e-9ca2-46ae-8ded-30bdc37ad607
+
//http://localhost:8080/mimir/bf25398ff0874224/search/documentsCountBin?queryId=c4da799e-9ca2-46ae-8ded-30bdc37ad607
StringBuilder str = new StringBuilder(remoteUrl);
str.append(SERVICE_SEARCH);
str.append('/');
@@ -304,16 +287,18 @@
public int getDocumentsCurrentCount() {
return (documentsCount < 0) ? currentDocumentsCount : documentsCount;
}
-
+
/* (non-Javadoc)
* @see gate.mimir.search.QueryRunner#getDocumentID(int)
*/
@Override
public int getDocumentID(int rank) throws IndexOutOfBoundsException,
IOException {
- return webUtils.getInt(getActionBaseUrl(ACTION_DOC_ID_BIN),
- "queryId", URLEncoder.encode(queryId, "UTF-8"),
- "documentRank", Integer.toString(rank));
+ if(rank >= documentIds.size()) {
+ // we need to get more document IDs&scores
+ downloadDocIdScores(rank);
+ }
+ return documentIds.getInt(rank);
}
/* (non-Javadoc)
@@ -322,12 +307,11 @@
@Override
public double getDocumentScore(int rank) throws IndexOutOfBoundsException,
IOException {
- if(documentsCount < 0) {
- // premature call
- throw new IndexOutOfBoundsException(
- "Score requested before collection of documents has completed.");
- }
- return documentScores != null ? documentScores.get(rank) : DEFAULT_SCORE;
+ if(rank >= documentScores.size()) {
+ // we need to get more document IDs&scores
+ downloadDocIdScores(rank);
+ }
+ return documentScores.get(rank);
}
/* (non-Javadoc)
@@ -421,4 +405,40 @@
closed = true;
documentCache.clear();
}
+
+ /**
+ * Gets from the remote end point a range of document IDs and document
scores,
+ * which is guaranteed to include the document at the given rank.
+ * @param rank
+ * @throws IOException
+ */
+ protected void downloadDocIdScores(int rank) throws IOException {
+ int firstRank = documentIds.size();
+ if(firstRank != documentScores.size()) {
+ throw new IllegalStateException("Document IDs and scores out of sync.");
+ }
+ int size = rank - firstRank;
+ if(size < docBlockSize) size = docBlockSize;
+
+ int[] newDocIds;
+ double[] newDocScores;
+ try {
+ newDocIds = (int[]) webUtils.getObject(
+ getActionBaseUrl(ACTION_DOC_IDS_BIN),
+ "queryId", URLEncoder.encode(queryId, "UTF-8"),
+ "firstRank", Integer.toString(firstRank),
+ "size", Integer.toString(size));
+ documentIds.addElements(firstRank, newDocIds);
+ newDocScores = (double[]) webUtils.getObject(
+ getActionBaseUrl(ACTION_DOC_SCORES_BIN),
+ "queryId", URLEncoder.encode(queryId, "UTF-8"),
+ "firstRank", Integer.toString(firstRank),
+ "size", Integer.toString(size));
+ documentScores.addElements(firstRank, newDocScores);
+ } catch(ClassNotFoundException e) {
+ // this should really not happen (the 'class' is double)
+ throw new RuntimeException("Error communicating to remote endpoint", e);
+ }
+ }
+
}
Modified:
mimir/trunk/mimir-web/grails-app/controllers/gate/mimir/web/SearchController.groovy
===================================================================
---
mimir/trunk/mimir-web/grails-app/controllers/gate/mimir/web/SearchController.groovy
2011-12-13 02:17:44 UTC (rev 14753)
+++
mimir/trunk/mimir-web/grails-app/controllers/gate/mimir/web/SearchController.groovy
2011-12-13 12:08:02 UTC (rev 14754)
@@ -616,40 +616,110 @@
}
}
+ // protected static final String ACTION_DOC_IDS_BIN = "documentIdsBin";
/**
- * Gets the ID of a document as a serialised String value.
+ * Gets the IDs of a range of documents, in ranking order.
*/
- def documentIdBin = {
+ def documentIdsBin = {
def p = params["request"] ?: params
//get the query ID
String queryId = p["queryId"]
QueryRunner runner = searchService.getQueryRunner(queryId);
if(runner){
+ if(runner.getDocumentsCount() < 0) {
+ // premature call
+ response.sendError(HttpServletResponse.SC_NOT_FOUND,
+ "Query ID ${queryId} has not completed collecting hits; please try
later")
+ }
//get the parameters: int documentRank
- def documentRankParam = p["documentRank"]
- if (documentRankParam) {
- try {
+ def firstRankParam = p["firstRank"]
+ if (firstRankParam) {
+ def sizeParam = p["size"]
+ if(sizeParam) {
//we have all required parameters
- int documentRank = documentRankParam.toInteger()
- int docId = runner.getDocumentID(documentRank)
- new ObjectOutputStream (response.outputStream).withStream {stream ->
- stream.writeObject(Integer.toString(docId))
+ try {
+ int from = firstRankParam.toInteger()
+ int resultSize = sizeParam.toInteger()
+ int to = from + resultSize
+ if(to > runner.getDocumentsCount()) {
+ to = runner.getDocumentsCount()
+ }
+ int[] docIds = new int[(to - from)]
+ for(int rank = from; rank < to; rank++) {
+ docIds[rank] = runner.getDocumentID(rank)
+ }
+ new ObjectOutputStream (response.outputStream).withStream {stream
->
+ stream.writeObject(docIds)
+ }
+ } catch(Exception e){
+ log.warn("Error while sending document ID", e)
+ response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
+ "Error while obtaining the document ID: \"" +
+ e.getMessage() + "\"!")
}
- } catch(Exception e){
- log.warn("Error while sending document ID", e)
- response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
- "Error while obtaining the document ID: \"" +
- e.getMessage() + "\"!")
+ } else {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST,
+ "No value provided for parameter size!")
}
} else {
response.sendError(HttpServletResponse.SC_BAD_REQUEST,
- "No value provided for parameter documentRank!")
+ "No value provided for parameter firstRank!")
}
} else {
response.sendError(HttpServletResponse.SC_NOT_FOUND,
"Query ID ${queryId} not known!")
}
}
+
+ // protected static final String ACTION_DOC_SCORES_BIN =
"documentsScoresBin";
+ /**
+ * Retrieves the scores for a range of documents
+ */
+ def documentsScoresBin = {
+ def p = params["request"] ?: params
+ //get the query ID
+ String queryId = p["queryId"]
+ QueryRunner runner = searchService.getQueryRunner(queryId);
+ if(runner){
+ //get the parameters: int documentRank
+ def firstRankParam = p["firstRank"]
+ if (firstRankParam) {
+ def sizeParam = p["size"]
+ if(sizeParam) {
+ //we have all required parameters
+ try {
+ int from = firstRankParam.toInteger()
+ int resultSize = sizeParam.toInteger()
+ int to = from + resultSize
+ if(to > runner.getDocumentsCount()) {
+ to = runner.getDocumentsCount()
+ }
+ double[] docScores = new double[(to - from)]
+ for(int rank = from; rank < to; rank++) {
+ docScores[rank] = runner.getDocumentScore(rank)
+ }
+ new ObjectOutputStream (response.outputStream).withStream {stream
->
+ stream.writeObject(docScores)
+ }
+ } catch(Exception e){
+ log.warn("Error while sending document ID", e)
+ response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
+ "Error while obtaining the document ID: \"" +
+ e.getMessage() + "\"!")
+ }
+ } else {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST,
+ "No value provided for parameter size!")
+ }
+ } else {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST,
+ "No value provided for parameter firstRank!")
+ }
+ } else {
+ response.sendError(HttpServletResponse.SC_NOT_FOUND,
+ "Query ID ${queryId} not known!")
+ }
+ }
/**
* Retrieves the hits within a given result document.
@@ -686,36 +756,6 @@
}
}
- /**
- * Retrieves all the document scores
- */
- def documentsScoresBin = {
- def p = params["request"] ?: params
- //get the query ID
- String queryId = p["queryId"]
- QueryRunner runner = searchService.getQueryRunner(queryId);
- if(runner){
- try{
- int docCount = runner.getDocumentsCount()
- double[] scores = new double[docCount]
- for(int i = 0; i < docCount; i++) {
- scores[i] = runner.getDocumentScore(i)
- }
- new ObjectOutputStream (response.outputStream).withStream {stream ->
- stream.writeObject(scores)
- }
- }catch(Exception e){
- log.warn("Error while sending document scores", e)
- response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
- "Error while obtaining the scores: \"" + e.getMessage() + "\"!")
- }
- } else{
- response.sendError(HttpServletResponse.SC_NOT_FOUND,
- "Query ID ${queryId} not known!")
- }
- }
-
- //
// protected static final String ACTION_DOC_DATA_BIN = "documentDataBin";
/**
* Gets the document data (title, URI, text) for a given document. The
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Systems Optimization Self Assessment
Improve efficiency and utilization of IT resources. Drive out cost and
improve service delivery. Take 5 minutes to use this Systems Optimization
Self Assessment. http://www.accelacomm.com/jaw/sdnl/114/51450054/
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
