Revision: 17307
http://sourceforge.net/p/gate/code/17307
Author: valyt
Date: 2014-02-14 11:47:27 +0000 (Fri, 14 Feb 2014)
Log Message:
-----------
Enough thread knitting for a medium size cardigan, but no more public methods
that give access to index internals.
Modified Paths:
--------------
mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicIndex.java
mimir/branches/5.0/mimir-core/src/gate/mimir/index/GATEDocument.java
Modified: mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
2014-02-13 19:26:05 UTC (rev 17306)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
2014-02-14 11:47:27 UTC (rev 17307)
@@ -45,6 +45,8 @@
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeSet;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
@@ -95,6 +97,19 @@
public static final int DEFAULT_INDEXING_QUEUE_SIZE = 30;
/**
+ * Special value used to indicate that the index is closing and there will
be
+ * no more sync tasks to process (an END_OF_QUEUE value for
+ * {@link #syncRequests}).
+ */
+ protected final static Future<Long> NO_MORE_TASKS = new FutureTask<Long>(
+ new Callable<Long>() {
+ @Override
+ public Long call() throws Exception {
+ return 0l;
+ }
+ });
+
+ /**
* How many occurrences to be accumulated in RAM before a new tail batch is
* written to disk.
*/
@@ -103,11 +118,15 @@
private static final Logger logger = Logger.getLogger(MimirIndex.class);
/**
- * A {@link Runnable} that performs various index maintenance in a separate
- * thread. The principal role is to collect the documents from the
- * sub-indexers and delete them from GATE.
+ * A {@link Runnable} used in a background thread to perform various index
+ * maintenance tasks:
+ * <ul>
+ * <li>check that the documents are being returned from the sub-indexers
+ * in the same order as they were submitted for indexing;</li>
+ * <li>delete indexed documents from GATE</li>
+ * </ul>
*/
- protected class IndexMaintenanceRunner implements Runnable{
+ protected class IndexMaintenanceRunner implements Runnable {
public void run(){
boolean finished = false;
while(!finished){
@@ -127,6 +146,7 @@
}
//we obtained the same document from all the sub-indexers
if(currentDocument != GATEDocument.END_OF_QUEUE) {
+ occurrencesInRam += currentDocument.getOccurrences();
// let's delete it
logger.debug("Deleting document "
+ currentDocument.getDocument().getName());
@@ -146,59 +166,65 @@
}
}
-
- private class WriteDeletedDocsTask extends TimerTask {
- public void run() {
- synchronized(maintenanceTimer) {
- File delFile = new File(indexDirectory,
DELETED_DOCUMENT_IDS_FILE_NAME);
- if(delFile.exists()) {
- delFile.delete();
- }
- try{
- logger.debug("Writing deleted documents set");
- ObjectOutputStream oos = new ObjectOutputStream(
- new GZIPOutputStream(
- new BufferedOutputStream(
- new FileOutputStream(delFile))));
- oos.writeObject(deletedDocumentIds);
- oos.flush();
- oos.close();
- logger.debug("Writing deleted documents set completed.");
- }catch (IOException e) {
- logger.error("Exception while writing deleted documents set", e);
- }
- }
- }
- }
-
/**
- * {@link TimerTask} used to regularly dump the latest document to an on-disk
- * batch, allowing them to become searchable.
+ * A {@link Runnable} used in a background thread to perform various index
+ * maintenance tasks:
+ * <ul>
+ * <li>update the {@link MimirIndex#occurrencesInRam} value by subtracting
+ * the occurrence counts for all the batches that have recently been
written
+ * to disk. It finds these by consuming the {@link Future}s in
+ * {@link MimirIndex#syncRequests}.</li>
+ * <li>Compact the index when too many on-disk batches have been
created.</li>
+ * </ul>
*/
- protected class DumpToDiskTask extends TimerTask {
+ protected class IndexMaintenanceRunner2 implements Runnable {
+
@Override
public void run() {
- if(occurrencesInRam > 0) {
- requestDumpToDisk();
+ try {
+ Future<Long> aTask = syncRequests.take();
+ while(aTask != NO_MORE_TASKS) {
+ try {
+ occurrencesInRam -= aTask.get();
+ if(syncRequests.isEmpty()) {
+ // latest dump finished: compact index if needed;
+ boolean compactNeeded = false;
+ for(AtomicIndex aSubIndex : subIndexes) {
+ if(aSubIndex.getBatchCount() >
indexConfig.getMaximumBatches()) {
+ compactNeeded = true;
+ break;
+ }
+ }
+ if(compactNeeded) compactIndexSync();
+ }
+ } catch(ExecutionException e) {
+ // a sync request has failed. The index may be damaged, so we will
+ // close it to avoid further damage.
+ logger.error("A sync-to-disk request has failed. Closing index "
+ + "to avoid further damage.", e);
+ try {
+ close();
+ } catch(IOException e1) {
+ logger.error("A further error was generated while attmepting "
+ + "to close index.", e1);
+ }
+ }
+ }
+ } catch(InterruptedException e) {
+ if(closed) {
+ // we've just been told to give up: yay, holiday!
+ } else {
+ Thread.currentThread().interrupt();
+ }
}
- // and re-schedule
- synchronized(maintenanceTimer) {
- dumpToDiskTask = new DumpToDiskTask();
- maintenanceTimer.schedule(dumpToDiskTask,
indexConfig.getTimeBetweenBatches());
- }
}
- }
-
- /**
- * {@link TimerTask} used to regularly check if the index needs compacting,
- * and to perform the compaction if so.
- */
- protected class CompactIndexTask extends TimerTask {
-
-
- @Override
- public void run() {
+ /**
+ * Request the index compaction, and waits for all the operations to
+ * complete.
+ * @throws InterruptedException
+ */
+ protected void compactIndexSync() throws InterruptedException {
List<Future<Void>> futures = requestCompactIndex();
try {
compactDocumentCollection();
@@ -211,6 +237,7 @@
} catch(InterruptedException e1) {
logger.error("Received interrupt request while closing "
+ "operation in progress", e);
+ Thread.currentThread().interrupt();
} catch(IOException e1) {
logger.error("Further IO exception while closing index.", e1);
}
@@ -222,9 +249,10 @@
// we were interrupted while waiting for a compacting operation
logger.error("Received interrupt request while compacting "
+ "operation in progress", e);
+ Thread.currentThread().interrupt();
} catch(ExecutionException e) {
- logger.error("Execution exception while comapting the index. "
- + "Index may now be corrupted, closing it to avoid further
damage", e);
+ logger.error("Execution exception while compacting the index. Index "
+ + "may now be corrupted, closing it to avoid further damage", e);
try {
close();
} catch(InterruptedException e1) {
@@ -234,11 +262,53 @@
logger.error("Further IO exception while closing index.", e1);
}
}
+ }
+ }
+ }
+
+
+ private class WriteDeletedDocsTask extends TimerTask {
+ public void run() {
+ synchronized(maintenanceTimer) {
+ File delFile = new File(indexDirectory,
DELETED_DOCUMENT_IDS_FILE_NAME);
+ if(delFile.exists()) {
+ delFile.delete();
+ }
+ try{
+ logger.debug("Writing deleted documents set");
+ ObjectOutputStream oos = new ObjectOutputStream(
+ new GZIPOutputStream(
+ new BufferedOutputStream(
+ new FileOutputStream(delFile))));
+ oos.writeObject(deletedDocumentIds);
+ oos.flush();
+ oos.close();
+ logger.debug("Writing deleted documents set completed.");
+ }catch (IOException e) {
+ logger.error("Exception while writing deleted documents set", e);
+ }
}
}
-
}
+
/**
+ * {@link TimerTask} used to regularly dump the latest document to an on-disk
+ * batch, allowing them to become searchable.
+ */
+ protected class DumpToDiskTask extends TimerTask {
+ @Override
+ public void run() {
+ if(occurrencesInRam > 0) {
+ try {
+ requestSyncToDisk();
+ } catch(InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
+
+ /**
* The {@link IndexConfig} used for this index.
*/
protected IndexConfig indexConfig;
@@ -261,17 +331,21 @@
/**
* The thread used to clean-up GATE documents after they have been indexed.
*/
- protected Thread documentsCollectorThread;
+ protected Thread maintenanceThread;
+ /**
+ * Background thread used to subtract occurrence counts for batches that have
+ * recently been dumped to disk.
+ */
+ protected Thread maintenanceThread2;
+
protected volatile boolean closed = false;
/**
- * A flag used to store the number of dump-to-disk operations that have
- * already been requested.
- * This is used to avoid repeatedly asking sub-indexes to dump while we've
- * over the limit for occurrences.
+ * A list of futures representing sync-to-disk operations currently
+ * in-progress in all of the sub-indexes.
*/
- protected volatile int dumpsRequested;
+ protected BlockingQueue<Future<Long>> syncRequests;
/**
* The set of IDs for the documents marked as deleted.
@@ -296,7 +370,7 @@
* documents become searcheable after at most {@link #timeBetweenBatches} #
* milliseconds.
*/
- private volatile transient DumpToDiskTask dumpToDiskTask;
+ private volatile transient DumpToDiskTask syncToDiskTask;
/**
* The token indexes, in the order they are listed in the {@link
#indexConfig}.
@@ -320,9 +394,8 @@
* The total number of occurrences in all sub-indexes that have not yet been
* written to disk.
*/
- protected long occurrencesInRam;
+ protected volatile long occurrencesInRam;
-
/**
* The {@link QueryEngine} used to run searches on this index.
*/
@@ -411,12 +484,20 @@
mentionIndexes.length);
occurrencesInRam = 0;
- dumpsRequested = 0;
+ syncRequests = new LinkedBlockingQueue<Future<Long>>();
// start the documents collector thread
- documentsCollectorThread = new Thread(new IndexMaintenanceRunner());
- documentsCollectorThread.start();
-
+ maintenanceThread = new Thread(new IndexMaintenanceRunner(),
+ indexDirectory.getAbsolutePath() + " index maintenance");
+ maintenanceThread.start();
+
+ // start the occurrences subtractor thread
+ maintenanceThread2 = new Thread(
+ new IndexMaintenanceRunner2(),
+ indexDirectory.getAbsolutePath() + " Occurrences subtractor");
+ maintenanceThread2.setPriority(Thread.MIN_PRIORITY);
+ maintenanceThread2.start();
+
// #####################
// Prepare for searching
// #####################
@@ -427,8 +508,9 @@
// #####################
maintenanceTimer = new Timer("MÃmir index maintenance timer");
synchronized(maintenanceTimer) {
- dumpToDiskTask = new DumpToDiskTask();
- maintenanceTimer.schedule(dumpToDiskTask,
+ syncToDiskTask = new DumpToDiskTask();
+ maintenanceTimer.schedule(syncToDiskTask,
+ indexConfig.getTimeBetweenBatches(),
indexConfig.getTimeBetweenBatches());
}
// open the zipped document collection
@@ -449,8 +531,8 @@
// check if we need to write a new batch:
// we have too many occurrences and
// there are no outstanding batch writing operations
- if( occurrencesInRam > occurrencesPerBatch && dumpsRequested <= 0) {
- requestDumpToDisk();
+ if( occurrencesInRam > occurrencesPerBatch && syncRequests.isEmpty()) {
+ requestSyncToDisk();
}
GATEDocument gDocument = new GATEDocument(document, indexConfig);
@@ -467,24 +549,24 @@
* background threads (one for each sub-index) at the earliest opportunity.
* @return a list of futures that can be used to find out when the operation
* has completed.
+ * @throws InterruptedException if the current thread has been interrupted
+ * while trying to queue the sync request.
*/
- public List<Future<Void>> requestDumpToDisk() {
- List<Future<Void>> futures = new ArrayList<Future<Void>>();
- synchronized(subIndexes) {
- for(AtomicIndex aSubIndex : subIndexes) {
- futures.add(aSubIndex.requestDumpToDisk());
- dumpsRequested++;
+ public List<Future<Long>> requestSyncToDisk() throws InterruptedException {
+ List<Future<Long>> futures = new ArrayList<Future<Long>>();
+ if(syncRequests.isEmpty()) {
+ synchronized(subIndexes) {
+ for(AtomicIndex aSubIndex : subIndexes) {
+ Future<Long> task = aSubIndex.requestSyncToDisk();
+ futures.add(task);
+ syncRequests.put(task);
+ }
}
+ } else {
+ // sync already in progress: instead of causing a new sync, notify caller
+ // when current operation completes
+ futures.addAll(syncRequests);
}
-
- synchronized(maintenanceTimer) {
- if(dumpToDiskTask != null) {
- dumpToDiskTask.cancel();
- }
- dumpToDiskTask = new DumpToDiskTask();
- maintenanceTimer.schedule(dumpToDiskTask,
- indexConfig.getTimeBetweenBatches());
- }
return futures;
}
@@ -494,8 +576,10 @@
* background threads (one for each sub-index) at the earliest opportunity.
* @return a list of futures that can be used to find out when the operation
* has completed.
+ * @throws InterruptedException if the current thread has been interrupted
+ * while trying to queue the compaction request.
*/
- public List<Future<Void>> requestCompactIndex() {
+ public List<Future<Void>> requestCompactIndex() throws InterruptedException {
List<Future<Void>> futures = new ArrayList<Future<Void>>();
synchronized(subIndexes) {
for(AtomicIndex aSubIndex : subIndexes) {
@@ -510,46 +594,6 @@
}
/**
- * Notifies this index that more occurrences have been stored in RAM by one
of
- * its sub-indexes.
- *
- * NOTE: this method is only public as an implementation detail. Do
- * <strong>not</strong> call this method unless your name is AtomicIndex!
- *
- * @param occurrences
- */
- synchronized public void addOccurrences(long occurrences) {
- occurrencesInRam += occurrences;
- }
-
- /**
- * Notifies this index that some occurrences have been dumped from RAM to
disk
- * by one of its sub-indexes.
- *
- * NOTE: this method is only public as an implementation detail. Do
- * <strong>not</strong> call this method unless your name is AtomicIndex!
- *
- * @param occurrences
- */
- synchronized public void subtractOccurrences(long occurrences) {
- occurrencesInRam -= occurrences;
- dumpsRequested--;
- if(dumpsRequested == 0) {
- // latest dump finished: compact index if needed;
- for(AtomicIndex aSubIndex : subIndexes) {
- if(aSubIndex.getBatchCount() > indexConfig.getMaximumBatches()) {
- // do the compaction on the timer thread
- synchronized(maintenanceTimer) {
- maintenanceTimer.schedule(new CompactIndexTask(), 100);
- }
- break;
- }
- }
- }
- }
-
-
- /**
* Called by the first token indexer when a new document has been indexed
* to ask the main index to save the necessary zip collection data
* @param gDocument
@@ -586,15 +630,15 @@
}
// explicitly call it one last time
new WriteDeletedDocsTask().run();
- if(dumpToDiskTask != null) {
- dumpToDiskTask.cancel();
- }
maintenanceTimer.cancel();
}
// wait for indexing to end
- documentsCollectorThread.join();
+ maintenanceThread.join();
+ syncRequests.put(NO_MORE_TASKS);
+ maintenanceThread2.join();
+
// close the document collection
documentCollection.close();
// write the config file
@@ -655,11 +699,12 @@
if(indexConfig.getTimeBetweenBatches() != timeBetweenBatches) {
indexConfig.setTimeBetweenBatches(timeBetweenBatches);
synchronized(maintenanceTimer) {
- if(dumpToDiskTask != null) {
- dumpToDiskTask.cancel();
+ if(syncToDiskTask != null) {
+ syncToDiskTask.cancel();
}
- dumpToDiskTask = new DumpToDiskTask();
- maintenanceTimer.schedule(dumpToDiskTask, timeBetweenBatches);
+ syncToDiskTask = new DumpToDiskTask();
+ maintenanceTimer.schedule(syncToDiskTask, timeBetweenBatches,
+ timeBetweenBatches);
}
}
}
Modified: mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicIndex.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicIndex.java
2014-02-13 19:26:05 UTC (rev 17306)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/index/AtomicIndex.java
2014-02-14 11:47:27 UTC (rev 17307)
@@ -88,6 +88,7 @@
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
+import java.util.concurrent.RunnableFuture;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.log4j.Logger;
@@ -113,36 +114,6 @@
public abstract class AtomicIndex implements Runnable {
/**
- * A custom implementation of Future, allowing us to execute the work in the
- * indexing thread, and report its result.
- *
- * @param <T>
- */
- protected static class CustomFuture<T> extends FutureTask<T> {
-
- public CustomFuture(Callable<T> operation) {
- super(operation);
- }
-
- @Override
- protected void done() {
- super.done();
- }
-
- @Override
- protected void set(T v) {
- super.set(v);
- }
-
- @Override
- protected void setException(Throwable t) {
- super.setException(t);
- }
-
-
- }
-
- /**
* A callable that does nothing. This is used to produce instances of
* {@link CustomFuture} which are only used to wait for the completion of an
* operation that returns nothing.
@@ -647,7 +618,7 @@
* A marker value that gets queued to indicate a request to combine all the
* on-disk batches into a new head.
*/
- private static final GATEDocument COMPRESS_INDEX = new GATEDocument(){};
+ private static final GATEDocument COMPACT_INDEX = new GATEDocument(){};
private static Logger logger = Logger.getLogger(AtomicIndex.class);
@@ -786,7 +757,7 @@
* performed on the indexing thread at the first opportunity. At that point
* this future will complete, and the value will be set back to null.
*/
- protected CustomFuture<Void> indexCompressionRequested;
+ protected RunnableFuture<Void> compactIndexTask;
/**
* If a request was made to write the in-RAM index data to disk this value
@@ -794,7 +765,7 @@
* thread at the first opportunity. At that point the Future will complete,
* and the value will be set back to null.
*/
- protected CustomFuture<Void> tailWriteRequested;
+ protected RunnableFuture<Long> batchWriteTask;
/**
* Creates a new AtomicIndex
@@ -940,11 +911,12 @@
* Writes all the data currently stored in RAM to a new index batch.
The first
* batch is the head index, all other batches are tail indexes.
* @throws IOException
- * @throws IndexException
+ * @throws IndexException
+ * @return the number of occurrences written to disk
*/
- protected void writeCurrentBatch() throws IOException, IndexException {
- if(documentsInRAM == 0) return;
-
+ protected long writeCurrentBatch() throws IOException, IndexException {
+ if(documentsInRAM == 0) return 0;
+
// find the name for the new tail
int tailNo = -1;
File headDir = new File(indexDirectory, HEAD_FILE_NAME);
@@ -1063,7 +1035,7 @@
writeDirectIndex(newTailDir);
}
// update parent
- parent.subtractOccurrences(occurrencesInRAM);
+ long res = occurrencesInRAM;
// clear out internal state, in preparation for the next tail
newBatch();
@@ -1082,14 +1054,7 @@
throw new IndexException("Could not open the index just written to " +
mg4jBasename , e);
}
-
- // notify "listeners"
- synchronized(this) {
- if(tailWriteRequested != null) {
- tailWriteRequested.done();
- tailWriteRequested = null;
- }
- }
+ return res;
}
/**
@@ -1366,12 +1331,6 @@
throw new IndexException("Cold not rename head at " +
headDir.getAbsolutePath() + " to " + headDirOld);
}
-
- // notify "listeners", while we're still inside the shynchronized block
- if(indexCompressionRequested != null) {
- indexCompressionRequested.done();
- indexCompressionRequested = null;
- }
}
}
@@ -1481,31 +1440,45 @@
/**
* Instructs this index to dump to disk all the in-RAM index data at
the fist
* opportunity.
- * @return a future value that, upon completion, will return the number
of
+ * @return a {@link Future} value that, upon completion, will return
the number of
* occurrences written to disk.
+ * @throws InterruptedException if this thread is interrupted while
trying to
+ * queue the dump request.
*/
- public synchronized Future<Void> requestDumpToDisk() {
- if(tailWriteRequested == null) {
- tailWriteRequested = new CustomFuture<Void>(noOpVoid);
+ public synchronized Future<Long> requestSyncToDisk() throws
InterruptedException {
+ if(batchWriteTask == null) {
+ batchWriteTask = new FutureTask<Long>(new Callable<Long>() {
+ @Override
+ public Long call() throws Exception {
+ return writeCurrentBatch();
+ }
+ });
+ inputQueue.put(DUMP_BATCH);
}
- try {
- inputQueue.put(DUMP_BATCH);
- } catch(InterruptedException e) {
- tailWriteRequested.setException(e);
- }
- return tailWriteRequested;
+ return batchWriteTask;
}
- public synchronized Future<Void> requestCompactIndex() {
- if(indexCompressionRequested == null) {
- indexCompressionRequested = new CustomFuture<Void>(noOpVoid);
+ /**
+ * Requests this atomic index to compact its on-disk batches into a
single
+ * batch.
+ *
+ * @return a {@link Future} which can be used to find out when the
compaction
+ * operation has completed.
+ * @throws InterruptedException if this thread is interrupted while
trying to
+ * queue the compaction request.
+ */
+ public synchronized Future<Void> requestCompactIndex() throws
InterruptedException {
+ if(compactIndexTask == null) {
+ compactIndexTask = new FutureTask<Void>(new Callable<Void>(){
+ @Override
+ public Void call() throws Exception {
+ compactIndex();
+ return null;
+ }
+ });
+ inputQueue.put(COMPACT_INDEX);
}
- try {
- inputQueue.put(COMPRESS_INDEX);
- } catch(InterruptedException e) {
- indexCompressionRequested.setException(e);
- }
- return indexCompressionRequested;
+ return compactIndexTask;
}
/**
@@ -1604,32 +1577,35 @@
if(aDocument != GATEDocument.END_OF_QUEUE) {
if(aDocument == DUMP_BATCH) {
//dump batch was requested
- writeCurrentBatch();
- } else if(aDocument == COMPRESS_INDEX) {
+ if(batchWriteTask != null){
+ batchWriteTask.run();
+ }
+ synchronized(this) {
+ batchWriteTask = null;
+ }
+ } else if(aDocument == COMPACT_INDEX) {
// compress index was requested
- try {
- compactIndex();
- } catch(Exception e) {
- if(indexCompressionRequested != null) {
- indexCompressionRequested.setException(e);
- }
- throw e;
- }
+ if(compactIndexTask != null) {
+ compactIndexTask.run();
+ }
+ synchronized(this) {
+ compactIndexTask = null;
+ }
} else {
try {
long occurencesBefore = occurrencesInRAM;
processDocument(aDocument);
- parent.addOccurrences(occurrencesInRAM - occurencesBefore);
+ aDocument.addOccurrences(occurrencesInRAM - occurencesBefore);
} catch(Throwable e) {
logger.error("Problem while indexing document!", e);
- }
+ }
}
} else {
// close down
writeCurrentBatch();
flush();
}
- if(aDocument != DUMP_BATCH && aDocument != COMPRESS_INDEX) {
+ if(aDocument != DUMP_BATCH && aDocument != COMPACT_INDEX) {
outputQueue.put(aDocument);
}
} while(aDocument != GATEDocument.END_OF_QUEUE);
Modified: mimir/branches/5.0/mimir-core/src/gate/mimir/index/GATEDocument.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/index/GATEDocument.java
2014-02-13 19:26:05 UTC (rev 17306)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/index/GATEDocument.java
2014-02-14 11:47:27 UTC (rev 17307)
@@ -58,6 +58,12 @@
private static long documentID = 0;
/**
+ * The number of occurrences (in all sub-indexes) generated as a result of
+ * indexing this document.
+ */
+ private long occurrences = 0;
+
+ /**
* An MG4J word reader for this document.
*/
private class GATEDocumentWordReader implements WordReader{
@@ -259,6 +265,30 @@
return uri;
}
+ /**
+ * Notifies this GATEDocument that some more index occurrences were produced
+ * in the process of indexing it.
+ *
+ * This method is synchronized because the same GATEDocument instance is
being
+ * indexed in parallel by multiple sub-indexers.
+ *
+ * @param newOccurrences the number of new occurrences generated
+ */
+ public synchronized void addOccurrences(long newOccurrences) {
+ occurrences += newOccurrences;
+ }
+
+ /**
+ * Returns the number of index occurrences that the indexing of this
+ * GATEDocument has generated. This value is only correct after the document
+ * has been indexed by all sub-indexers.
+ *
+ * @return the number of occurrences.
+ */
+ public long getOccurrences() {
+ return occurrences;
+ }
+
/* (non-Javadoc)
* @see it.unimi.dsi.mg4j.document.Document#wordReader(int)
*/
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Android apps run on BlackBerry 10
Introducing the new BlackBerry 10.2.1 Runtime for Android apps.
Now with support for Jelly Bean, Bluetooth, Mapview and more.
Get your Android app in front of a whole new audience. Start now.
http://pubads.g.doubleclick.net/gampad/clk?id=124407151&iu=/4140/ostg.clktrk
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
