Revision: 17306
http://sourceforge.net/p/gate/code/17306
Author: valyt
Date: 2014-02-13 19:26:05 +0000 (Thu, 13 Feb 2014)
Log Message:
-----------
Re-working the multi-threading for dumping batches
Saving for the day. Will get back to this tomorrow.
Modified Paths:
--------------
mimir/branches/5.0/mimir-core/src/gate/mimir/IndexConfig.java
mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
mimir/branches/5.0/mimir-core/src/gate/mimir/index/DocumentCollection.java
Modified: mimir/branches/5.0/mimir-core/src/gate/mimir/IndexConfig.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/IndexConfig.java
2014-02-13 18:50:40 UTC (rev 17305)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/IndexConfig.java
2014-02-13 19:26:05 UTC (rev 17306)
@@ -380,7 +380,7 @@
* Gets the maximum number of on-disk index batches before an index
compaction
* is triggered.
*
- * Defaults to {@link #DEFAULT_MAXIMUM_BATCHES}.
+ * Defaults to {@value #DEFAULT_MAXIMUM_BATCHES}.
* @return
*/
public int getMaximumBatches() {
Modified: mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
2014-02-13 18:50:40 UTC (rev 17305)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/MimirIndex.java
2014-02-13 19:26:05 UTC (rev 17306)
@@ -195,25 +195,36 @@
*/
protected class CompactIndexTask extends TimerTask {
- public static final int SCHEDULE_INTERVAL = 10 * 60 * 1000;
+
@Override
public void run() {
- boolean shouldCompact = false;
- for(AtomicIndex aSubIndex : subIndexes) {
- if(aSubIndex.getBatchCount() > indexConfig.getMaximumBatches()) {
- shouldCompact = true;
- break;
+ List<Future<Void>> futures = requestCompactIndex();
+ try {
+ compactDocumentCollection();
+ } catch(Exception e) {
+ logger.error("Error while compacting document collection. "
+ + "Index is now invalid. Closing index to avoid further damage.",
+ e);
+ try {
+ close();
+ } catch(InterruptedException e1) {
+ logger.error("Received interrupt request while closing "
+ + "operation in progress", e);
+ } catch(IOException e1) {
+ logger.error("Further IO exception while closing index.", e1);
}
}
- if(shouldCompact) {
- List<Future<Void>> futures = requestCompactIndex();
+ for(Future<Void> f : futures){
try {
- compactDocumentCollection();
- } catch(Exception e) {
- logger.error("Error while compacting document collection. "
- + "Index is now invalid. Closing index to avoid further damage.",
- e);
+ f.get();
+ } catch(InterruptedException e) {
+ // we were interrupted while waiting for a compacting operation
+ logger.error("Received interrupt request while compacting "
+ + "operation in progress", e);
+ } catch(ExecutionException e) {
+ logger.error("Execution exception while comapting the index. "
+ + "Index may now be corrupted, closing it to avoid further
damage", e);
try {
close();
} catch(InterruptedException e1) {
@@ -223,30 +234,7 @@
logger.error("Further IO exception while closing index.", e1);
}
}
- for(Future<Void> f : futures){
- try {
- f.get();
- } catch(InterruptedException e) {
- // we were interrupted while waiting for a compacting operation
- logger.error("Received interrupt request while compacting "
- + "operation in progress", e);
- } catch(ExecutionException e) {
- logger.error("Execution exception while comapting the index. "
- + "Index may now be corrupted, closing it to avoid further
damage", e);
- try {
- close();
- } catch(InterruptedException e1) {
- logger.error("Received interrupt request while closing "
- + "operation in progress", e);
- } catch(IOException e1) {
- logger.error("Further IO exception while closing index.", e1);
- }
- }
- }
}
- synchronized(maintenanceTimer) {
- maintenanceTimer.schedule(new CompactIndexTask(), SCHEDULE_INTERVAL);
- }
}
}
@@ -442,8 +430,6 @@
dumpToDiskTask = new DumpToDiskTask();
maintenanceTimer.schedule(dumpToDiskTask,
indexConfig.getTimeBetweenBatches());
- maintenanceTimer.schedule(new CompactIndexTask(),
- CompactIndexTask.SCHEDULE_INTERVAL);
}
// open the zipped document collection
documentCollection = new DocumentCollection(indexDirectory);
@@ -527,6 +513,9 @@
* Notifies this index that more occurrences have been stored in RAM by one
of
* its sub-indexes.
*
+ * NOTE: this method is only public as an implementation detail. Do
+ * <strong>not</strong> call this method unless your name is AtomicIndex!
+ *
* @param occurrences
*/
synchronized public void addOccurrences(long occurrences) {
@@ -537,13 +526,29 @@
* Notifies this index that some occurrences have been dumped from RAM to
disk
* by one of its sub-indexes.
*
+ * NOTE: this method is only public as an implementation detail. Do
+ * <strong>not</strong> call this method unless your name is AtomicIndex!
+ *
* @param occurrences
*/
synchronized public void subtractOccurrences(long occurrences) {
occurrencesInRam -= occurrences;
dumpsRequested--;
- }
+ if(dumpsRequested == 0) {
+ // latest dump finished: compact index if needed;
+ for(AtomicIndex aSubIndex : subIndexes) {
+ if(aSubIndex.getBatchCount() > indexConfig.getMaximumBatches()) {
+ // do the compaction on the timer thread
+ synchronized(maintenanceTimer) {
+ maintenanceTimer.schedule(new CompactIndexTask(), 100);
+ }
+ break;
+ }
+ }
+ }
+ }
+
/**
* Called by the first token indexer when a new document has been indexed
* to ask the main index to save the necessary zip collection data
Modified:
mimir/branches/5.0/mimir-core/src/gate/mimir/index/DocumentCollection.java
===================================================================
--- mimir/branches/5.0/mimir-core/src/gate/mimir/index/DocumentCollection.java
2014-02-13 18:50:40 UTC (rev 17305)
+++ mimir/branches/5.0/mimir-core/src/gate/mimir/index/DocumentCollection.java
2014-02-13 19:26:05 UTC (rev 17306)
@@ -528,7 +528,7 @@
* @throws IOException
* @throws IndexException
*/
- public void compact() throws ZipException, IOException, IndexException {
+ public synchronized void compact() throws ZipException, IOException,
IndexException {
// find an interval of files that can be joined together
// we search from the end toward the start so that we can modify the
// list without changing the yet-unvisited IDs.
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
------------------------------------------------------------------------------
Android apps run on BlackBerry 10
Introducing the new BlackBerry 10.2.1 Runtime for Android apps.
Now with support for Jelly Bean, Bluetooth, Mapview and more.
Get your Android app in front of a whole new audience. Start now.
http://pubads.g.doubleclick.net/gampad/clk?id=124407151&iu=/4140/ostg.clktrk
_______________________________________________
GATE-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/gate-cvs
