mcvsubbu commented on a change in pull request #4993: Support Text column type
in Pinot (both offline and realtime)
URL: https://github.com/apache/incubator-pinot/pull/4993#discussion_r378576556
##########
File path:
pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/invertedindex/RealtimeLuceneIndexRefreshState.java
##########
@@ -0,0 +1,98 @@
+package org.apache.pinot.core.realtime.impl.invertedindex;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+
+/**
+ * This class manages the realtime lucene index readers. Creates a global
+ * queue with all the realtime segment lucene index readers across
+ * all tables and manages their refresh using {@link
RealtimeLuceneIndexReaderRefreshThread}
+ */
+public class RealtimeLuceneIndexRefreshState {
+ private static RealtimeLuceneIndexRefreshState _singletonInstance;
+ private static ScheduledExecutorService _scheduledExecutorService;
+ private static
ConcurrentLinkedQueue<RealtimeLuceneReadersForRealtimeSegment>
_luceneRealtimeReaders;
+
+ private RealtimeLuceneIndexRefreshState() {
+ // This constructor is called by getInstance() exactly once. Since
+ // getInstance() is invoked by MutableSegmentImpl only if it encounters
+ // a TEXT index column, we ensure that background refresh thread is not
created
+ // if there is no column with TEXT index enabled
+ _scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+ // TODO: eventually we should explore partitioning this queue on per table
basis
+ _luceneRealtimeReaders = new ConcurrentLinkedQueue<>();
+ _scheduledExecutorService.scheduleWithFixedDelay(new
RealtimeLuceneIndexReaderRefreshThread(_luceneRealtimeReaders),
+ RealtimeLuceneIndexReaderRefreshThread.INITIAL_DELAY_MS_DEFAULT,
RealtimeLuceneIndexReaderRefreshThread.DELAY_BETWEEN_SUCCESSIVE_EXECUTION_MS_DEFAULT,
TimeUnit.MILLISECONDS);
+ }
+
+ public static RealtimeLuceneIndexRefreshState getInstance() {
+ if (_singletonInstance == null) {
+ synchronized (RealtimeLuceneIndexRefreshState.class) {
+ if (_singletonInstance == null) {
+ _singletonInstance = new RealtimeLuceneIndexRefreshState();
+ }
+ }
+ }
+ return _singletonInstance;
+ }
+
+ public void
addRealtimeReadersToQueue(RealtimeLuceneReadersForRealtimeSegment
readersForRealtimeSegment) {
+ _luceneRealtimeReaders.offer(readersForRealtimeSegment);
+ }
+
+ /**
+ * Since the text index is maintained per TEXT column (similar to other
Pinot indexes),
Review comment:
please change the comment, we dont have TEXT columns anymore.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
