siddharthteotia commented on a change in pull request #4993: Support Text
column type in Pinot (both offline and realtime)
URL: https://github.com/apache/incubator-pinot/pull/4993#discussion_r368758007
##########
File path:
pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
##########
@@ -118,6 +131,54 @@
private volatile long _lastIndexedTimeMs = Long.MIN_VALUE;
private volatile long _latestIngestionTimeMs = Long.MIN_VALUE;
+ private static final ScheduledExecutorService _scheduledExecutorService;
+ private static final
ConcurrentLinkedQueue<RealtimeLuceneReadersForRealtimeSegment>
_luceneRealtimeReaders;
+ private static final ConcurrentHashMap<String,
RealtimeLuceneReadersForRealtimeSegment> _segmentToRealtimeLuceneReadersMap;
+
+ static {
+ _scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+ _luceneRealtimeReaders = new ConcurrentLinkedQueue<>();
+ _segmentToRealtimeLuceneReadersMap = new ConcurrentHashMap<>();
+ _scheduledExecutorService.scheduleWithFixedDelay(new
RealtimeLuceneIndexReaderRefreshTask(_luceneRealtimeReaders),
+ RealtimeLuceneIndexReaderRefreshTask.INITIAL_DELAY_MS_DEFAULT,
RealtimeLuceneIndexReaderRefreshTask.DELAY_BETWEEN_SUCCESSIVE_EXECUTION_MS_DEFAULT,
TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Since the text index is maintained per TEXT column (similar to other
Pinot indexes),
+ * there could be multiple lucene indexes for a given segment and therefore
there can be
+ * multiple realtime lucene readers (one for each index/column) for the
particular
+ * realtime segment.
+ */
+ public static class RealtimeLuceneReadersForRealtimeSegment {
+ private final String segmentName;
+ private final Lock lock;
+ private volatile boolean segmentAboutToBeDestroyed;
+ private final List<RealtimeLuceneTextIndexReader> realtimeLuceneReaders;
Review comment:
This list need not be concurrent since this is updated only when the
MutableSegmentImpl is created. The overall queue of realtime segments is
concurrent.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
