siddharthteotia commented on a change in pull request #4993: Support Text column type in Pinot (both offline and realtime) URL: https://github.com/apache/incubator-pinot/pull/4993#discussion_r368758007
########## File path: pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java ########## @@ -118,6 +131,54 @@ private volatile long _lastIndexedTimeMs = Long.MIN_VALUE; private volatile long _latestIngestionTimeMs = Long.MIN_VALUE; + private static final ScheduledExecutorService _scheduledExecutorService; + private static final ConcurrentLinkedQueue<RealtimeLuceneReadersForRealtimeSegment> _luceneRealtimeReaders; + private static final ConcurrentHashMap<String, RealtimeLuceneReadersForRealtimeSegment> _segmentToRealtimeLuceneReadersMap; + + static { + _scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + _luceneRealtimeReaders = new ConcurrentLinkedQueue<>(); + _segmentToRealtimeLuceneReadersMap = new ConcurrentHashMap<>(); + _scheduledExecutorService.scheduleWithFixedDelay(new RealtimeLuceneIndexReaderRefreshTask(_luceneRealtimeReaders), + RealtimeLuceneIndexReaderRefreshTask.INITIAL_DELAY_MS_DEFAULT, RealtimeLuceneIndexReaderRefreshTask.DELAY_BETWEEN_SUCCESSIVE_EXECUTION_MS_DEFAULT, TimeUnit.MILLISECONDS); + } + + /** + * Since the text index is maintained per TEXT column (similar to other Pinot indexes), + * there could be multiple lucene indexes for a given segment and therefore there can be + * multiple realtime lucene readers (one for each index/column) for the particular + * realtime segment. + */ + public static class RealtimeLuceneReadersForRealtimeSegment { + private final String segmentName; + private final Lock lock; + private volatile boolean segmentAboutToBeDestroyed; + private final List<RealtimeLuceneTextIndexReader> realtimeLuceneReaders; Review comment: This list need not be concurrent since this is updated only when the MutableSegmentImpl is created. The overall queue of realtime segments is concurrent. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org