shizy818 commented on code in PR #14616:
URL: https://github.com/apache/iotdb/pull/14616#discussion_r1909689619
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java:
##########
@@ -245,17 +312,96 @@ private void sortTVList() {
@Override
public synchronized void sortTvListForFlush() {
- sortTVList();
+ TVList cloneList = null;
+ list.lockQueryList();
+ try {
+ // During flush, if the working TVList is not sorted and referenced by
some query, we need to
+ // clone it. The query still refer to original unsorted TVList.
+ if (!list.isSorted() && !list.getQueryContextList().isEmpty()) {
+ QueryContext firstQuery = list.getQueryContextList().get(0);
+ // reserve query memory
+ if (firstQuery instanceof FragmentInstanceContext) {
+ MemoryReservationManager memoryReservationManager =
+ ((FragmentInstanceContext)
firstQuery).getMemoryReservationContext();
+
memoryReservationManager.reserveMemoryCumulatively(list.calculateRamSize());
+ }
+ list.setOwnerQuery(firstQuery);
+ cloneList = list.clone();
+ }
+ } finally {
+ list.unlockQueryList();
+ }
+ if (cloneList != null) {
+ setWorkingTVList(cloneList);
+ }
+
+ if (!list.isSorted()) {
+ list.sort();
+ }
+ }
+
+ private void filterDeletedTimestamp(
+ TVList tvlist, List<TimeRange> deletionList, List<Long> timestampList) {
+ long lastTime = Long.MIN_VALUE;
+ int[] deletionCursor = {0};
+ int rowCount = tvlist.rowCount();
+ for (int i = 0; i < rowCount; i++) {
+ if (tvlist.getBitMap() != null && tvlist.isNullValue(i)) {
+ continue;
+ }
+ long curTime = tvlist.getTime(i);
+ if (deletionList != null
+ && ModificationUtils.isPointDeleted(curTime, deletionList,
deletionCursor)) {
+ continue;
+ }
+
+ if (i == rowCount - 1 || curTime != lastTime) {
+ timestampList.add(curTime);
+ }
+ lastTime = curTime;
+ }
+ }
+
+ public long[] getFilteredTimestamp(List<TimeRange> deletionList) {
+ List<Long> timestampList = new ArrayList<>();
+ filterDeletedTimestamp(list, deletionList, timestampList);
+ for (TVList tvList : sortedList) {
+ filterDeletedTimestamp(tvList, deletionList, timestampList);
+ }
+
+ // remove duplicated time
+ List<Long> distinctTimestamps =
timestampList.stream().distinct().collect(Collectors.toList());
+ // sort timestamps
+ long[] filteredTimestamps =
distinctTimestamps.stream().mapToLong(Long::longValue).toArray();
+ Arrays.sort(filteredTimestamps);
Review Comment:
This is part of queryForSeriesRegionScan. `getMemChunkHandleFromMemTable`
needs to find out all sorted timestamps in the memchunk. Because it only deal
with timestamp, I do not sort working list and use merge sort iterator. Instead
I copy timestamps from all lists and sort directly.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
