This is an automated email from the ASF dual-hosted git repository.
alexpl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new d385d78c877 IGNITE-27430 SQL Calcite: Optimize IndexScan node creation
- Fixes #12599.
d385d78c877 is described below
commit d385d78c8779d764e85b7592d8e84816a88e29a5
Author: Aleksey Plekhanov <[email protected]>
AuthorDate: Tue Dec 30 10:02:37 2025 +0300
IGNITE-27430 SQL Calcite: Optimize IndexScan node creation - Fixes #12599.
Signed-off-by: Aleksey Plekhanov <[email protected]>
---
.../query/calcite/exec/AbstractCacheScan.java | 56 +++++++++++++---------
.../query/calcite/exec/ExecutionContext.java | 10 ++--
.../query/calcite/exec/LogicalRelImplementor.java | 5 +-
.../processors/query/calcite/exec/TableScan.java | 16 +++++++
.../calcite/exec/exp/ExpressionFactoryImpl.java | 20 ++++----
.../rel/ProjectableFilterableTableScan.java | 13 +++++
.../calcite/schema/CacheTableDescriptorImpl.java | 12 ++++-
.../internal/util/collection/BitSetIntSet.java | 5 ++
.../internal/util/collection/ImmutableIntSet.java | 14 ++++++
.../ignite/internal/util/collection/IntSet.java | 6 ++-
.../spi/indexing/IndexingQueryFilterImpl.java | 30 +++++++-----
11 files changed, 127 insertions(+), 60 deletions(-)
diff --git
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/AbstractCacheScan.java
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/AbstractCacheScan.java
index 1f46ed0d2da..bc48d3fc5d4 100644
---
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/AbstractCacheScan.java
+++
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/AbstractCacheScan.java
@@ -17,19 +17,18 @@
package org.apache.ignite.internal.processors.query.calcite.exec;
-import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collection;
import java.util.Iterator;
-import java.util.List;
-import java.util.stream.IntStream;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterTopologyException;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import
org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
-import
org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
import
org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
import
org.apache.ignite.internal.processors.cache.distributed.dht.topology.PartitionReservation;
+import org.apache.ignite.internal.util.collection.IntSet;
+import org.jetbrains.annotations.Nullable;
/** */
public abstract class AbstractCacheScan<Row> implements Iterable<Row>,
AutoCloseable {
@@ -43,36 +42,47 @@ public abstract class AbstractCacheScan<Row> implements
Iterable<Row>, AutoClose
protected final AffinityTopologyVersion topVer;
/** */
- protected final int[] parts;
+ protected final BitSet parts;
/** */
- protected final boolean explicitParts;
+ private final @Nullable int[] explicitParts;
/** */
private PartitionReservation reservation;
/** */
- protected volatile List<GridDhtLocalPartition> reservedParts;
-
- /** */
- AbstractCacheScan(ExecutionContext<Row> ectx, GridCacheContext<?, ?> cctx,
int[] parts) {
+ AbstractCacheScan(ExecutionContext<Row> ectx, GridCacheContext<?, ?> cctx,
@Nullable int[] explicitParts) {
this.ectx = ectx;
this.cctx = cctx;
topVer = ectx.topologyVersion();
- explicitParts = parts != null;
+ this.explicitParts = explicitParts;
+
+ int partsCnt = cctx.affinity().partitions();
- if (cctx.isReplicated())
- this.parts = IntStream.range(0,
cctx.affinity().partitions()).toArray();
+ if (cctx.isReplicated()) {
+ parts = new BitSet(partsCnt);
+ parts.set(0, partsCnt);
+ }
else {
- if (parts != null)
- this.parts = parts;
+ if (explicitParts != null) {
+ parts = new BitSet(partsCnt);
+
+ for (int i = 0; i < explicitParts.length; i++)
+ parts.set(explicitParts[i]);
+ }
else {
Collection<Integer> primaryParts =
cctx.affinity().primaryPartitions(
cctx.kernalContext().localNodeId(), topVer);
- this.parts =
primaryParts.stream().mapToInt(Integer::intValue).toArray();
+ if (primaryParts instanceof IntSet)
+ parts = ((IntSet)primaryParts).toBitSet();
+ else {
+ parts = new BitSet(partsCnt);
+
+ primaryParts.forEach(parts::set);
+ }
}
}
}
@@ -124,7 +134,7 @@ public abstract class AbstractCacheScan<Row> implements
Iterable<Row>, AutoClose
try {
reservation =
cctx.kernalContext().query().partitionReservationManager().reservePartitions(
- cctx, topVer, explicitParts ? parts : null,
ectx.originatingNodeId(), "qryId=" + ectx.queryId());
+ cctx, topVer, explicitParts, ectx.originatingNodeId(),
"qryId=" + ectx.queryId());
}
catch (IgniteCheckedException e) {
throw new ClusterTopologyException("Failed to reserve
partition for query execution", e);
@@ -138,18 +148,18 @@ public abstract class AbstractCacheScan<Row> implements
Iterable<Row>, AutoClose
this.reservation = reservation;
- List<GridDhtLocalPartition> reservedParts = new
ArrayList<>(parts.length);
-
- for (int i = 0; i < parts.length; i++)
- reservedParts.add(top.localPartition(parts[i]));
-
- this.reservedParts = reservedParts;
+ processReservedTopology(top);
}
finally {
top.readUnlock();
}
}
+ /** */
+ protected void processReservedTopology(GridDhtPartitionTopology top) {
+ // No-op.
+ }
+
/** */
private synchronized void release() {
if (reservation != null)
diff --git
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionContext.java
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionContext.java
index b028459de59..2c533c06c21 100644
---
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionContext.java
+++
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionContext.java
@@ -19,7 +19,7 @@ package
org.apache.ignite.internal.processors.query.calcite.exec;
import java.lang.reflect.Type;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.BitSet;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
@@ -369,17 +369,13 @@ public class ExecutionContext<Row> extends
AbstractQueryContext implements DataC
*/
public <R> TransactionChanges<R> transactionChanges(
int cacheId,
- int[] parts,
+ BitSet parts,
Function<CacheDataRow, R> mapper,
@Nullable Comparator<R> cmp
) {
if (F.isEmpty(qryTxEntries))
return TransactionChanges.empty();
- // Expecting parts are sorted or almost sorted and amount of
transaction entries are relatively small.
- if (parts != null && !F.isSorted(parts))
- Arrays.sort(parts);
-
Set<KeyCacheObject> changedKeys = new HashSet<>(qryTxEntries.size());
List<R> newAndUpdatedRows = new ArrayList<>(qryTxEntries.size());
@@ -391,7 +387,7 @@ public class ExecutionContext<Row> extends
AbstractQueryContext implements DataC
if (e.cacheId() != cacheId)
continue;
- if (parts != null && Arrays.binarySearch(parts, part) < 0)
+ if (parts != null && !parts.get(part))
continue;
changedKeys.add(e.key());
diff --git
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/LogicalRelImplementor.java
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/LogicalRelImplementor.java
index 9e05f5bf2c2..e1687bcbdc6 100644
---
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/LogicalRelImplementor.java
+++
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/LogicalRelImplementor.java
@@ -368,7 +368,7 @@ public class LogicalRelImplementor<Row> implements
IgniteRelVisitor<Node<Row>> {
ImmutableBitSet requiredColumns = rel.requiredColumns();
List<SearchBounds> searchBounds = rel.searchBounds();
- RelDataType rowType = tbl.getRowType(typeFactory, requiredColumns);
+ RelDataType rowType = rel.getDataSourceRowType();
Predicate<Row> filters = condition == null ? null :
expressionFactory.predicate(condition, rowType);
Function<Row, Row> prj = projects == null ? null :
expressionFactory.project(projects, rowType);
@@ -546,9 +546,8 @@ public class LogicalRelImplementor<Row> implements
IgniteRelVisitor<Node<Row>> {
ImmutableBitSet requiredColumns = rel.requiredColumns();
IgniteTable tbl = rel.getTable().unwrap(IgniteTable.class);
- IgniteTypeFactory typeFactory = ctx.getTypeFactory();
- RelDataType rowType = tbl.getRowType(typeFactory, requiredColumns);
+ RelDataType rowType = rel.getDataSourceRowType();
Predicate<Row> filters = condition == null ? null :
expressionFactory.predicate(condition, rowType);
Function<Row, Row> prj = projects == null ? null :
expressionFactory.project(projects, rowType);
diff --git
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/TableScan.java
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/TableScan.java
index c16252b902a..46c7cde4e67 100644
---
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/TableScan.java
+++
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/TableScan.java
@@ -18,14 +18,17 @@
package org.apache.ignite.internal.processors.query.calcite.exec;
import java.util.ArrayDeque;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.NoSuchElementException;
import java.util.Queue;
import java.util.function.Function;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.ignite.IgniteCheckedException;
import
org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import
org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow;
import
org.apache.ignite.internal.processors.cache.transactions.TransactionChanges;
@@ -38,6 +41,9 @@ import org.jetbrains.annotations.Nullable;
/** */
public class TableScan<Row> extends AbstractCacheColumnsScan<Row> {
+ /** */
+ protected volatile List<GridDhtLocalPartition> reservedParts;
+
/** */
public TableScan(
ExecutionContext<Row> ectx,
@@ -53,6 +59,16 @@ public class TableScan<Row> extends
AbstractCacheColumnsScan<Row> {
return new IteratorImpl();
}
+ /** {@inheritDoc} */
+ @Override protected void processReservedTopology(GridDhtPartitionTopology
top) {
+ List<GridDhtLocalPartition> reservedParts = new
ArrayList<>(parts.cardinality());
+
+ for (int part = parts.nextSetBit(0); part >= 0; part =
parts.nextSetBit(part + 1))
+ reservedParts.add(top.localPartition(part));
+
+ this.reservedParts = reservedParts;
+ }
+
/**
* Table scan iterator.
*/
diff --git
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ExpressionFactoryImpl.java
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ExpressionFactoryImpl.java
index 7f543385d33..6ea08427021 100644
---
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ExpressionFactoryImpl.java
+++
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/ExpressionFactoryImpl.java
@@ -105,13 +105,13 @@ public class ExpressionFactoryImpl<Row> implements
ExpressionFactory<Row> {
private final RexBuilder rexBuilder;
/** */
- private final RelDataType emptyType;
+ private static final RelDataType EMPTY_TYPE = new
RelDataTypeFactory.Builder(Commons.typeFactory()).build();
/** */
- private final RelDataType nullType;
+ private static final RelDataType NULL_TYPE =
Commons.typeFactory().createSqlType(SqlTypeName.NULL);
/** */
- private final RelDataType booleanType;
+ private static final RelDataType BOOLEAN_TYPE =
Commons.typeFactory().createJavaType(Boolean.class);
/** */
private final ExecutionContext<Row> ctx;
@@ -127,10 +127,6 @@ public class ExpressionFactoryImpl<Row> implements
ExpressionFactory<Row> {
this.typeFactory = typeFactory;
this.conformance = conformance;
this.rexBuilder = rexBuilder;
-
- emptyType = new RelDataTypeFactory.Builder(this.typeFactory).build();
- nullType = typeFactory.createSqlType(SqlTypeName.NULL);
- booleanType = typeFactory.createJavaType(Boolean.class);
}
/** {@inheritDoc} */
@@ -296,7 +292,7 @@ public class ExpressionFactoryImpl<Row> implements
ExpressionFactory<Row> {
/** {@inheritDoc} */
@Override public Supplier<Row> rowSource(List<RexNode> values) {
return new ValuesImpl(scalar(values, null),
ctx.rowHandler().factory(typeFactory,
- Commons.transform(values, v -> v != null ? v.getType() :
nullType)));
+ Commons.transform(values, v -> v != null ? v.getType() :
NULL_TYPE)));
}
/** {@inheritDoc} */
@@ -494,7 +490,7 @@ public class ExpressionFactoryImpl<Row> implements
ExpressionFactory<Row> {
/** */
private Scalar compile(List<RexNode> nodes, RelDataType type, boolean
biInParams) {
if (type == null)
- type = emptyType;
+ type = EMPTY_TYPE;
RexProgramBuilder programBuilder = new RexProgramBuilder(type,
rexBuilder);
@@ -508,8 +504,8 @@ public class ExpressionFactoryImpl<Row> implements
ExpressionFactory<Row> {
else {
unspecifiedValues.set(i);
- programBuilder.addProject(rexBuilder.makeNullLiteral(type ==
emptyType ?
- nullType : type.getFieldList().get(i).getType()), null);
+ programBuilder.addProject(rexBuilder.makeNullLiteral(type ==
EMPTY_TYPE ?
+ NULL_TYPE : type.getFieldList().get(i).getType()), null);
}
}
@@ -637,7 +633,7 @@ public class ExpressionFactoryImpl<Row> implements
ExpressionFactory<Row> {
private AbstractScalarPredicate(T scalar) {
this.scalar = scalar;
hnd = ctx.rowHandler();
- out = hnd.factory(typeFactory, booleanType).create();
+ out = hnd.factory(typeFactory, BOOLEAN_TYPE).create();
}
}
diff --git
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/ProjectableFilterableTableScan.java
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/ProjectableFilterableTableScan.java
index 5508ee6f69f..208e8b70e86 100644
---
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/ProjectableFilterableTableScan.java
+++
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/ProjectableFilterableTableScan.java
@@ -63,6 +63,9 @@ public abstract class ProjectableFilterableTableScan extends
TableScan {
/** Participating columns. */
protected final ImmutableBitSet requiredColumns;
+ /** Required columns from table row type (No need to be serialized, for
caching only). */
+ protected RelDataType dataSourceRowType;
+
/** */
protected ProjectableFilterableTableScan(
RelOptCluster cluster,
@@ -151,6 +154,16 @@ public abstract class ProjectableFilterableTableScan
extends TableScan {
return
table.unwrap(IgniteTable.class).getRowType(Commons.typeFactory(getCluster()),
requiredColumns);
}
+ /** */
+ public RelDataType getDataSourceRowType() {
+ if (dataSourceRowType == null) {
+ dataSourceRowType =
table.unwrap(IgniteTable.class).getRowType(Commons.typeFactory(getCluster()),
+ requiredColumns);
+ }
+
+ return dataSourceRowType;
+ }
+
/** */
public RexNode pushUpPredicate() {
if (condition == null || projects == null)
diff --git
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java
index d60eb448ba6..22f6c2dcb8f 100644
---
a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java
+++
b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/schema/CacheTableDescriptorImpl.java
@@ -112,6 +112,9 @@ public class CacheTableDescriptorImpl extends
NullInitializerExpressionFactory
/** */
private final ImmutableBitSet insertFields;
+ /** */
+ private RelDataType tableRowType;
+
/** */
public CacheTableDescriptorImpl(GridCacheContextInfo<?, ?> cacheInfo,
GridQueryTypeDescriptor typeDesc,
Object affinityIdentity) {
@@ -490,18 +493,23 @@ public class CacheTableDescriptorImpl extends
NullInitializerExpressionFactory
/** {@inheritDoc} */
@Override public RelDataType rowType(IgniteTypeFactory factory,
ImmutableBitSet usedColumns) {
+ if (usedColumns == null && tableRowType != null)
+ return tableRowType;
+
RelDataTypeFactory.Builder b = new RelDataTypeFactory.Builder(factory);
if (usedColumns == null) {
for (int i = 0; i < descriptors.length; i++)
b.add(descriptors[i].name(),
descriptors[i].logicalType(factory));
+
+ return tableRowType = b.build();
}
else {
for (int i = usedColumns.nextSetBit(0); i != -1; i =
usedColumns.nextSetBit(i + 1))
b.add(descriptors[i].name(),
descriptors[i].logicalType(factory));
- }
- return b.build();
+ return b.build();
+ }
}
/** {@inheritDoc} */
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/util/collection/BitSetIntSet.java
b/modules/core/src/main/java/org/apache/ignite/internal/util/collection/BitSetIntSet.java
index a63573bbdc7..602d88dbf4a 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/util/collection/BitSetIntSet.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/util/collection/BitSetIntSet.java
@@ -177,6 +177,11 @@ public class BitSetIntSet extends
GridSerializableCollection<Integer> implements
return arr;
}
+ /** {@inheritDoc} */
+ @Override public BitSet toBitSet() {
+ return (BitSet)bitSet.clone();
+ }
+
/** {@inheritDoc} */
@Override public boolean containsAll(@NotNull Collection<?> c) {
for (Object o : c) {
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/util/collection/ImmutableIntSet.java
b/modules/core/src/main/java/org/apache/ignite/internal/util/collection/ImmutableIntSet.java
index f952d489525..00109862cb8 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/util/collection/ImmutableIntSet.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/util/collection/ImmutableIntSet.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.util.collection;
+import java.util.BitSet;
import java.util.Collection;
import java.util.Iterator;
import java.util.Set;
@@ -88,6 +89,19 @@ public class ImmutableIntSet implements IntSet {
return delegate.toArray();
}
+ /** {@inheritDoc} */
+ @Override public BitSet toBitSet() {
+ if (delegate instanceof IntSet)
+ return ((IntSet)delegate).toBitSet();
+ else {
+ BitSet bitSet = new BitSet();
+
+ forEach(bitSet::set);
+
+ return bitSet;
+ }
+ }
+
/** {@inheritDoc} */
@NotNull @Override public <T> T[] toArray(@NotNull T[] a) {
return delegate.toArray(a);
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/util/collection/IntSet.java
b/modules/core/src/main/java/org/apache/ignite/internal/util/collection/IntSet.java
index caac3d35c93..9b5db212a12 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/util/collection/IntSet.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/util/collection/IntSet.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.util.collection;
+import java.util.BitSet;
import java.util.Set;
/**
@@ -32,6 +33,9 @@ public interface IntSet extends Set<Integer> {
/** Removes the specified element from this set. */
boolean remove(int element);
- /** Returns array with primitive types **/
+ /** Returns array with primitive types. **/
int[] toIntArray();
+
+ /** Returns BitSet. **/
+ BitSet toBitSet();
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingQueryFilterImpl.java
b/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingQueryFilterImpl.java
index a996ed4bb57..c9d74f481b3 100644
---
a/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingQueryFilterImpl.java
+++
b/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingQueryFilterImpl.java
@@ -54,6 +54,24 @@ public class IndexingQueryFilterImpl implements
IndexingQueryFilter {
*/
private final boolean treatReplicatedAsPartitioned;
+ /**
+ * Constructor.
+ *
+ * @param ctx Kernal context.
+ * @param topVer Topology version.
+ * @param parts Partitions set.
+ */
+ public IndexingQueryFilterImpl(
+ GridKernalContext ctx,
+ @Nullable AffinityTopologyVersion topVer,
+ @Nullable BitSet parts
+ ) {
+ this.ctx = ctx;
+ this.topVer = topVer != null ? topVer : AffinityTopologyVersion.NONE;
+ this.parts = parts;
+ treatReplicatedAsPartitioned = false;
+ }
+
/**
* Constructor.
*
@@ -80,18 +98,6 @@ public class IndexingQueryFilterImpl implements
IndexingQueryFilter {
this.treatReplicatedAsPartitioned = treatReplicatedAsPartitioned;
}
- /**
- * Constructor.
- *
- * @param ctx Kernal context.
- * @param topVer Topology version.
- * @param partsArr Partitions array.
- */
- public IndexingQueryFilterImpl(GridKernalContext ctx, @Nullable
AffinityTopologyVersion topVer,
- @Nullable int[] partsArr) {
- this(ctx, topVer, partsArr, false);
- }
-
/** {@inheritDoc} */
@Nullable @Override public IndexingQueryCacheFilter forCache(String
cacheName) {
final GridCacheAdapter<Object, Object> cache =
ctx.cache().internalCache(cacheName);
