This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 5da3ff569 [spark] Fix reported statistics does not do column pruning
(#4137)
5da3ff569 is described below
commit 5da3ff5695c59d0c661687528d398de7dc678ffc
Author: Xiduo You <[email protected]>
AuthorDate: Fri Sep 13 11:52:42 2024 +0800
[spark] Fix reported statistics does not do column pruning (#4137)
---
.../java/org/apache/paimon/types/ArrayType.java | 5 ++
.../java/org/apache/paimon/types/BigIntType.java | 5 ++
.../java/org/apache/paimon/types/BinaryType.java | 5 ++
.../java/org/apache/paimon/types/BooleanType.java | 5 ++
.../java/org/apache/paimon/types/CharType.java | 5 ++
.../java/org/apache/paimon/types/DataType.java | 3 ++
.../java/org/apache/paimon/types/DateType.java | 5 ++
.../java/org/apache/paimon/types/DecimalType.java | 6 +++
.../java/org/apache/paimon/types/DoubleType.java | 5 ++
.../java/org/apache/paimon/types/FloatType.java | 5 ++
.../main/java/org/apache/paimon/types/IntType.java | 5 ++
.../paimon/types/LocalZonedTimestampType.java | 5 ++
.../main/java/org/apache/paimon/types/MapType.java | 5 ++
.../java/org/apache/paimon/types/MultisetType.java | 5 ++
.../main/java/org/apache/paimon/types/RowType.java | 5 ++
.../java/org/apache/paimon/types/SmallIntType.java | 5 ++
.../java/org/apache/paimon/types/TimeType.java | 5 ++
.../org/apache/paimon/types/TimestampType.java | 5 ++
.../java/org/apache/paimon/types/TinyIntType.java | 5 ++
.../org/apache/paimon/types/VarBinaryType.java | 5 ++
.../java/org/apache/paimon/types/VarCharType.java | 5 ++
.../org/apache/paimon/spark/PaimonBaseScan.scala | 9 ++--
.../org/apache/paimon/spark/PaimonStatistics.scala | 53 ++++++++++++++++++++--
.../paimon/spark/sql/AnalyzeTableTestBase.scala | 51 +++++++++++++++++++++
24 files changed, 208 insertions(+), 9 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/ArrayType.java
b/paimon-common/src/main/java/org/apache/paimon/types/ArrayType.java
index 7d86a94bd..0b5512c4c 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/ArrayType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/ArrayType.java
@@ -57,6 +57,11 @@ public final class ArrayType extends DataType {
return elementType;
}
+ @Override
+ public int defaultSize() {
+ return elementType.defaultSize();
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new ArrayType(isNullable, elementType.copy());
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/BigIntType.java
b/paimon-common/src/main/java/org/apache/paimon/types/BigIntType.java
index 564133551..e625fae68 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/BigIntType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/BigIntType.java
@@ -41,6 +41,11 @@ public final class BigIntType extends DataType {
this(true);
}
+ @Override
+ public int defaultSize() {
+ return 8;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new BigIntType(isNullable);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/BinaryType.java
b/paimon-common/src/main/java/org/apache/paimon/types/BinaryType.java
index c50d463c8..bb74a9587 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/BinaryType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/BinaryType.java
@@ -65,6 +65,11 @@ public class BinaryType extends DataType {
return length;
}
+ @Override
+ public int defaultSize() {
+ return length;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new BinaryType(isNullable, length);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/BooleanType.java
b/paimon-common/src/main/java/org/apache/paimon/types/BooleanType.java
index fe97717b9..9868d1955 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/BooleanType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/BooleanType.java
@@ -40,6 +40,11 @@ public class BooleanType extends DataType {
this(true);
}
+ @Override
+ public int defaultSize() {
+ return 1;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new BooleanType(isNullable);
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/CharType.java
b/paimon-common/src/main/java/org/apache/paimon/types/CharType.java
index efba0cd44..77694d9c1 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/CharType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/CharType.java
@@ -67,6 +67,11 @@ public class CharType extends DataType {
return length;
}
+ @Override
+ public int defaultSize() {
+ return length;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new CharType(isNullable, length);
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/DataType.java
b/paimon-common/src/main/java/org/apache/paimon/types/DataType.java
index 3a8b3b02d..786f226a3 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/DataType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/DataType.java
@@ -104,6 +104,9 @@ public abstract class DataType implements Serializable {
return typeRoot.getFamilies().contains(family);
}
+ /** The default size of a value of this data type, used internally for
size estimation. */
+ public abstract int defaultSize();
+
/**
* Returns a deep copy of this type with possibly different nullability.
*
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/DateType.java
b/paimon-common/src/main/java/org/apache/paimon/types/DateType.java
index e82c3de2e..48285ee09 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/DateType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/DateType.java
@@ -44,6 +44,11 @@ public final class DateType extends DataType {
this(true);
}
+ @Override
+ public int defaultSize() {
+ return 4;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new DateType(isNullable);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/DecimalType.java
b/paimon-common/src/main/java/org/apache/paimon/types/DecimalType.java
index 7c2e46499..88e5c8300 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/DecimalType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/DecimalType.java
@@ -19,6 +19,7 @@
package org.apache.paimon.types;
import org.apache.paimon.annotation.Public;
+import org.apache.paimon.data.Decimal;
import java.util.Objects;
@@ -86,6 +87,11 @@ public class DecimalType extends DataType {
return scale;
}
+ @Override
+ public int defaultSize() {
+ return Decimal.isCompact(precision) ? 8 : 16;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new DecimalType(isNullable, precision, scale);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/DoubleType.java
b/paimon-common/src/main/java/org/apache/paimon/types/DoubleType.java
index 8a18ba2c4..2e4e5e47c 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/DoubleType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/DoubleType.java
@@ -40,6 +40,11 @@ public class DoubleType extends DataType {
this(true);
}
+ @Override
+ public int defaultSize() {
+ return 8;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new DoubleType(isNullable);
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/FloatType.java
b/paimon-common/src/main/java/org/apache/paimon/types/FloatType.java
index 0575cb52e..f06c7855b 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/FloatType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/FloatType.java
@@ -41,6 +41,11 @@ public class FloatType extends DataType {
this(true);
}
+ @Override
+ public int defaultSize() {
+ return 4;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new FloatType(isNullable);
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/IntType.java
b/paimon-common/src/main/java/org/apache/paimon/types/IntType.java
index 9f99267d8..a1a0a05f8 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/IntType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/IntType.java
@@ -40,6 +40,11 @@ public class IntType extends DataType {
this(true);
}
+ @Override
+ public int defaultSize() {
+ return 4;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new IntType(isNullable);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/LocalZonedTimestampType.java
b/paimon-common/src/main/java/org/apache/paimon/types/LocalZonedTimestampType.java
index d5d1eb11d..5e7a60416 100644
---
a/paimon-common/src/main/java/org/apache/paimon/types/LocalZonedTimestampType.java
+++
b/paimon-common/src/main/java/org/apache/paimon/types/LocalZonedTimestampType.java
@@ -75,6 +75,11 @@ public final class LocalZonedTimestampType extends DataType {
return precision;
}
+ @Override
+ public int defaultSize() {
+ return 8;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new LocalZonedTimestampType(isNullable, precision);
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/MapType.java
b/paimon-common/src/main/java/org/apache/paimon/types/MapType.java
index 817fe8676..238f681ee 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/MapType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/MapType.java
@@ -64,6 +64,11 @@ public class MapType extends DataType {
return valueType;
}
+ @Override
+ public int defaultSize() {
+ return keyType.defaultSize() + valueType.defaultSize();
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new MapType(isNullable, keyType.copy(), valueType.copy());
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/MultisetType.java
b/paimon-common/src/main/java/org/apache/paimon/types/MultisetType.java
index d3f0d1e55..616738772 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/MultisetType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/MultisetType.java
@@ -61,6 +61,11 @@ public class MultisetType extends DataType {
return elementType;
}
+ @Override
+ public int defaultSize() {
+ return elementType.defaultSize() + 4;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new MultisetType(isNullable, elementType);
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java
b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java
index a691dbc0b..0fa6a9c1b 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java
@@ -131,6 +131,11 @@ public final class RowType extends DataType {
throw new RuntimeException("Cannot find field: " + fieldName);
}
+ @Override
+ public int defaultSize() {
+ return fields.stream().mapToInt(f -> f.type().defaultSize()).sum();
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new RowType(
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/SmallIntType.java
b/paimon-common/src/main/java/org/apache/paimon/types/SmallIntType.java
index b63800deb..87dbc4f56 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/SmallIntType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/SmallIntType.java
@@ -40,6 +40,11 @@ public final class SmallIntType extends DataType {
this(true);
}
+ @Override
+ public int defaultSize() {
+ return 2;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new SmallIntType(isNullable);
diff --git a/paimon-common/src/main/java/org/apache/paimon/types/TimeType.java
b/paimon-common/src/main/java/org/apache/paimon/types/TimeType.java
index 781194b42..2f1784002 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/TimeType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/TimeType.java
@@ -72,6 +72,11 @@ public final class TimeType extends DataType {
return precision;
}
+ @Override
+ public int defaultSize() {
+ return 4;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new TimeType(isNullable, precision);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/TimestampType.java
b/paimon-common/src/main/java/org/apache/paimon/types/TimestampType.java
index 07e0fcdb6..9c9306f54 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/TimestampType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/TimestampType.java
@@ -70,6 +70,11 @@ public class TimestampType extends DataType {
return precision;
}
+ @Override
+ public int defaultSize() {
+ return 8;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new TimestampType(isNullable, precision);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/TinyIntType.java
b/paimon-common/src/main/java/org/apache/paimon/types/TinyIntType.java
index a7be5ae62..0967fab43 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/TinyIntType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/TinyIntType.java
@@ -40,6 +40,11 @@ public class TinyIntType extends DataType {
this(true);
}
+ @Override
+ public int defaultSize() {
+ return 1;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new TinyIntType(isNullable);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/VarBinaryType.java
b/paimon-common/src/main/java/org/apache/paimon/types/VarBinaryType.java
index 7331172f7..6afc6692d 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/VarBinaryType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/VarBinaryType.java
@@ -67,6 +67,11 @@ public final class VarBinaryType extends DataType {
return length;
}
+ @Override
+ public int defaultSize() {
+ return length == MAX_LENGTH ? 20 : length;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new VarBinaryType(isNullable, length);
diff --git
a/paimon-common/src/main/java/org/apache/paimon/types/VarCharType.java
b/paimon-common/src/main/java/org/apache/paimon/types/VarCharType.java
index 9585ecd8f..445876b0b 100644
--- a/paimon-common/src/main/java/org/apache/paimon/types/VarCharType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/types/VarCharType.java
@@ -71,6 +71,11 @@ public final class VarCharType extends DataType {
return length;
}
+ @Override
+ public int defaultSize() {
+ return length == MAX_LENGTH ? 20 : length;
+ }
+
@Override
public DataType copy(boolean isNullable) {
return new VarCharType(isNullable, length);
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
index 6fdfe4f6a..2188879c7 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScan.scala
@@ -54,7 +54,7 @@ abstract class PaimonBaseScan(
private lazy val tableSchema = SparkTypeUtils.fromPaimonRowType(tableRowType)
- private val (tableFields, metadataFields) = {
+ private[paimon] val (requiredTableFields, metadataFields) = {
val nameToField = tableSchema.map(field => (field.name, field)).toMap
val _tableFields = requiredSchema.flatMap(field =>
nameToField.get(field.name))
val _metadataFields =
@@ -75,14 +75,15 @@ abstract class PaimonBaseScan(
private lazy val paimonMetricsRegistry: SparkMetricRegistry =
SparkMetricRegistry()
lazy val requiredStatsSchema: StructType = {
- val fieldNames = tableFields.map(_.name) ++
reservedFilters.flatMap(_.references)
+ val fieldNames = requiredTableFields.map(_.name) ++
reservedFilters.flatMap(_.references)
StructType(tableSchema.filter(field => fieldNames.contains(field.name)))
}
lazy val readBuilder: ReadBuilder = {
val _readBuilder = table.newReadBuilder()
- val projection = tableFields.map(field =>
tableSchema.fieldNames.indexOf(field.name)).toArray
+ val projection =
+ requiredTableFields.map(field =>
tableSchema.fieldNames.indexOf(field.name)).toArray
_readBuilder.withProjection(projection)
if (filters.nonEmpty) {
val pushedPredicate = PredicateBuilder.and(filters: _*)
@@ -113,7 +114,7 @@ abstract class PaimonBaseScan(
}
override def readSchema(): StructType = {
- StructType(tableFields ++ metadataFields)
+ StructType(requiredTableFields ++ metadataFields)
}
override def toBatch: Batch = {
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
index da9239dd0..c64ec7310 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
@@ -18,9 +18,9 @@
package org.apache.paimon.spark
-import org.apache.paimon.stats
+import org.apache.paimon.spark.schema.PaimonMetadataColumn
import org.apache.paimon.stats.ColStats
-import org.apache.paimon.types.DataType
+import org.apache.paimon.types.{DataField, DataType, RowType}
import org.apache.spark.sql.PaimonUtils
import org.apache.spark.sql.catalyst.plans.logical.ColumnStat
@@ -40,10 +40,53 @@ case class PaimonStatistics[T <: PaimonBaseScan](scan: T)
extends Statistics {
private lazy val paimonStats = if (scan.statistics.isPresent)
scan.statistics.get() else null
- lazy val paimonStatsEnabled: Boolean = paimonStats != null
+ lazy val paimonStatsEnabled: Boolean = {
+ paimonStats != null &&
+ paimonStats.mergedRecordSize().isPresent &&
+ paimonStats.mergedRecordCount().isPresent
+ }
+
+ private def getSizeForField(field: DataField): Long = {
+ Option(paimonStats.colStats().get(field.name()))
+ .map(_.avgLen())
+ .filter(_.isPresent)
+ .map(_.getAsLong)
+ .getOrElse(field.`type`().defaultSize().toLong)
+ }
+
+ private def getSizeForRow(schema: RowType): Long = {
+ schema.getFields.asScala.map(field => getSizeForField(field)).sum
+ }
+
+ override def sizeInBytes(): OptionalLong = {
+ if (!paimonStatsEnabled) {
+ return OptionalLong.of(scannedTotalSize)
+ }
- override def sizeInBytes(): OptionalLong =
- if (paimonStatsEnabled) paimonStats.mergedRecordSize() else
OptionalLong.of(scannedTotalSize)
+ val wholeSchemaSize = getSizeForRow(scan.tableRowType)
+
+ val requiredDataSchemaSize = scan.requiredTableFields.map {
+ field =>
+ val dataField = scan.tableRowType.getField(field.name)
+ getSizeForField(dataField)
+ }.sum
+ val requiredDataSizeInBytes =
+ paimonStats.mergedRecordSize().getAsLong *
(requiredDataSchemaSize.toDouble / wholeSchemaSize)
+
+ val metadataSchemaSize = scan.metadataFields.map {
+ field =>
+ val dataField = PaimonMetadataColumn.get(field.name).toPaimonDataField
+ getSizeForField(dataField)
+ }.sum
+ val metadataSizeInBytes = paimonStats.mergedRecordCount().getAsLong *
metadataSchemaSize
+
+ val sizeInBytes = (requiredDataSizeInBytes + metadataSizeInBytes).toLong
+ // Avoid return 0 bytes if there are some valid rows.
+ // Avoid return too small size in bytes which may less than row count,
+ // note the compression ratio on disk is usually bigger than memory.
+ val normalized = Math.max(sizeInBytes,
paimonStats.mergedRecordCount().getAsLong)
+ OptionalLong.of(normalized)
+ }
override def numRows(): OptionalLong =
if (paimonStatsEnabled) paimonStats.mergedRecordCount() else
OptionalLong.of(rowCount)
diff --git
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
index c7b51fc94..be7579b20 100644
---
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
+++
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
@@ -30,6 +30,8 @@ import
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
import org.assertj.core.api.Assertions.assertThatThrownBy
import org.junit.jupiter.api.Assertions
+import java.util.UUID
+
abstract class AnalyzeTableTestBase extends PaimonSparkTestBase {
test("Paimon analyze: analyze table only") {
@@ -409,6 +411,55 @@ abstract class AnalyzeTableTestBase extends
PaimonSparkTestBase {
})
}
+ test("Fix reported statistics does not do column pruning") {
+ spark.sql("""
+ |CREATE TABLE T (c1 INT, c2 INT, c3 LONG, c4 STRING)
+ |USING PAIMON
+ |TBLPROPERTIES ('primary-key'='c1')
+ |""".stripMargin)
+ spark.sql("ANALYZE TABLE T COMPUTE STATISTICS")
+
+ val wholeSize1 = getScanStatistic("SELECT * FROM T")
+ assert(wholeSize1.rowCount.get.toLong == 0)
+ assert(wholeSize1.sizeInBytes.toLong == 0)
+ val metadataSize1 = getScanStatistic("SELECT __paimon_row_index FROM T")
+ assert(metadataSize1.rowCount.get.toLong == 0)
+ assert(metadataSize1.sizeInBytes.toLong == 0)
+
+ spark.sql(s"INSERT INTO T VALUES (1, 1, 100,
'${UUID.randomUUID().toString()}')")
+ spark.sql(s"INSERT INTO T VALUES (2, 2, 200,
'${UUID.randomUUID().toString()}')")
+ spark.sql(s"INSERT INTO T VALUES (3, 3, 300,
'${UUID.randomUUID().toString()}')")
+
+ def checkStatistics(): Long = {
+ val wholeSize2 = getScanStatistic("SELECT * FROM T")
+ assert(wholeSize2.rowCount.get.toLong == 3)
+ assert(wholeSize2.sizeInBytes.toLong > 0)
+ val wholeSizeWithMetadata = getScanStatistic("SELECT *,
__paimon_file_path FROM T")
+ assert(wholeSizeWithMetadata.rowCount.get.toLong == 3)
+ assert(wholeSizeWithMetadata.sizeInBytes.toLong ==
wholeSize2.sizeInBytes.toLong + 20 * 3)
+
+ val oneColSize = getScanStatistic("SELECT c3 FROM T")
+ val threeColSize = getScanStatistic("SELECT c1, c2, c3 FROM T")
+ val longMetadataSize = getScanStatistic("SELECT __paimon_row_index FROM
T")
+ assert(oneColSize.rowCount.get.toLong == 3)
+ assert(threeColSize.rowCount.get.toLong == 3)
+ assert(longMetadataSize.rowCount.get.toLong == 3)
+ assert(longMetadataSize.sizeInBytes == 8 * 3)
+ assert(oneColSize.sizeInBytes > 0 && oneColSize.sizeInBytes <
wholeSize2.sizeInBytes)
+ assert(threeColSize.sizeInBytes < wholeSize2.sizeInBytes)
+ assert(threeColSize.sizeInBytes > oneColSize.sizeInBytes)
+ wholeSize2.sizeInBytes.toLong
+ }
+
+ spark.sql("ANALYZE TABLE T COMPUTE STATISTICS")
+ val noColStat = checkStatistics()
+
+ spark.sql("ANALYZE TABLE T COMPUTE STATISTICS FOR ALL COLUMNS")
+ val withColStat = checkStatistics()
+
+ assert(withColStat == noColStat)
+ }
+
protected def statsFileCount(tableLocation: Path, fileIO: FileIO): Int = {
fileIO.listStatus(new Path(tableLocation, "statistics")).length
}
