This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f6b47c34b3 [improvement](stats) show stats with updated time (#21377)
f6b47c34b3 is described below
commit f6b47c34b3825bbc223bc51a307ef014b5755e81
Author: AKIRA <[email protected]>
AuthorDate: Tue Jul 25 11:22:08 2023 +0800
[improvement](stats) show stats with updated time (#21377)
Support to view the stats updated time.
After
```sql
mysql> show column stats t1;
+-------------+-------+------+----------+-----------+---------------+------+------+---------------------+
| column_name | count | ndv | num_null | data_size | avg_size_byte | min
| max | updated_time |
+-------------+-------+------+----------+-----------+---------------+------+------+---------------------+
| col2 | 2.0 | 2.0 | 0.0 | 0.0 | 0.0 | 2
| 5 | 2023-06-30 15:50:24 |
| col3 | 2.0 | 2.0 | 0.0 | 0.0 | 0.0 | 3
| 6 | 2023-06-30 15:50:48 |
| col1 | 2.0 | 2.0 | 0.0 | 0.0 | 0.0 | '1'
| '4' | 2023-06-30 15:50:48 |
+-------------+-------+------+----------+-----------+---------------+------+------+---------------------+
```
Before
```sql
mysql> show column stats t1;
+-------------+-------+------+----------+-----------+---------------+------+------+
| column_name | count | ndv | num_null | data_size | avg_size_byte | min
| max |
+-------------+-------+------+----------+-----------+---------------+------+------+
| col2 | 2.0 | 2.0 | 0.0 | 0.0 | 0.0 | 2
| 5 |
| col3 | 2.0 | 2.0 | 0.0 | 0.0 | 0.0 | 3
| 6 |
| col1 | 2.0 | 2.0 | 0.0 | 0.0 | 0.0 | '1'
| '4' |
+-------------+-------+------+----------+-----------+---------------+------+------+
```
---
.../org/apache/doris/analysis/ShowColumnStatsStmt.java | 2 ++
.../org/apache/doris/statistics/ColumnStatistic.java | 10 ++++++++--
.../apache/doris/statistics/ColumnStatisticBuilder.java | 16 ++++++++++++++--
.../doris/statistics/util/InternalQueryResult.java | 4 ++--
.../org/apache/doris/nereids/util/HyperGraphBuilder.java | 4 +++-
.../test/java/org/apache/doris/statistics/CacheTest.java | 10 ++++++++--
.../apache/doris/statistics/StatsDeriveResultTest.java | 4 +++-
7 files changed, 40 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
index 0ebf2050ed..982480cb00 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
@@ -56,6 +56,7 @@ public class ShowColumnStatsStmt extends ShowStmt {
.add("avg_size_byte")
.add("min")
.add("max")
+ .add("updated_time")
.build();
private final TableName tableName;
@@ -148,6 +149,7 @@ public class ShowColumnStatsStmt extends ShowStmt {
row.add(String.valueOf(p.second.avgSizeByte));
row.add(String.valueOf(p.second.minExpr == null ? "N/A" :
p.second.minExpr.toSql()));
row.add(String.valueOf(p.second.maxExpr == null ? "N/A" :
p.second.maxExpr.toSql()));
+ row.add(String.valueOf(p.second.updatedTime));
result.add(row);
});
return new ShowResultSet(getMetaData(), result);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index e60d7d8697..4d9a4ecf20 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -121,9 +121,12 @@ public class ColumnStatistic {
public final Map<Long, ColumnStatistic> partitionIdToColStats = new
HashMap<>();
+ public final String updatedTime;
+
public ColumnStatistic(double count, double ndv, ColumnStatistic original,
double avgSizeByte,
double numNulls, double dataSize, double minValue, double maxValue,
- double selectivity, LiteralExpr minExpr, LiteralExpr maxExpr,
boolean isUnKnown, Histogram histogram) {
+ double selectivity, LiteralExpr minExpr, LiteralExpr maxExpr,
boolean isUnKnown, Histogram histogram,
+ String updatedTime) {
this.count = count;
this.ndv = ndv;
this.original = original;
@@ -137,6 +140,7 @@ public class ColumnStatistic {
this.maxExpr = maxExpr;
this.isUnKnown = isUnKnown;
this.histogram = histogram;
+ this.updatedTime = updatedTime;
}
public static ColumnStatistic fromResultRow(List<ResultRow> resultRows) {
@@ -205,6 +209,7 @@ public class ColumnStatistic {
Histogram histogram =
Env.getCurrentEnv().getStatisticsCache().getHistogram(tblId, idxId, colName)
.orElse(null);
columnStatisticBuilder.setHistogram(histogram);
+
columnStatisticBuilder.setUpdatedTime(resultRow.getColumnValue("update_time"));
return columnStatisticBuilder.build();
} catch (Exception e) {
LOG.warn("Failed to deserialize column statistics, column not
exists", e);
@@ -393,7 +398,8 @@ public class ColumnStatistic {
null,
null,
stat.getBoolean("IsUnKnown"),
- Histogram.deserializeFromJson(stat.getString("Histogram"))
+ Histogram.deserializeFromJson(stat.getString("Histogram")),
+ stat.getString("lastUpdatedTine")
);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
index f69ab56815..df1ddb50fe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticBuilder.java
@@ -37,6 +37,8 @@ public class ColumnStatisticBuilder {
private ColumnStatistic original;
+ private String updatedTime;
+
public ColumnStatisticBuilder() {
}
@@ -54,6 +56,7 @@ public class ColumnStatisticBuilder {
this.isUnknown = columnStatistic.isUnKnown;
this.histogram = columnStatistic.histogram;
this.original = columnStatistic.original;
+ this.updatedTime = columnStatistic.updatedTime;
}
public ColumnStatisticBuilder setCount(double count) {
@@ -169,13 +172,22 @@ public class ColumnStatisticBuilder {
return this;
}
+ public String getUpdatedTime() {
+ return updatedTime;
+ }
+
+ public void setUpdatedTime(String updatedTime) {
+ this.updatedTime = updatedTime;
+ }
+
public ColumnStatistic build() {
dataSize = Math.max((count - numNulls + 1) * avgSizeByte, 0);
if (original == null && !isUnknown) {
original = new ColumnStatistic(count, ndv, null, avgSizeByte,
numNulls,
- dataSize, minValue, maxValue, selectivity, minExpr,
maxExpr, false, histogram);
+ dataSize, minValue, maxValue, selectivity, minExpr,
maxExpr, false,
+ histogram, updatedTime);
}
return new ColumnStatistic(count, ndv, original, avgSizeByte, numNulls,
- dataSize, minValue, maxValue, selectivity, minExpr, maxExpr,
isUnknown, histogram);
+ dataSize, minValue, maxValue, selectivity, minExpr, maxExpr,
isUnknown, histogram, updatedTime);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQueryResult.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQueryResult.java
index 422f7af4a9..e791986010 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQueryResult.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/InternalQueryResult.java
@@ -100,7 +100,7 @@ public class InternalQueryResult {
List<PrimitiveType> types = getTypes();
int index = getColumnIndex(columnName);
if (index == -1) {
- throw new DdlException("The column name does not exist.");
+ throw new DdlException(String.format("The column name:[%s]
does not exist.", columnName));
}
return types.get(index);
}
@@ -117,7 +117,7 @@ public class InternalQueryResult {
public String getColumnValue(String columnName) throws DdlException {
int index = getColumnIndex(columnName);
if (index == -1) {
- throw new DdlException("The column name does not exist.");
+ throw new DdlException(String.format("The column name:[%s]
does not exist.", columnName));
}
return values.get(index);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java
index ce76b58183..7fe8b97977 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/HyperGraphBuilder.java
@@ -42,6 +42,7 @@ import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -312,7 +313,8 @@ public class HyperGraphBuilder {
for (Slot slot : scanPlan.getOutput()) {
slotIdToColumnStats.put(slot,
new ColumnStatistic(count, count, null, 1, 0, 0, 0,
- count, 1, null, null, true, null));
+ count, 1, null, null, true, null,
+ new Date().toString()));
}
return new Statistics(count, slotIdToColumnStats);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
index 6ac8c7432a..d6570ecebc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
@@ -44,6 +44,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Date;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
@@ -102,6 +103,7 @@ public class CacheTest extends TestWithFeService {
colNames.add("min");
colNames.add("max");
colNames.add("part_id");
+ colNames.add("update_time");
List<PrimitiveType> primitiveTypes = new ArrayList<>();
primitiveTypes.add(PrimitiveType.BIGINT);
primitiveTypes.add(PrimitiveType.BIGINT);
@@ -114,7 +116,8 @@ public class CacheTest extends TestWithFeService {
primitiveTypes.add(PrimitiveType.VARCHAR);
primitiveTypes.add(PrimitiveType.VARCHAR);
primitiveTypes.add(PrimitiveType.VARCHAR);
- primitiveTypes.add(PrimitiveType.BIGINT);
+ primitiveTypes.add(PrimitiveType.VARCHAR);
+ primitiveTypes.add(PrimitiveType.VARCHAR);
List<String> values = new ArrayList<>();
values.add("1");
values.add("2");
@@ -128,6 +131,7 @@ public class CacheTest extends TestWithFeService {
values.add("9");
values.add("10");
values.add(null);
+ values.add(new Date().toString());
ResultRow resultRow = new ResultRow(colNames, primitiveTypes,
values);
return Arrays.asList(resultRow);
}
@@ -285,7 +289,9 @@ public class CacheTest extends TestWithFeService {
result = table;
table.getColumnStatistic("col");
- result = new ColumnStatistic(1, 2, null, 3, 4, 5, 6, 7, 8,
null, null, false, null);
+ result = new ColumnStatistic(1, 2,
+ null, 3, 4, 5, 6, 7, 8,
+ null, null, false, null, new Date().toString());
}
};
StatisticsCache statisticsCache = new StatisticsCache();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatsDeriveResultTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatsDeriveResultTest.java
index 6c157bece6..78872a547d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatsDeriveResultTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatsDeriveResultTest.java
@@ -22,12 +22,14 @@ import org.apache.doris.common.Id;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.Date;
+
public class StatsDeriveResultTest {
@Test
public void testUpdateRowCountByLimit() {
StatsDeriveResult stats = new StatsDeriveResult(100);
ColumnStatistic a = new ColumnStatistic(100, 10, null, 1, 5, 10,
- 1, 100, 0.5, null, null, false, null);
+ 1, 100, 0.5, null, null, false, null, new Date().toString());
Id id = new Id(1);
stats.addColumnStats(id, a);
StatsDeriveResult res = stats.updateByLimit(0);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
