This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fc8f6a0715 [fix](multi-catalog) throw NPE when reading data after EOF
(#15358)
fc8f6a0715 is described below
commit fc8f6a0715aae0478be2aa1b80338204117199b4
Author: Ashin Gau <[email protected]>
AuthorDate: Mon Dec 26 22:49:35 2022 +0800
[fix](multi-catalog) throw NPE when reading data after EOF (#15358)
1. Fix 1 bug:
Throw null pointer exception when reading data after the reader reaches the
end of file, so should return directly when `_do_lazy_read` read no data.
2. Optimize code:
Remove unused parameters.
3. Fix regression test
---
.../exec/format/parquet/vparquet_column_reader.cpp | 1 -
.../exec/format/parquet/vparquet_column_reader.h | 2 +-
.../exec/format/parquet/vparquet_group_reader.cpp | 3 +-
.../hive/test_external_brown.out | 62 +++++++++++-----------
.../hive/test_external_brown.groovy | 11 ++--
.../hive/test_external_catalog_hive.groovy | 3 +-
6 files changed, 43 insertions(+), 39 deletions(-)
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index a7d95225b6..da08d1299e 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -27,7 +27,6 @@
namespace doris::vectorized {
Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
- const ParquetReadColumn& column,
const tparquet::RowGroup& row_group,
const std::vector<RowRange>& row_ranges,
cctz::time_zone* ctz,
std::unique_ptr<ParquetColumnReader>&
reader,
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h
b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
index 0f61c5742c..8dff684730 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
@@ -108,7 +108,7 @@ public:
ColumnSelectVector& select_vector, size_t
batch_size,
size_t* read_rows, bool* eof) = 0;
static Status create(io::FileReaderSPtr file, FieldSchema* field,
- const ParquetReadColumn& column, const
tparquet::RowGroup& row_group,
+ const tparquet::RowGroup& row_group,
const std::vector<RowRange>& row_ranges,
cctz::time_zone* ctz,
std::unique_ptr<ParquetColumnReader>& reader, size_t
max_buf_size);
void init_column_metadata(const tparquet::ColumnChunk& chunk);
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
index b406919b4c..b88f6981b3 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
@@ -57,7 +57,7 @@ Status RowGroupReader::init(const FieldDescriptor& schema,
std::vector<RowRange>
for (auto& read_col : _read_columns) {
auto field =
const_cast<FieldSchema*>(schema.get_column(read_col._file_slot_name));
std::unique_ptr<ParquetColumnReader> reader;
- RETURN_IF_ERROR(ParquetColumnReader::create(_file_reader, field,
read_col, _row_group_meta,
+ RETURN_IF_ERROR(ParquetColumnReader::create(_file_reader, field,
_row_group_meta,
_read_ranges, _ctz,
reader, max_buf_size));
auto col_iter = col_offsets.find(read_col._parquet_col_id);
if (col_iter != col_offsets.end()) {
@@ -233,6 +233,7 @@ Status RowGroupReader::_do_lazy_read(Block* block, size_t
batch_size, size_t* re
DCHECK_EQ(pre_read_rows + _cached_filtered_rows, 0);
*read_rows = 0;
*batch_eof = true;
+ return Status::OK();
}
ColumnSelectVector& select_vector = *select_vector_ptr;
diff --git
a/regression-test/data/external_table_emr_p2/hive/test_external_brown.out
b/regression-test/data/external_table_emr_p2/hive/test_external_brown.out
index c4dca4f351..58936b7593 100644
--- a/regression-test/data/external_table_emr_p2/hive/test_external_brown.out
+++ b/regression-test/data/external_table_emr_p2/hive/test_external_brown.out
@@ -769,17 +769,17 @@ cslab9e 2017-01-10T14:19:45
2019-12-01T09:59:58 16C448031108ABA balcony_5 door 5
temperature F 32.0
-- !15 --
+balcony_3 door 3 SUMMER
balcony_3 door 3 WINTER
+balcony_5 door 5 SUMMER
balcony_5 door 5 WINTER
+front_center_1 door 1 SUMMER
front_center_1 door 1 WINTER
+front_left_1 door 1 SUMMER
front_left_1 door 1 WINTER
+front_right_1 door 1 SUMMER
front_right_1 door 1 WINTER
loading_dock_1 door 1 WINTER
-balcony_3 door 3 SUMMER
-balcony_5 door 5 SUMMER
-front_center_1 door 1 SUMMER
-front_left_1 door 1 SUMMER
-front_right_1 door 1 SUMMER
-- !01 --
anansi 0E-9 22.499999744 0.142686086 938.950000640
2355335.637499904 9177.160292675 216.880005120 2354536.236711936
13467.316295748
@@ -1551,17 +1551,17 @@ cslab9e 2017-01-10T14:19:45
2019-12-01T09:59:58 16C448031108ABA balcony_5 door 5
temperature F 32.0
-- !15 --
+balcony_3 door 3 SUMMER
balcony_3 door 3 WINTER
+balcony_5 door 5 SUMMER
balcony_5 door 5 WINTER
+front_center_1 door 1 SUMMER
front_center_1 door 1 WINTER
+front_left_1 door 1 SUMMER
front_left_1 door 1 WINTER
+front_right_1 door 1 SUMMER
front_right_1 door 1 WINTER
loading_dock_1 door 1 WINTER
-balcony_3 door 3 SUMMER
-balcony_5 door 5 SUMMER
-front_left_1 door 1 SUMMER
-front_center_1 door 1 SUMMER
-front_right_1 door 1 SUMMER
-- !01 --
anansi 0E-9 22.499999744 0.142686086 938.950000640
2355335.637499904 9177.160292675 216.880005120 2354536.236711936
13467.316295748
@@ -2302,25 +2302,25 @@ cslab9e 2017-01-10T14:19:45
2018 1 800695.0232504869 1268.9303062606766
52913.861798515994 83.85715023536608 36392.48058457171
83.46899216644888 191628.70119383492 303.69049317564964
2018 2 788872.057316979 1188.060327284607
85621.32358955599 128.94777649029515 83922.79400835953
176.6795663333885 230227.25133465935 346.7277881546074
2018 3 896816.2190436664 1205.3981438758956
88325.94803945151 118.71767209603698 114322.79839365493
193.76745490449989 269070.6060912007 361.65404044516225
-2018 4 581954.6102904765 1199.9064129700546
55146.13846078819 113.70337826966637 107909.63419773088
260.6512903326833 172977.56835847016 356.654780120557
-2018 5 810269.3225697601 1246.5681885688618
69687.03583794385 107.21082436606747 83644.78159173668
143.4730387508348 248284.77976738225 381.97658425751115
-2018 6 824150.0588949091 1252.5076882901353
52047.11873700281 79.09896464590094 61491.79513703259
113.24455826341176 242561.98528653922 368.63523599777994
-2018 7 920527.341169839 1237.2679316798913
54001.80526436993 72.58307159189506 110136.29401019629
175.65597130812804 284977.680542893 383.03451685872716
-2018 8 934028.5503494724 1255.4147182116565
56534.790159531614 75.98762118216615 18440.748190359904
37.25403674820183 281409.03078009805 378.237944596906
-2018 9 796294.7250120321 1201.047850696881
59750.62790355883 90.12161071426671 123357.5146659909
219.10748608524136 251368.59077414422 379.1381459640184
-2018 10 876930.3483784577 1178.6698230893248
68192.54789305248 91.65665039388774 161682.95582566143
259.94044344961645 285017.38156064076 383.08787844172144
-2018 11 220190.80577936108 1177.4909399965834
17258.738257541878 92.2927179547694 31282.504059750238
215.7414073086223 74832.72439675416 400.1749967740864
-2018 12 600604.6369429262 1213.3427008948004
40572.28465858767 81.96421143149024 45621.90108822518
103.21697078783978 174784.20606314813 353.099406188178
-2019 1 875861.8015196375 1177.2336041930612
62356.908033973974 83.81304843276072 38427.20028916451
68.37580122627136 266395.02450662013 358.0578286379303
-2019 2 766839.5051314116 1141.1302159693626
58769.95527064852 87.4552905813222 98888.76720553763
174.40699683516337 263572.6687568533 392.22123326912697
+2018 4 581954.6102904766 1199.9064129700548
55146.138460788185 113.70337826966636 107909.63419773089
260.6512903326833 172977.56835847016 356.654780120557
+2018 5 810269.3225697605 1246.5681885688623
69687.03583794385 107.21082436606747 83644.78159173668
143.4730387508348 248284.77976738242 381.97658425751143
+2018 6 824150.0588949089 1252.507688290135
52047.118737002835 79.09896464590096 61491.79513703265
113.24455826341188 242561.98528653936 368.63523599778017
+2018 7 920527.3411698382 1237.2679316798901
54001.80526436992 72.58307159189505 110136.29401019629
175.65597130812804 284977.68054289324 383.0345168587275
+2018 8 934028.5503494727 1255.414718211657
56534.79015953164 75.9876211821662 18440.748190359904
37.25403674820183 281409.03078009794 378.23794459690583
+2018 9 796294.7250120323 1201.0478506968814
59750.627903558845 90.12161071426674 123357.51466599082
219.10748608524125 251368.5907741443 379.1381459640186
+2018 10 876930.3483784578 1178.669823089325
68192.5478930525 91.65665039388777 161682.95582566143
259.94044344961645 285017.38156064047 383.08787844172105
+2018 11 220190.80577936105 1177.4909399965832
17258.73825754189 92.29271795476946 31282.504059750245
215.74140730862237 74832.7243967542 400.17499677408665
+2018 12 600604.6369429264 1213.3427008948008
40572.284658587676 81.96421143149026 45621.901088225204
103.21697078783983 174784.206063148 353.0994061881778
+2019 1 875861.801519637 1177.2336041930605
62356.90803397393 83.81304843276065 38427.200289164524
68.37580122627139 266395.0245066205 358.05782863793075
+2019 2 766839.5051314117 1141.1302159693628
58769.95527064857 87.45529058132227 98888.76720553766
174.40699683516343 263572.6687568533 392.22123326912697
2019 3 875531.8723175366 1176.790150964431
64134.01856961811 86.20163786238993 137670.795483173
220.9804100853499 307147.8785608932 412.8331701087274
-2019 4 882405.7822994062 1225.563586526953
62498.88624107809 86.80400866816402 118655.91830720674
190.7651419730012 310192.1270515332 430.822398682685
+2019 4 882405.782299406 1225.5635865269528
62498.88624107809 86.80400866816402 118655.91830720675
190.76514197300122 310192.1270515332 430.822398682685
2019 5 915804.9453381414 1230.920625454491
62733.50787805464 84.31923101889065 51908.476725653585
92.36383759013093 333347.07849929936 448.04714852056367
-2019 6 743714.4735556246 1264.820533257865
47914.37351050592 81.48702977977197 33701.15766617421
99.1210519593359 258663.9435447939 439.9046658925066
-2019 7 936697.9580959912 1259.0026318494506
60426.1178931879 81.21790039406976 10876.808554053554
30.213357094593206 322797.22597334185 433.867239211481
+2019 6 743714.4735556253 1264.8205332578661
47914.37351050592 81.48702977977197 33701.15766617421
99.1210519593359 258663.9435447939 439.9046658925066
+2019 7 936697.9580959912 1259.0026318494506
60426.11789318791 81.21790039406977 10876.808554053552
30.2133570945932 322797.22597334185 433.867239211481
2019 8 930640.7550409784 1250.8612298937883
60639.08568100215 81.50414742070181 13063.088331067778
44.58391921866136 336075.4530245259 451.714318581352
2019 9 862784.4318728528 1198.3117109345178
63348.64890275069 87.98423458715374 56973.39566326609
126.32681965247471 322162.84531000606 447.4483962638973
-2019 10 900170.2429565523 1209.9062405330003
64953.12136266859 87.3025824767051 98168.02805722525
218.63703353502282 310577.0949153714 417.4423318754992
+2019 10 900170.2429565524 1209.9062405330005
64953.1213626686 87.30258247670511 98168.02805722521
218.63703353502274 310577.0949153713 417.442331875499
2019 11 881836.9061517053 1224.7734807662573
61941.828695957716 86.0303176332746 181716.3770488152
294.99411858573893 258322.18736711307 358.78081578765705
2019 12 45118.354529795746 1253.2876258276597
2884.8668441554055 80.13519011542793 13576.481448950073
377.1244846930576 12249.12485575384 340.25346821538443
@@ -2333,15 +2333,15 @@ cslab9e 2017-01-10T14:19:45
2019-12-01T09:59:58 16C448031108ABA balcony_5 door 5
temperature F 32.0
-- !15 --
-front_center_1 door 1 WINTER
-front_left_1 door 1 WINTER
-front_right_1 door 1 WINTER
+balcony_3 door 3 SUMMER
balcony_3 door 3 WINTER
+balcony_5 door 5 SUMMER
balcony_5 door 5 WINTER
-loading_dock_1 door 1 WINTER
front_center_1 door 1 SUMMER
+front_center_1 door 1 WINTER
front_left_1 door 1 SUMMER
+front_left_1 door 1 WINTER
front_right_1 door 1 SUMMER
-balcony_3 door 3 SUMMER
-balcony_5 door 5 SUMMER
+front_right_1 door 1 WINTER
+loading_dock_1 door 1 WINTER
diff --git
a/regression-test/suites/external_table_emr_p2/hive/test_external_brown.groovy
b/regression-test/suites/external_table_emr_p2/hive/test_external_brown.groovy
index 8ba09f0596..455d0333f4 100644
---
a/regression-test/suites/external_table_emr_p2/hive/test_external_brown.groovy
+++
b/regression-test/suites/external_table_emr_p2/hive/test_external_brown.groovy
@@ -38,7 +38,8 @@ suite("test_external_brown", "p2") {
WHERE machine_name IN ('anansi','aragog','urd')
AND log_time >= TIMESTAMP '2017-01-11 00:00:00'
) AS r
- GROUP BY machine_name;"""
+ GROUP BY machine_name
+ ORDER BY machine_name;"""
def averageMetrics = """
SELECT dt,
hr,
@@ -172,7 +173,8 @@ suite("test_external_brown", "p2") {
AND status_code < 300
AND request LIKE '%/etc/passwd%'
AND log_time >= TIMESTAMP '2012-05-06 00:00:00'
- AND log_time < TIMESTAMP '2012-05-20 00:00:00';"""
+ AND log_time < TIMESTAMP '2012-05-20 00:00:00'
+ ORDER BY log_time;"""
def powerConsumptionMetrics = """
SELECT yr,
mo,
@@ -225,7 +227,8 @@ suite("test_external_brown", "p2") {
FROM logs3SUFFIX
WHERE event_type = 'temperature'
AND event_value <= 32.0
- AND log_time >= '2019-11-29 17:00:00.000';"""
+ AND log_time >= '2019-11-29 17:00:00.000'
+ ORDER BY log_time;"""
def temperatureVariation_order = """
WITH temperature AS (
SELECT dt,
@@ -318,7 +321,7 @@ suite("test_external_brown", "p2") {
qt_12 powerConsumptionMetrics.replace("SUFFIX", format)
qt_13 serverError.replace("SUFFIX", format)
qt_14 temperatureReachFreezing.replace("SUFFIX", format)
- qt_15 temperatureVariation_order.replace("SUFFIX", format)
+ order_qt_15 temperatureVariation_order.replace("SUFFIX", format)
}
}
}
diff --git
a/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
b/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
index a73ba0062f..7df45eab5d 100644
---
a/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
+++
b/regression-test/suites/external_table_emr_p2/hive/test_external_catalog_hive.groovy
@@ -97,11 +97,12 @@ suite("test_external_catalog_hive", "p2") {
sql """switch internal;"""
+ sql """drop catalog if exists hms"""
sql """alter catalog ${catalog_name} rename hms;"""
sql """switch hms;"""
- def res3 = sql """select count(*) from hive_test limit 10;"""
+ def res3 = sql """select count(*) from test.hive_test limit 10;"""
logger.info("recoding select: " + res3.toString())
sql """alter catalog hms rename ${catalog_name};"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
