This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 986c200027b7ef79ee5700f222e3d1beba58af27 Author: zhangdong <[email protected]> AuthorDate: Mon Aug 14 13:48:33 2023 +0800 [fix](catalog)paimon support more data type (#22899) --- .../apache/doris/common/jni/vec/ColumnType.java | 11 +- .../org/apache/doris/paimon/PaimonColumnValue.java | 5 +- .../org/apache/doris/paimon/PaimonJniScanner.java | 2 +- .../org/apache/doris/paimon/PaimonTypeUtils.java | 120 --------------------- .../catalog/external/PaimonExternalTable.java | 3 + .../planner/external/paimon/PaimonScanNode.java | 20 ++++ .../external/paimon/PaimonValueConverter.java | 88 ++++++++------- regression-test/conf/regression-conf.groovy | 3 + .../external_table_p2/paimon/paimon_base_types.out | 56 ++++++++++ .../paimon/paimon_base_types.groovy | 79 ++++++++++++++ 10 files changed, 212 insertions(+), 175 deletions(-) diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java index eba69e51c8..1cf7e887d2 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java @@ -268,6 +268,7 @@ public class ColumnType { type = Type.DATEV2; break; case "binary": + case "bytes": type = Type.BINARY; break; case "string": @@ -279,27 +280,27 @@ public class ColumnType { precision = 6; // default Matcher match = digitPattern.matcher(lowerCaseType); if (match.find()) { - precision = Integer.parseInt(match.group(1)); + precision = Integer.parseInt(match.group(1).trim()); } } else if (lowerCaseType.startsWith("char")) { Matcher match = digitPattern.matcher(lowerCaseType); if (match.find()) { type = Type.CHAR; - length = Integer.parseInt(match.group(1)); + length = Integer.parseInt(match.group(1).trim()); } } else if (lowerCaseType.startsWith("varchar")) { Matcher match = digitPattern.matcher(lowerCaseType); if (match.find()) { type = Type.VARCHAR; - length = Integer.parseInt(match.group(1)); + length = Integer.parseInt(match.group(1).trim()); } } else if (lowerCaseType.startsWith("decimal")) { int s = lowerCaseType.indexOf('('); int e = lowerCaseType.indexOf(')'); if (s != -1 && e != -1) { String[] ps = lowerCaseType.substring(s + 1, e).split(","); - precision = Integer.parseInt(ps[0]); - scale = Integer.parseInt(ps[1]); + precision = Integer.parseInt(ps[0].trim()); + scale = Integer.parseInt(ps[1].trim()); if (lowerCaseType.startsWith("decimalv2")) { type = Type.DECIMALV2; } else if (lowerCaseType.startsWith("decimal32")) { diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index 9dff3ea743..a8783fe529 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -109,14 +109,13 @@ public class PaimonColumnValue implements ColumnValue { @Override public LocalDate getDate() { - return Instant.ofEpochMilli(record.getTimestamp(idx, 3) - .getMillisecond()).atZone(ZoneOffset.ofHours(8)).toLocalDate(); + return LocalDate.ofEpochDay(record.getLong(idx)); } @Override public LocalDateTime getDateTime() { return Instant.ofEpochMilli(record.getTimestamp(idx, 3) - .getMillisecond()).atZone(ZoneOffset.ofHours(8)).toLocalDateTime(); + .getMillisecond()).atZone(ZoneOffset.ofHours(0)).toLocalDateTime(); } @Override diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 07841ae92d..3fe4efac9e 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -112,7 +112,7 @@ public class PaimonJniScanner extends JniScanner { fields[i], paimonAllFieldNames)); } DataType dataType = table.rowType().getTypeAt(index); - columnTypes[i] = ColumnType.parseType(fields[i], PaimonTypeUtils.fromPaimonType(dataType)); + columnTypes[i] = ColumnType.parseType(fields[i], dataType.toString()); } super.types = columnTypes; } diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java deleted file mode 100644 index ae3d46c400..0000000000 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.paimon; - -import org.apache.paimon.types.ArrayType; -import org.apache.paimon.types.BigIntType; -import org.apache.paimon.types.BinaryType; -import org.apache.paimon.types.BooleanType; -import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypeDefaultVisitor; -import org.apache.paimon.types.DateType; -import org.apache.paimon.types.DecimalType; -import org.apache.paimon.types.DoubleType; -import org.apache.paimon.types.FloatType; -import org.apache.paimon.types.IntType; -import org.apache.paimon.types.MapType; -import org.apache.paimon.types.RowType; -import org.apache.paimon.types.SmallIntType; -import org.apache.paimon.types.TimestampType; -import org.apache.paimon.types.VarCharType; - -import java.util.stream.Collectors; - -/** - * Convert paimon type to doris string representation. - */ -public class PaimonTypeUtils { - - private PaimonTypeUtils() { - } - - public static String fromPaimonType(DataType type) { - return type.accept(PaimonToDorisTypeVisitor.INSTANCE); - } - - private static class PaimonToDorisTypeVisitor extends DataTypeDefaultVisitor<String> { - - private static final PaimonToDorisTypeVisitor INSTANCE = new PaimonToDorisTypeVisitor(); - - public String visit(VarCharType varCharType) { - return "string"; - } - - public String visit(BooleanType booleanType) { - return "boolean"; - } - - public String visit(BinaryType binaryType) { - return "binary"; - } - - public String visit(DecimalType decimalType) { - return String.format("decimal(%d,%d)", decimalType.getPrecision(), decimalType.getScale()); - } - - public String visit(SmallIntType smallIntType) { - return "short"; - } - - public String visit(IntType intType) { - return "int"; - } - - public String visit(BigIntType bigIntType) { - return "bigint"; - } - - public String visit(FloatType floatType) { - return "float"; - } - - public String visit(DoubleType doubleType) { - return "double"; - } - - public String visit(DateType dateType) { - return "date"; - } - - public String visit(TimestampType timestampType) { - return "timestamp-millis"; - } - - public String visit(ArrayType arrayType) { - return String.format("array<%s>", arrayType.getElementType().accept(this)); - } - - public String visit(MapType mapType) { - return String.format("map<%s,%s>", - mapType.getKeyType().accept(this), - mapType.getValueType().accept(this)); - } - - public String visit(RowType rowType) { - String type = rowType.getFields().stream().map(f -> f.type().accept(this) + ",") - .collect(Collectors.joining(",")); - return String.format("struct<%s>", type.substring(0, type.length() - 1)); - } - - @Override - protected String defaultMethod(DataType dataType) { - return "unsupported_type"; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java index 6a2719053e..6d7c124a0d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java @@ -94,9 +94,12 @@ public class PaimonExternalTable extends ExternalTable { return Type.DOUBLE; case SMALLINT: return Type.SMALLINT; + case TINYINT: + return Type.TINYINT; case VARCHAR: case BINARY: case CHAR: + case VARBINARY: return Type.STRING; case DECIMAL: DecimalType decimal = (DecimalType) dataType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java index 354c25c0e8..be3b04fb33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java @@ -17,6 +17,7 @@ package org.apache.doris.planner.external.paimon; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.external.ExternalTable; @@ -26,6 +27,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.UserException; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.external.FileQueryScanNode; import org.apache.doris.spi.Split; @@ -35,6 +37,7 @@ import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileType; import org.apache.doris.thrift.TPaimonFileDesc; +import org.apache.doris.thrift.TScanRangeLocations; import org.apache.doris.thrift.TTableFormatFileDesc; import avro.shaded.com.google.common.base.Preconditions; @@ -47,6 +50,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; public class PaimonScanNode extends FileQueryScanNode { @@ -119,6 +123,22 @@ public class PaimonScanNode extends FileQueryScanNode { return splits; } + //When calling 'setPaimonParams' and 'getSplits', the column trimming has not been performed yet, + // Therefore, paimon_column_names is temporarily reset here + @Override + public void updateRequiredSlots(PlanTranslatorContext planTranslatorContext, + Set<SlotId> requiredByProjectSlotIdSet) throws UserException { + super.updateRequiredSlots(planTranslatorContext, requiredByProjectSlotIdSet); + String cols = desc.getSlots().stream().map(slot -> slot.getColumn().getName()) + .collect(Collectors.joining(",")); + for (TScanRangeLocations tScanRangeLocations : scanRangeLocations) { + List<TFileRangeDesc> ranges = tScanRangeLocations.scan_range.ext_scan_range.file_scan_range.ranges; + for (TFileRangeDesc tFileRangeDesc : ranges) { + tFileRangeDesc.table_format_params.paimon_params.setPaimonColumnNames(cols); + } + } + } + @Override public TFileType getLocationType() throws DdlException, MetaNotFoundException { return getLocationType(((AbstractFileStoreTable) source.getPaimonTable()).location().toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonValueConverter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonValueConverter.java index 04cf297328..127c4797a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonValueConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonValueConverter.java @@ -23,14 +23,15 @@ import org.apache.doris.analysis.DecimalLiteral; import org.apache.doris.analysis.FloatLiteral; import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.StringLiteral; +import com.google.common.base.Strings; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Decimal; import org.apache.paimon.data.Timestamp; -import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; -import org.apache.paimon.types.BinaryType; import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeDefaultVisitor; import org.apache.paimon.types.DateType; @@ -38,18 +39,15 @@ import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; import org.apache.paimon.types.IntType; -import org.apache.paimon.types.MapType; -import org.apache.paimon.types.RowType; import org.apache.paimon.types.SmallIntType; import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.TinyIntType; import org.apache.paimon.types.VarCharType; import java.math.BigDecimal; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.Date; +import java.time.LocalDate; +import java.util.Calendar; +import java.util.TimeZone; /** * Convert LiteralExpr to paimon value. @@ -65,6 +63,14 @@ public class PaimonValueConverter extends DataTypeDefaultVisitor<Object> { return BinaryString.fromString(expr.getStringValue()); } + public BinaryString visit(CharType charType) { + if (expr instanceof StringLiteral) { + StringLiteral stringLiteral = (StringLiteral) expr; + return BinaryString.fromString(Strings.padEnd(stringLiteral.getStringValue(), charType.getLength(), ' ')); + } + return null; + } + public Boolean visit(BooleanType booleanType) { if (expr instanceof BoolLiteral) { BoolLiteral boolLiteral = (BoolLiteral) expr; @@ -73,10 +79,6 @@ public class PaimonValueConverter extends DataTypeDefaultVisitor<Object> { return null; } - public Object visit(BinaryType binaryType) { - return null; - } - public Decimal visit(DecimalType decimalType) { if (expr instanceof DecimalLiteral) { DecimalLiteral decimalLiteral = (DecimalLiteral) expr; @@ -94,6 +96,15 @@ public class PaimonValueConverter extends DataTypeDefaultVisitor<Object> { return null; } + public Byte visit(TinyIntType tinyIntType) { + if (expr instanceof IntLiteral) { + IntLiteral intLiteral = (IntLiteral) expr; + return (byte) intLiteral.getValue(); + } + return null; + } + + public Integer visit(IntType intType) { if (expr instanceof IntLiteral) { IntLiteral intLiteral = (IntLiteral) expr; @@ -110,11 +121,11 @@ public class PaimonValueConverter extends DataTypeDefaultVisitor<Object> { return null; } + // when a = 9.1,paimon can get data,doris can not get data + // when a > 9.1,paimon can not get data,doris can get data + // paimon is no problem,but we consistent with Doris internal table + // Therefore, comment out this code public Float visit(FloatType floatType) { - if (expr instanceof FloatLiteral) { - FloatLiteral floatLiteral = (FloatLiteral) expr; - return (float) floatLiteral.getValue(); - } return null; } @@ -126,40 +137,25 @@ public class PaimonValueConverter extends DataTypeDefaultVisitor<Object> { return null; } - public Object visit(DateType dateType) { + public Integer visit(DateType dateType) { + if (expr instanceof DateLiteral) { + DateLiteral dateLiteral = (DateLiteral) expr; + long l = LocalDate.of((int) dateLiteral.getYear(), (int) dateLiteral.getMonth(), (int) dateLiteral.getDay()) + .toEpochDay(); + return (int) l; + } return null; } public Timestamp visit(TimestampType timestampType) { - DateLiteral dateLiteral = (DateLiteral) expr; - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss") - .withZone(ZoneId.systemDefault()); - StringBuilder sb = new StringBuilder(); - sb.append(dateLiteral.getYear()) - .append(dateLiteral.getMonth()) - .append(dateLiteral.getDay()) - .append(dateLiteral.getHour()) - .append(dateLiteral.getMinute()) - .append(dateLiteral.getSecond()); - Date date; - try { - date = Date.from( - LocalDateTime.parse(sb.toString(), formatter).atZone(ZoneId.systemDefault()).toInstant()); - } catch (DateTimeParseException e) { - return null; + if (expr instanceof DateLiteral) { + DateLiteral dateLiteral = (DateLiteral) expr; + Calendar instance = Calendar.getInstance(TimeZone.getTimeZone("GMT")); + instance.set((int) dateLiteral.getYear(), (int) (dateLiteral.getMonth() - 1), (int) dateLiteral.getDay(), + (int) dateLiteral.getHour(), (int) dateLiteral.getMinute(), (int) dateLiteral.getSecond()); + return Timestamp + .fromEpochMillis(instance.getTimeInMillis() / 1000 * 1000 + dateLiteral.getMicrosecond() / 1000); } - return Timestamp.fromEpochMillis(date.getTime()); - } - - public Object visit(ArrayType arrayType) { - return null; - } - - public Object visit(MapType mapType) { - return null; - } - - public Object visit(RowType rowType) { return null; } diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 437a6e6340..1c77c9ed97 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -109,6 +109,9 @@ extHdfsPort = 4007 extHiveHmsUser = "****" extHiveHmsPassword= "***********" +//paimon catalog test config for bigdata +enableExternalPaimonTest = false + //mysql jdbc connector test config for bigdata enableExternalMysqlTest = false extMysqlHost = "***.**.**.**" diff --git a/regression-test/data/external_table_p2/paimon/paimon_base_types.out b/regression-test/data/external_table_p2/paimon/paimon_base_types.out new file mode 100644 index 0000000000..59d953b721 --- /dev/null +++ b/regression-test/data/external_table_p2/paimon/paimon_base_types.out @@ -0,0 +1,56 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 +10 20 30 40 50 60 70 80 90.1 100.1 110.10 2020-03-02 130str 140varchar b false bbbb 2023-08-14T08:32:52.821 + +-- !c1 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c2 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c3 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c4 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c5 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c6 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c7 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c8 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c9 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c10 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c11 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c12 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c13 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c14 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c15 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c16 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + +-- !c18 -- +1 2 3 4 5 6 7 8 9.1 10.1 11.10 2020-02-02 13str 14varchar a true aaaa 2023-08-13T09:32:38.530 + diff --git a/regression-test/suites/external_table_p2/paimon/paimon_base_types.groovy b/regression-test/suites/external_table_p2/paimon/paimon_base_types.groovy new file mode 100644 index 0000000000..5d14a7defa --- /dev/null +++ b/regression-test/suites/external_table_p2/paimon/paimon_base_types.groovy @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("paimon_base_types", "p2,external,paimon,external_remote,external_remote_paimon") { + def all = """select * from all_table;""" + def c1 = """select * from all_table where c1=1;""" + def c2 = """select * from all_table where c2=2;""" + def c3 = """select * from all_table where c3=3;""" + def c4 = """select * from all_table where c4=4;""" + def c5 = """select * from all_table where c5=5;""" + def c6 = """select * from all_table where c6=6;""" + def c7 = """select * from all_table where c7=7;""" + def c8 = """select * from all_table where c8=8;""" + def c9 = """select * from all_table where c9<10;""" + def c10 = """select * from all_table where c10=10.1;""" + def c11 = """select * from all_table where c11=11.1;""" + def c12 = """select * from all_table where c12='2020-02-02';""" + def c13 = """select * from all_table where c13='13str';""" + def c14 = """select * from all_table where c14='14varchar';""" + def c15 = """select * from all_table where c15='a';""" + def c16 = """select * from all_table where c16=true;""" + def c18 = """select * from all_table where c18='2023-08-13 09:32:38.53';""" + + String enabled = context.config.otherConfigs.get("enableExternalPaimonTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String catalog_name = "paimon" + String user_name = context.config.otherConfigs.get("extHiveHmsUser") + + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + "type" = "paimon", + "paimon.catalog.type" = "filesystem", + "warehouse" = "hdfs:///paimon/paimon1", + "hadoop.username" = "${user_name}" + ); + """ + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + logger.info("switched to catalog " + catalog_name) + sql """use db1;""" + logger.info("use db1") + + qt_all all + qt_c1 c1 + qt_c2 c2 + qt_c3 c3 + qt_c4 c4 + qt_c5 c5 + qt_c6 c6 + qt_c7 c7 + qt_c8 c8 + qt_c9 c9 + qt_c10 c10 + qt_c11 c11 + qt_c12 c12 + qt_c13 c13 + qt_c14 c14 + qt_c15 c15 + qt_c16 c16 + qt_c18 c18 + + } +} + --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
