This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 049410596f4 [feature](http) Add structured type metadata to table
schema API (#66670)
049410596f4 is described below
commit 049410596f4d8c0a5c23bb5519e6b8f09dc63ab4
Author: Alan Tang <[email protected]>
AuthorDate: Wed Sep 2 15:13:32 2026 +0800
[feature](http) Add structured type metadata to table schema API (#66670)
### What problem does this PR solve?
Issue Number: close #66675
Related PR: #xxx
Problem Summary:
The table schema HTTP API previously returned only the top-level
primitive type name through the `type` field. This is insufficient for
complex types such as ARRAY, MAP, and STRUCT because clients cannot
reliably reconstruct nested type definitions, nullability, or scalar
attributes.
This PR extends the schema response with two additive fields while
preserving all existing fields:
- `type_sql`: the complete SQL representation of the column type.
- `type_desc`: a recursively structured type description.
`type_desc` contains:
- Common attributes: `kind` and `sql`.
- Decimal attributes: `precision` and `scale`.
- CHAR/VARCHAR attribute: `length`.
- DATETIMEV2/TIMEV2/TIMESTAMPTZ attribute: `scale`.
- ARRAY attributes: `contains_null` and `element`.
- MAP attributes: `key_contains_null`, `value_contains_null`, `key`, and
`value`.
- STRUCT attributes: ordered `fields`, including field name,
nullability, and recursive type metadata.
The new metadata is returned for both base table schema columns and
materialized index columns.
The existing `type`, `precision`, `scale`, and other response fields
remain unchanged for backward compatibility.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [x] Yes. The table schema API response includes the additive
`type_sql` and `type_desc` fields. Existing fields and their values are
preserved.
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---------
Signed-off-by: StandingMan <[email protected]>
---
.../doris/httpv2/rest/TableSchemaAction.java | 20 +-
.../doris/httpv2/rest/response/SchemaTypeDesc.java | 234 ++++++++++++++++
.../apache/doris/http/TableSchemaActionTest.java | 12 +
.../rest/TableSchemaActionColumnInfoTest.java | 172 ++++++++++++
.../httpv2/rest/response/SchemaTypeDescTest.java | 306 +++++++++++++++++++++
.../http_rest_api/get/test_schema_api.groovy | 58 +++-
6 files changed, 793 insertions(+), 9 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
index fc486d43c5d..9f06ba6e115 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
@@ -36,10 +36,12 @@ import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
import org.apache.doris.httpv2.rest.response.GsonSchemaResponse;
+import org.apache.doris.httpv2.rest.response.SchemaTypeDesc;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.ConnectContext;
+import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import jakarta.servlet.http.HttpServletRequest;
@@ -70,8 +72,9 @@ public class TableSchemaAction extends RestBaseController {
* @param column the column to build info for
* @return map containing column information
*/
- private Map<String, String> buildColumnInfo(Column column) {
- Map<String, String> columnInfo = new HashMap<>(2);
+ @VisibleForTesting
+ static Map<String, Object> buildColumnInfo(Column column) {
+ Map<String, Object> columnInfo = new HashMap<>();
Type colType = column.getOriginType();
PrimitiveType primitiveType = colType.getPrimitiveType();
@@ -83,6 +86,11 @@ public class TableSchemaAction extends RestBaseController {
columnInfo.put("column_uid", String.valueOf(column.getUniqueId()));
columnInfo.put("type", primitiveType.toString());
+ SchemaTypeDesc typeDesc = SchemaTypeDesc.fromType(colType);
+ if (typeDesc.getSql() != null) {
+ columnInfo.put("type_sql", typeDesc.getSql());
+ }
+ columnInfo.put("type_desc", typeDesc);
columnInfo.put("comment", column.getComment());
columnInfo.put("name", column.getDisplayName());
@@ -127,9 +135,9 @@ public class TableSchemaAction extends RestBaseController {
try {
try {
List<Column> columns = table.getBaseSchema();
- List<Map<String, String>> propList = new
ArrayList(columns.size());
+ List<Map<String, Object>> propList = new
ArrayList<>(columns.size());
for (Column column : columns) {
- Map<String, String> baseInfo = buildColumnInfo(column);
+ Map<String, Object> baseInfo = buildColumnInfo(column);
propList.add(baseInfo);
}
resultMap.put("status", 200);
@@ -156,10 +164,10 @@ public class TableSchemaAction extends RestBaseController
{
// Get schema columns for this materialized index
List<Column> indexColumns = indexMeta.getSchema();
- List<Map<String, String>> indexColumnList = new
ArrayList<>();
+ List<Map<String, Object>> indexColumnList = new
ArrayList<>();
for (Column column : indexColumns) {
- Map<String, String> columnInfo =
buildColumnInfo(column);
+ Map<String, Object> columnInfo =
buildColumnInfo(column);
indexColumnList.add(columnInfo);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/response/SchemaTypeDesc.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/response/SchemaTypeDesc.java
new file mode 100644
index 00000000000..74e46337314
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/response/SchemaTypeDesc.java
@@ -0,0 +1,234 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.httpv2.rest.response;
+
+import org.apache.doris.catalog.AggStateType;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.StructField;
+import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.catalog.VariantField;
+import org.apache.doris.catalog.VariantType;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import lombok.Getter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Recursively structured type description for table schema HTTP APIs.
+ *
+ * <p>Only attributes that apply to a type are serialized. Complex types point
to child
+ * {@code SchemaTypeDesc} instances, allowing clients to traverse arbitrarily
nested ARRAY, MAP,
+ * STRUCT, VARIANT, and AGG_STATE definitions. JSON property names use snake
case to match the
+ * surrounding schema API.</p>
+ */
+@Getter
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+public class SchemaTypeDesc {
+ // Doris primitive type name. Decimal V3 storage widths remain
distinguishable, for example
+ // DECIMAL32 and DECIMAL64.
+ private final String kind;
+
+ // Complete SQL representation intended for display and compatibility
fallback. Unsupported
+ // types have no valid SQL representation, so this field is null for them.
Clients should prefer
+ // the structured fields below when interpreting a type.
+ private final String sql;
+
+ // Scalar attributes. Null means that the attribute does not apply to this
type.
+ private Integer precision;
+ private Integer scale;
+ private Integer length;
+
+ // ARRAY attributes. Doris ARRAY elements are always nullable, so
containsNull is always true
+ // and is emitted for symmetry with MAP. element is the recursively
described item type.
+ private Boolean containsNull;
+ private SchemaTypeDesc element;
+
+ // MAP attributes. Nullability and type metadata are reported
independently for keys and values.
+ private Boolean keyContainsNull;
+ private Boolean valueContainsNull;
+ private SchemaTypeDesc key;
+ private SchemaTypeDesc value;
+
+ // STRUCT attributes in declaration order.
+ private List<StructFieldDesc> fields;
+
+ // VARIANT predefined fields in declaration order. Dynamic fields
discovered at runtime are not
+ // part of the catalog type and therefore are not included.
+ private List<VariantFieldDesc> predefinedFields;
+
+ // AGG_STATE attributes. Each subtype keeps its nullability next to its
recursive type metadata.
+ private String functionName;
+ private Boolean resultIsNullable;
+ private List<AggStateSubTypeDesc> subTypes;
+
+ private SchemaTypeDesc(Type type) {
+ this.kind = type.getPrimitiveType().toString();
+ this.sql = type.isSupported() ? type.toSql() : null;
+ }
+
+ /**
+ * Converts a Doris catalog type into its public recursive schema
representation.
+ *
+ * <p>The primitive name is present on every node. The SQL form is present
only for supported
+ * types. Type-specific metadata is added only on nodes where it is
meaningful, so Jackson can
+ * omit unrelated fields from the response.</p>
+ *
+ * @param type catalog type to expose through the schema API
+ * @return recursively structured description of {@code type}
+ */
+ public static SchemaTypeDesc fromType(Type type) {
+ SchemaTypeDesc result = new SchemaTypeDesc(type);
+ PrimitiveType primitiveType = type.getPrimitiveType();
+ switch (primitiveType) {
+ case ARRAY:
+ ArrayType arrayType = (ArrayType) type;
+ result.containsNull = arrayType.getContainsNull();
+ result.element = fromType(arrayType.getItemType());
+ break;
+ case MAP:
+ MapType mapType = (MapType) type;
+ result.keyContainsNull = mapType.getIsKeyContainsNull();
+ result.valueContainsNull = mapType.getIsValueContainsNull();
+ result.key = fromType(mapType.getKeyType());
+ result.value = fromType(mapType.getValueType());
+ break;
+ case STRUCT:
+ StructType structType = (StructType) type;
+ result.fields = structType.getFields().stream()
+ .map(StructFieldDesc::fromField)
+ .collect(Collectors.toList());
+ break;
+ case VARIANT:
+ VariantType variantType = (VariantType) type;
+ result.predefinedFields =
variantType.getPredefinedFields().stream()
+ .map(VariantFieldDesc::fromField)
+ .collect(Collectors.toList());
+ break;
+ case AGG_STATE:
+ AggStateType aggStateType = (AggStateType) type;
+ result.functionName = aggStateType.getFunctionName();
+ result.resultIsNullable = aggStateType.getResultIsNullable();
+ result.subTypes = AggStateSubTypeDesc.fromType(aggStateType);
+ break;
+ case DECIMALV2:
+ case DECIMAL32:
+ case DECIMAL64:
+ case DECIMAL128:
+ case DECIMAL256:
+ ScalarType decimalType = (ScalarType) type;
+ result.precision = decimalType.getPrecision();
+ result.scale = decimalType.getScalarScale();
+ break;
+ case CHAR:
+ case VARCHAR:
+ case VARBINARY:
+ result.length = ((ScalarType) type).getLength();
+ break;
+ case DATETIMEV2:
+ case TIMEV2:
+ case TIMESTAMPTZ:
+ result.scale = ((ScalarType) type).getScalarScale();
+ break;
+ default:
+ break;
+ }
+ return result;
+ }
+
+ /**
+ * One named child of a STRUCT type.
+ *
+ * <p>Field order is preserved by the containing list. {@code
containsNull} describes the
+ * nested field itself, while {@code type} recursively describes its value
type.</p>
+ */
+ @Getter
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+ public static class StructFieldDesc {
+ private final String name;
+ private final boolean containsNull;
+ private final String comment;
+ private final SchemaTypeDesc type;
+
+ private StructFieldDesc(StructField field) {
+ this.name = field.getName();
+ this.containsNull = field.getContainsNull();
+ this.comment = field.isCommentSpecified() ? field.getComment() :
null;
+ this.type = SchemaTypeDesc.fromType(field.getType());
+ }
+
+ private static StructFieldDesc fromField(StructField field) {
+ return new StructFieldDesc(field);
+ }
+ }
+
+ /** One predefined field pattern of a VARIANT type. */
+ @Getter
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+ public static class VariantFieldDesc {
+ private final String pattern;
+ private final String patternType;
+ private final String comment;
+ private final SchemaTypeDesc type;
+
+ private VariantFieldDesc(VariantField field) {
+ this.pattern = field.getPattern();
+ this.patternType = field.getPatternType().toString();
+ this.comment = field.getComment();
+ this.type = SchemaTypeDesc.fromType(field.getType());
+ }
+
+ private static VariantFieldDesc fromField(VariantField field) {
+ return new VariantFieldDesc(field);
+ }
+ }
+
+ /** One recursively described input type of an AGG_STATE type. */
+ @Getter
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
+ public static class AggStateSubTypeDesc {
+ private final boolean containsNull;
+ private final SchemaTypeDesc type;
+
+ private AggStateSubTypeDesc(Type type, boolean containsNull) {
+ this.containsNull = containsNull;
+ this.type = SchemaTypeDesc.fromType(type);
+ }
+
+ private static List<AggStateSubTypeDesc> fromType(AggStateType type) {
+ List<AggStateSubTypeDesc> subTypes = new
ArrayList<>(type.getSubTypes().size());
+ for (int i = 0; i < type.getSubTypes().size(); i++) {
+ subTypes.add(new AggStateSubTypeDesc(
+ type.getSubTypes().get(i),
type.getSubTypeNullables().get(i)));
+ }
+ return subTypes;
+ }
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java
index 7e2347f61f4..5a9690a04ef 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java
@@ -47,5 +47,17 @@ public class TableSchemaActionTest extends DorisHttpTestCase
{
JSONArray propArray = (JSONArray) ((JSONObject)
object.get("data")).get("properties");
// k1, k2
Assert.assertEquals(2, propArray.size());
+
+ JSONObject column = (JSONObject) propArray.get(0);
+ Assert.assertEquals("BIGINT", column.get("type"));
+ Assert.assertEquals("bigint", column.get("type_sql"));
+ Assert.assertEquals("BIGINT", ((JSONObject)
column.get("type_desc")).get("kind"));
+
+ JSONObject materializedIndexes =
+ (JSONObject) ((JSONObject)
object.get("data")).get("materialized_indexes");
+ JSONObject baseIndex = (JSONObject)
materializedIndexes.get("testIndex");
+ JSONArray indexColumns = (JSONArray) baseIndex.get("columns");
+ Assert.assertEquals("BIGINT",
+ ((JSONObject) ((JSONObject)
indexColumns.get(0)).get("type_desc")).get("kind"));
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/TableSchemaActionColumnInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/TableSchemaActionColumnInfoTest.java
new file mode 100644
index 00000000000..20e2c5e3be7
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/TableSchemaActionColumnInfoTest.java
@@ -0,0 +1,172 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.httpv2.rest;
+
+import org.apache.doris.catalog.AggStateType;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.StructField;
+import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.catalog.VariantField;
+import org.apache.doris.catalog.VariantType;
+import org.apache.doris.httpv2.rest.response.SchemaTypeDesc;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+public class TableSchemaActionColumnInfoTest {
+
+ @Test
+ public void testArrayDecimalColumnInfo() {
+ Column column = new Column("array_decimal",
+ new ArrayType(ScalarType.createDecimalV3Type(18, 4)), false,
null, true, null,
+ "nested decimal");
+ Map<String, Object> columnInfo =
TableSchemaAction.buildColumnInfo(column);
+
+ Assertions.assertEquals("array_decimal", columnInfo.get("name"));
+ Assertions.assertEquals("ARRAY", columnInfo.get("type"));
+ Assertions.assertEquals("array<decimalv3(18,4)>",
columnInfo.get("type_sql"));
+ Assertions.assertEquals("Yes", columnInfo.get("is_nullable"));
+ Assertions.assertEquals("No", columnInfo.get("is_key"));
+ Assertions.assertEquals("nested decimal", columnInfo.get("comment"));
+ Assertions.assertFalse(columnInfo.containsKey("precision"));
+ Assertions.assertFalse(columnInfo.containsKey("scale"));
+
+ SchemaTypeDesc typeDesc = (SchemaTypeDesc) columnInfo.get("type_desc");
+ Assertions.assertEquals("ARRAY", typeDesc.getKind());
+ Assertions.assertTrue(typeDesc.getContainsNull());
+ Assertions.assertEquals("DECIMAL64", typeDesc.getElement().getKind());
+ Assertions.assertEquals(Integer.valueOf(18),
typeDesc.getElement().getPrecision());
+ Assertions.assertEquals(Integer.valueOf(4),
typeDesc.getElement().getScale());
+
+ JsonNode json = new ObjectMapper().valueToTree(columnInfo);
+ Assertions.assertEquals("ARRAY", json.path("type").asText());
+ assertBooleanField(json.path("type_desc"), "contains_null", true);
+ Assertions.assertEquals(4,
json.path("type_desc").path("element").path("scale").asInt());
+ }
+
+ @Test
+ public void testScalarDecimalKeepsLegacyAttributes() {
+ Column column = new Column("amount",
ScalarType.createDecimalV3Type(18, 4));
+ Map<String, Object> columnInfo =
TableSchemaAction.buildColumnInfo(column);
+
+ Assertions.assertEquals("DECIMAL64", columnInfo.get("type"));
+ Assertions.assertEquals("18", columnInfo.get("precision"));
+ Assertions.assertEquals("4", columnInfo.get("scale"));
+ SchemaTypeDesc typeDesc = (SchemaTypeDesc) columnInfo.get("type_desc");
+ Assertions.assertEquals(Integer.valueOf(18), typeDesc.getPrecision());
+ Assertions.assertEquals(Integer.valueOf(4), typeDesc.getScale());
+ }
+
+ @Test
+ public void testVarbinaryColumnInfoContainsStructuredLength() {
+ Map<String, Object> columnInfo = TableSchemaAction.buildColumnInfo(
+ new Column("payload", ScalarType.createVarbinaryType(64)));
+
+ Assertions.assertEquals("VARBINARY", columnInfo.get("type"));
+ Assertions.assertEquals("varbinary(64)", columnInfo.get("type_sql"));
+
+ JsonNode json = new ObjectMapper().valueToTree(columnInfo);
+ Assertions.assertEquals("VARBINARY",
json.path("type_desc").path("kind").asText());
+ Assertions.assertEquals("varbinary(64)",
json.path("type_desc").path("sql").asText());
+ Assertions.assertEquals(64,
json.path("type_desc").path("length").asInt());
+ }
+
+ @Test
+ public void testVariantAndAggStateColumnInfoContainsStructuredChildren() {
+ VariantType variantType = new VariantType(Lists.newArrayList(
+ new VariantField("event_id", Type.BIGINT, "event
identifier")));
+ JsonNode variantJson = new
ObjectMapper().valueToTree(TableSchemaAction.buildColumnInfo(
+ new Column("payload", variantType)));
+
+ JsonNode variantField =
variantJson.path("type_desc").path("predefined_fields").path(0);
+ Assertions.assertEquals("event_id",
variantField.path("pattern").asText());
+ Assertions.assertEquals("BIGINT",
variantField.path("type").path("kind").asText());
+
+ AggStateType aggStateType = new AggStateType("sum", true,
+ Lists.newArrayList(Type.BIGINT), Lists.newArrayList(false));
+ JsonNode aggStateJson = new
ObjectMapper().valueToTree(TableSchemaAction.buildColumnInfo(
+ new Column("sum_state", aggStateType)));
+
+ JsonNode typeDesc = aggStateJson.path("type_desc");
+ Assertions.assertEquals("sum",
typeDesc.path("function_name").asText());
+ assertBooleanField(typeDesc, "result_is_nullable", true);
+ assertBooleanField(typeDesc.path("sub_types").path(0),
"contains_null", false);
+ Assertions.assertEquals("BIGINT",
typeDesc.path("sub_types").path(0).path("type").path("kind").asText());
+ }
+
+ @Test
+ public void testUnsupportedColumnInfoOmitsSql() {
+ Map<String, Object> columnInfo = TableSchemaAction.buildColumnInfo(
+ new Column("geometry", Type.UNSUPPORTED));
+
+ Assertions.assertEquals("UNSUPPORTED_TYPE", columnInfo.get("type"));
+ Assertions.assertFalse(columnInfo.containsKey("type_sql"));
+
+ SchemaTypeDesc typeDesc = (SchemaTypeDesc) columnInfo.get("type_desc");
+ Assertions.assertEquals("UNSUPPORTED_TYPE", typeDesc.getKind());
+ Assertions.assertNull(typeDesc.getSql());
+
+ JsonNode json = new ObjectMapper().valueToTree(columnInfo);
+ Assertions.assertFalse(json.has("type_sql"));
+ Assertions.assertEquals("UNSUPPORTED_TYPE",
json.path("type_desc").path("kind").asText());
+ Assertions.assertFalse(json.path("type_desc").has("sql"));
+ }
+
+ @Test
+ public void testStructAndMapColumnInfo() {
+ MapType mapType = new MapType(Type.STRING,
+ new ArrayType(ScalarType.createDecimalV3Type(9, 2)), false,
true);
+ StructType structType = new StructType(Lists.newArrayList(
+ new StructField("attributes", mapType, "map attributes",
false),
+ new StructField("tags", new ArrayType(Type.STRING))));
+ Map<String, Object> columnInfo = TableSchemaAction.buildColumnInfo(
+ new Column("detail", structType));
+
+ SchemaTypeDesc structDesc = (SchemaTypeDesc)
columnInfo.get("type_desc");
+ Assertions.assertEquals("STRUCT", structDesc.getKind());
+ Assertions.assertEquals(2, structDesc.getFields().size());
+ Assertions.assertFalse(structDesc.getFields().get(0).isContainsNull());
+ Assertions.assertEquals("map attributes",
structDesc.getFields().get(0).getComment());
+ Assertions.assertNull(structDesc.getFields().get(1).getComment());
+
+ SchemaTypeDesc mapDesc = structDesc.getFields().get(0).getType();
+ Assertions.assertEquals("MAP", mapDesc.getKind());
+ Assertions.assertFalse(mapDesc.getKeyContainsNull());
+ Assertions.assertTrue(mapDesc.getValueContainsNull());
+ Assertions.assertEquals("STRING", mapDesc.getKey().getKind());
+ Assertions.assertEquals("DECIMAL32",
mapDesc.getValue().getElement().getKind());
+ Assertions.assertEquals(Integer.valueOf(2),
mapDesc.getValue().getElement().getScale());
+ }
+
+ private void assertBooleanField(JsonNode node, String fieldName, boolean
expected) {
+ Assertions.assertTrue(node.has(fieldName), "Missing field: " +
fieldName);
+ JsonNode value = node.get(fieldName);
+ Assertions.assertTrue(value.isBoolean(), "Field is not boolean: " +
fieldName);
+ Assertions.assertEquals(expected, value.booleanValue(),
+ "Unexpected value for field: " + fieldName);
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/response/SchemaTypeDescTest.java
b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/response/SchemaTypeDescTest.java
new file mode 100644
index 00000000000..ae98e696dc3
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/response/SchemaTypeDescTest.java
@@ -0,0 +1,306 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.httpv2.rest.response;
+
+import org.apache.doris.catalog.AggStateType;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.PatternType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.StructField;
+import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.catalog.VariantField;
+import org.apache.doris.catalog.VariantType;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class SchemaTypeDescTest {
+
+ @Test
+ public void testNestedType() {
+ ScalarType decimalType = ScalarType.createDecimalV3Type(18, 4);
+ StructType structType = new StructType(Lists.newArrayList(
+ new StructField("price", decimalType, "", false)));
+ SchemaTypeDesc typeDesc = SchemaTypeDesc.fromType(new
ArrayType(structType));
+
+ Assertions.assertEquals("ARRAY", typeDesc.getKind());
+ Assertions.assertTrue(typeDesc.getContainsNull());
+ Assertions.assertEquals("STRUCT", typeDesc.getElement().getKind());
+ Assertions.assertEquals(1, typeDesc.getElement().getFields().size());
+
+ SchemaTypeDesc.StructFieldDesc field =
typeDesc.getElement().getFields().get(0);
+ Assertions.assertEquals("price", field.getName());
+ Assertions.assertFalse(field.isContainsNull());
+ Assertions.assertEquals("DECIMAL64", field.getType().getKind());
+ Assertions.assertEquals(Integer.valueOf(18),
field.getType().getPrecision());
+ Assertions.assertEquals(Integer.valueOf(4),
field.getType().getScale());
+ }
+
+ @Test
+ public void testMapType() {
+ MapType mapType = new MapType(Type.STRING,
+ new ArrayType(ScalarType.createDecimalV3Type(9, 2)), false,
true);
+ SchemaTypeDesc typeDesc = SchemaTypeDesc.fromType(mapType);
+
+ Assertions.assertEquals("MAP", typeDesc.getKind());
+ Assertions.assertFalse(typeDesc.getKeyContainsNull());
+ Assertions.assertTrue(typeDesc.getValueContainsNull());
+ Assertions.assertEquals("STRING", typeDesc.getKey().getKind());
+ Assertions.assertEquals("ARRAY", typeDesc.getValue().getKind());
+ Assertions.assertEquals(Integer.valueOf(9),
typeDesc.getValue().getElement().getPrecision());
+ Assertions.assertEquals(Integer.valueOf(2),
typeDesc.getValue().getElement().getScale());
+ }
+
+ @Test
+ public void testDeeplyNestedArrayMapStructType() {
+ StructType leafStruct = new StructType(Lists.newArrayList(
+ new StructField("id", Type.BIGINT, "", false),
+ new StructField("amounts",
+ new ArrayType(ScalarType.createDecimalV3Type(18, 4)),
"", true)));
+ MapType mapType = new MapType(ScalarType.createVarcharType(16),
leafStruct, false, true);
+ SchemaTypeDesc typeDesc = SchemaTypeDesc.fromType(new
ArrayType(mapType));
+
+ Assertions.assertEquals("ARRAY", typeDesc.getKind());
+ Assertions.assertTrue(typeDesc.getContainsNull());
+
+ SchemaTypeDesc mapDesc = typeDesc.getElement();
+ Assertions.assertEquals("MAP", mapDesc.getKind());
+ Assertions.assertFalse(mapDesc.getKeyContainsNull());
+ Assertions.assertTrue(mapDesc.getValueContainsNull());
+ Assertions.assertEquals(Integer.valueOf(16),
mapDesc.getKey().getLength());
+
+ SchemaTypeDesc structDesc = mapDesc.getValue();
+ Assertions.assertEquals("STRUCT", structDesc.getKind());
+ Assertions.assertEquals(2, structDesc.getFields().size());
+ Assertions.assertFalse(structDesc.getFields().get(0).isContainsNull());
+ Assertions.assertEquals("BIGINT",
structDesc.getFields().get(0).getType().getKind());
+ Assertions.assertTrue(structDesc.getFields().get(1).isContainsNull());
+
+ SchemaTypeDesc amountsDesc = structDesc.getFields().get(1).getType();
+ Assertions.assertEquals("ARRAY", amountsDesc.getKind());
+ Assertions.assertTrue(amountsDesc.getContainsNull());
+ Assertions.assertEquals("DECIMAL64",
amountsDesc.getElement().getKind());
+ Assertions.assertEquals(Integer.valueOf(18),
amountsDesc.getElement().getPrecision());
+ Assertions.assertEquals(Integer.valueOf(4),
amountsDesc.getElement().getScale());
+ }
+
+ @Test
+ public void testNestedArraysPreserveElementChain() {
+ Type nestedArrays = new ArrayType(new ArrayType(new
ArrayType(Type.INT)));
+ SchemaTypeDesc outer = SchemaTypeDesc.fromType(nestedArrays);
+
+ Assertions.assertEquals("ARRAY", outer.getKind());
+ Assertions.assertEquals("ARRAY", outer.getElement().getKind());
+ Assertions.assertEquals("ARRAY",
outer.getElement().getElement().getKind());
+ Assertions.assertEquals("INT",
outer.getElement().getElement().getElement().getKind());
+ }
+
+ @Test
+ public void testDeeplyNestedJsonSerialization() {
+ StructType leafStruct = new StructType(Lists.newArrayList(
+ new StructField("created_at",
ScalarType.createDatetimeV2Type(6), "", false)));
+ Type nestedType = new StructType(Lists.newArrayList(
+ new StructField("events",
+ new ArrayType(new MapType(Type.STRING, leafStruct,
false, true)),
+ "", true)));
+ JsonNode json = new
ObjectMapper().valueToTree(SchemaTypeDesc.fromType(nestedType));
+
+ JsonNode eventsField = json.path("fields").path(0);
+ Assertions.assertEquals("events", eventsField.path("name").asText());
+ assertBooleanField(eventsField, "contains_null", true);
+
+ JsonNode map = eventsField.path("type").path("element");
+ assertBooleanField(map, "key_contains_null", false);
+ assertBooleanField(map, "value_contains_null", true);
+ Assertions.assertEquals("STRING",
map.path("key").path("kind").asText());
+
+ JsonNode createdAt = map.path("value").path("fields").path(0);
+ Assertions.assertEquals("created_at", createdAt.path("name").asText());
+ assertBooleanField(createdAt, "contains_null", false);
+ Assertions.assertEquals("DATETIMEV2",
createdAt.path("type").path("kind").asText());
+ Assertions.assertEquals(6,
createdAt.path("type").path("scale").asInt());
+ Assertions.assertFalse(createdAt.path("type").has("fields"));
+ Assertions.assertFalse(createdAt.path("type").has("element"));
+ }
+
+ @Test
+ public void testStructFieldComments() {
+ StructType structType = new StructType(Lists.newArrayList(
+ new StructField("documented", Type.INT, "unit price", true),
+ new StructField("undocumented", Type.INT),
+ new StructField("empty_comment", Type.INT, "", true, true)));
+ JsonNode fields = new ObjectMapper().valueToTree(
+ SchemaTypeDesc.fromType(structType)).path("fields");
+
+ Assertions.assertEquals("unit price",
fields.path(0).path("comment").asText());
+ Assertions.assertFalse(fields.path(1).has("comment"));
+ Assertions.assertTrue(fields.path(2).has("comment"));
+ Assertions.assertEquals("", fields.path(2).path("comment").asText());
+ }
+
+ @Test
+ public void testScalarAttributes() {
+ SchemaTypeDesc character =
SchemaTypeDesc.fromType(ScalarType.createCharType(16));
+ SchemaTypeDesc varchar =
SchemaTypeDesc.fromType(ScalarType.createVarcharType(32));
+ SchemaTypeDesc varbinary =
SchemaTypeDesc.fromType(ScalarType.createVarbinaryType(64));
+ SchemaTypeDesc datetime =
SchemaTypeDesc.fromType(ScalarType.createDatetimeV2Type(3));
+ SchemaTypeDesc time =
SchemaTypeDesc.fromType(ScalarType.createTimeV2Type(5));
+ SchemaTypeDesc timestamp =
SchemaTypeDesc.fromType(ScalarType.createTimeStampTzType(6));
+
+ Assertions.assertEquals(Integer.valueOf(16), character.getLength());
+ Assertions.assertEquals(Integer.valueOf(32), varchar.getLength());
+ Assertions.assertEquals(Integer.valueOf(64), varbinary.getLength());
+ Assertions.assertEquals(Integer.valueOf(3), datetime.getScale());
+ Assertions.assertEquals(Integer.valueOf(5), time.getScale());
+ Assertions.assertEquals(Integer.valueOf(6), timestamp.getScale());
+
Assertions.assertNull(SchemaTypeDesc.fromType(Type.STRING).getLength());
+ }
+
+ @Test
+ public void testDecimalStorageKinds() {
+ assertDecimalType(ScalarType.createDecimalV3Type(9, 1), "DECIMAL32",
9, 1);
+ assertDecimalType(ScalarType.createDecimalV3Type(18, 2), "DECIMAL64",
18, 2);
+ assertDecimalType(ScalarType.createDecimalV3Type(38, 3), "DECIMAL128",
38, 3);
+ }
+
+ @Test
+ public void testJsonUsesSnakeCaseAndOmitsNullFields() {
+ SchemaTypeDesc typeDesc = SchemaTypeDesc.fromType(
+ new ArrayType(ScalarType.createDecimalV3Type(18, 4)));
+ JsonNode json = new ObjectMapper().valueToTree(typeDesc);
+
+ assertBooleanField(json, "contains_null", true);
+ Assertions.assertFalse(json.has("containsNull"));
+ Assertions.assertFalse(json.has("precision"));
+ Assertions.assertEquals("array<decimalv3(18,4)>",
json.path("sql").asText());
+ Assertions.assertEquals("decimalv3(18,4)",
json.path("element").path("sql").asText());
+ Assertions.assertEquals(18,
json.path("element").path("precision").asInt());
+ Assertions.assertEquals(4, json.path("element").path("scale").asInt());
+ }
+
+ @Test
+ public void testVarbinaryJsonContainsLength() {
+ JsonNode json = new ObjectMapper().valueToTree(
+ SchemaTypeDesc.fromType(ScalarType.createVarbinaryType(64)));
+
+ Assertions.assertEquals("VARBINARY", json.path("kind").asText());
+ Assertions.assertEquals("varbinary(64)", json.path("sql").asText());
+ Assertions.assertEquals(64, json.path("length").asInt());
+ Assertions.assertEquals(3, json.size());
+ }
+
+ @Test
+ public void testVariantPredefinedFieldsAreStructuredRecursively() {
+ VariantType variantType = new VariantType(Lists.newArrayList(
+ new VariantField("amount", ScalarType.createDecimalV3Type(18,
4),
+ "monetary value", PatternType.MATCH_NAME),
+ new VariantField("tags_*", new
ArrayType(ScalarType.createVarcharType(32)),
+ "", PatternType.MATCH_NAME_GLOB)));
+ JsonNode json = new
ObjectMapper().valueToTree(SchemaTypeDesc.fromType(variantType));
+
+ JsonNode amount = json.path("predefined_fields").path(0);
+ Assertions.assertEquals("amount", amount.path("pattern").asText());
+ Assertions.assertEquals("MATCH_NAME",
amount.path("pattern_type").asText());
+ Assertions.assertEquals("monetary value",
amount.path("comment").asText());
+ Assertions.assertEquals("DECIMAL64",
amount.path("type").path("kind").asText());
+ Assertions.assertEquals(18,
amount.path("type").path("precision").asInt());
+ Assertions.assertEquals(4, amount.path("type").path("scale").asInt());
+
+ JsonNode tags = json.path("predefined_fields").path(1);
+ Assertions.assertEquals("tags_*", tags.path("pattern").asText());
+ Assertions.assertEquals("MATCH_NAME_GLOB",
tags.path("pattern_type").asText());
+ Assertions.assertEquals("ARRAY",
tags.path("type").path("kind").asText());
+ Assertions.assertEquals(32,
tags.path("type").path("element").path("length").asInt());
+ }
+
+ @Test
+ public void testAggStateSubTypesAreStructuredRecursively() {
+ AggStateType aggStateType = new AggStateType("weighted_sum", false,
+ Lists.newArrayList(Type.INT,
+ new ArrayType(ScalarType.createDecimalV3Type(18, 4))),
+ Lists.newArrayList(true, false));
+ JsonNode json = new
ObjectMapper().valueToTree(SchemaTypeDesc.fromType(aggStateType));
+
+ Assertions.assertEquals("AGG_STATE", json.path("kind").asText());
+ Assertions.assertEquals("weighted_sum",
json.path("function_name").asText());
+ assertBooleanField(json, "result_is_nullable", false);
+
+ JsonNode integer = json.path("sub_types").path(0);
+ assertBooleanField(integer, "contains_null", true);
+ Assertions.assertEquals("INT",
integer.path("type").path("kind").asText());
+
+ JsonNode amounts = json.path("sub_types").path(1);
+ assertBooleanField(amounts, "contains_null", false);
+ Assertions.assertEquals("ARRAY",
amounts.path("type").path("kind").asText());
+ Assertions.assertEquals(18,
amounts.path("type").path("element").path("precision").asInt());
+ Assertions.assertEquals(4,
amounts.path("type").path("element").path("scale").asInt());
+ }
+
+ @Test
+ public void testPrimitiveJsonOnlyContainsKindAndSql() {
+ JsonNode json = new
ObjectMapper().valueToTree(SchemaTypeDesc.fromType(Type.BIGINT));
+
+ Assertions.assertEquals(2, json.size());
+ Assertions.assertEquals("BIGINT", json.path("kind").asText());
+ Assertions.assertEquals("bigint", json.path("sql").asText());
+ }
+
+ @Test
+ public void testUnsupportedTypeOmitsSql() {
+ SchemaTypeDesc unsupported = SchemaTypeDesc.fromType(Type.UNSUPPORTED);
+ JsonNode json = new ObjectMapper().valueToTree(unsupported);
+
+ Assertions.assertEquals("UNSUPPORTED_TYPE", unsupported.getKind());
+ Assertions.assertNull(unsupported.getSql());
+ Assertions.assertEquals(1, json.size());
+ Assertions.assertEquals("UNSUPPORTED_TYPE",
json.path("kind").asText());
+ Assertions.assertFalse(json.has("sql"));
+ }
+
+ @Test
+ public void testComplexTypeContainingUnsupportedTypeOmitsSql() {
+ SchemaTypeDesc array = SchemaTypeDesc.fromType(new
ArrayType(Type.UNSUPPORTED));
+ JsonNode json = new ObjectMapper().valueToTree(array);
+
+ Assertions.assertNull(array.getSql());
+ Assertions.assertNull(array.getElement().getSql());
+ Assertions.assertFalse(json.has("sql"));
+ Assertions.assertFalse(json.path("element").has("sql"));
+ Assertions.assertEquals("UNSUPPORTED_TYPE",
json.path("element").path("kind").asText());
+ }
+
+ private void assertBooleanField(JsonNode node, String fieldName, boolean
expected) {
+ Assertions.assertTrue(node.has(fieldName), "Missing field: " +
fieldName);
+ JsonNode value = node.get(fieldName);
+ Assertions.assertTrue(value.isBoolean(), "Field is not boolean: " +
fieldName);
+ Assertions.assertEquals(expected, value.booleanValue(),
+ "Unexpected value for field: " + fieldName);
+ }
+
+ private void assertDecimalType(ScalarType type, String kind, int
precision, int scale) {
+ SchemaTypeDesc typeDesc = SchemaTypeDesc.fromType(type);
+ Assertions.assertEquals(kind, typeDesc.getKind());
+ Assertions.assertEquals(Integer.valueOf(precision),
typeDesc.getPrecision());
+ Assertions.assertEquals(Integer.valueOf(scale), typeDesc.getScale());
+ }
+}
diff --git a/regression-test/suites/http_rest_api/get/test_schema_api.groovy
b/regression-test/suites/http_rest_api/get/test_schema_api.groovy
index 474b7a5a5ea..974786466dd 100644
--- a/regression-test/suites/http_rest_api/get/test_schema_api.groovy
+++ b/regression-test/suites/http_rest_api/get/test_schema_api.groovy
@@ -33,7 +33,13 @@ suite("test_schema_api") {
`c2` date NOT NULL COMMENT "date columns",
`c3` VARCHAR(20) COMMENT "nullable columns",
`c4` VARCHAR COMMENT "varchar columns",
- `c5` BIGINT DEFAULT "0" COMMENT "test columns"
+ `c5` BIGINT DEFAULT "0" COMMENT "test columns",
+ `c6` ARRAY<DECIMAL(18, 4)> COMMENT "array column",
+ `c7` MAP<VARCHAR(16), BIGINT> COMMENT "map column",
+ `c8` STRUCT<
+ price:DECIMAL(18, 4) COMMENT 'unit price',
+ tags:ARRAY<VARCHAR(32)>
+ > COMMENT "struct column"
)
UNIQUE KEY(`id`)
DISTRIBUTED BY HASH(`id`) BUCKETS 8
@@ -62,7 +68,54 @@ suite("test_schema_api") {
assertEquals(result.msg, "success")
// parsing
def resultList = result.data.properties
- assertTrue(resultList.size() == 6)
+ assertEquals(9, resultList.size())
+
+ def columns = resultList.collectEntries { [(it.name): it] }
+
+ def arrayColumn = columns.c6
+ assertEquals("ARRAY", arrayColumn.type)
+ assertEquals("array<decimalv3(18,4)>", arrayColumn.type_sql)
+ def arrayDesc = arrayColumn.type_desc
+ assertEquals("ARRAY", arrayDesc.kind)
+ assertTrue(arrayDesc.containsKey("contains_null"))
+ assertFalse(arrayDesc.containsKey("containsNull"))
+ assertTrue(arrayDesc.contains_null)
+ assertEquals("DECIMAL64", arrayDesc.element.kind)
+ assertEquals(18, arrayDesc.element.precision)
+ assertEquals(4, arrayDesc.element.scale)
+
+ def mapColumn = columns.c7
+ assertEquals("MAP", mapColumn.type)
+ assertEquals("map<varchar(16),bigint>", mapColumn.type_sql)
+ def mapDesc = mapColumn.type_desc
+ assertEquals("MAP", mapDesc.kind)
+ assertTrue(mapDesc.containsKey("key_contains_null"))
+ assertTrue(mapDesc.containsKey("value_contains_null"))
+ assertFalse(mapDesc.containsKey("keyContainsNull"))
+ assertFalse(mapDesc.containsKey("valueContainsNull"))
+ assertTrue(mapDesc.key_contains_null)
+ assertTrue(mapDesc.value_contains_null)
+ assertEquals("VARCHAR", mapDesc.key.kind)
+ assertEquals(16, mapDesc.key.length)
+ assertEquals("BIGINT", mapDesc.value.kind)
+
+ def structColumn = columns.c8
+ assertEquals("STRUCT", structColumn.type)
+ def structDesc = structColumn.type_desc
+ assertEquals("STRUCT", structDesc.kind)
+ def structFields = structDesc.fields.collectEntries { [(it.name): it] }
+ assertTrue(structFields.price.containsKey("contains_null"))
+ assertFalse(structFields.price.containsKey("containsNull"))
+ assertTrue(structFields.price.contains_null)
+ assertEquals("unit price", structFields.price.comment)
+ assertEquals("DECIMAL64", structFields.price.type.kind)
+ assertEquals(18, structFields.price.type.precision)
+ assertEquals(4, structFields.price.type.scale)
+ assertEquals("ARRAY", structFields.tags.type.kind)
+ assertTrue(structFields.tags.type.containsKey("contains_null"))
+ assertFalse(structFields.tags.type.containsKey("containsNull"))
+ assertEquals("VARCHAR", structFields.tags.type.element.kind)
+ assertEquals(32, structFields.tags.type.element.length)
// not exist catalog
def url2 = String.format("http://%s/api/%s/%s/%s/_schema";,
context.config.feHttpAddress, "notexistctl", thisDb, tbName)
@@ -71,4 +124,3 @@ suite("test_schema_api") {
assertTrue(result2.data.contains("Unknown catalog"))
}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
