This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 51e4ab00f46 HIVE-29547: Iceberg: [V3] Handle nested and missing Struct
while using defaults (#6409)
51e4ab00f46 is described below
commit 51e4ab00f4699f1a038cc1278c2a6d653a0371ab
Author: Ayush Saxena <[email protected]>
AuthorDate: Tue Apr 7 04:06:40 2026 +0530
HIVE-29547: Iceberg: [V3] Handle nested and missing Struct while using
defaults (#6409)
---
.../apache/iceberg/hive/HiveSchemaConverter.java | 2 +-
.../org/apache/iceberg/hive/HiveSchemaUtil.java | 112 +++++++++++++++------
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 3 +-
.../mr/hive/writer/HiveIcebergDefaultWriter.java | 11 +-
.../positive/iceberg_alter_default_column.q | 8 +-
.../test/queries/positive/iceberg_default_column.q | 31 +++++-
.../queries/positive/iceberg_initial_default.q | 12 +--
.../positive/iceberg_alter_default_column.q.out | 16 +--
.../results/positive/iceberg_default_column.q.out | 84 ++++++++++++++--
.../results/positive/iceberg_initial_default.q.out | 24 ++---
10 files changed, 222 insertions(+), 81 deletions(-)
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
index 0f993714284..1b743343163 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
@@ -163,7 +163,7 @@ Type convertType(TypeInfo typeInfo, String defaultValue) {
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
List<Types.NestedField> fields =
convertInternal(structTypeInfo.getAllStructFieldNames(),
structTypeInfo.getAllStructFieldTypeInfos(),
- HiveSchemaUtil.getDefaultValuesMap(defaultValue),
Collections.emptyList());
+ HiveSchemaUtil.getDefaultValuesMap(null, defaultValue),
Collections.emptyList());
return Types.StructType.of(fields);
case MAP:
MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
index 1fbbb9c68dd..b1563040bcb 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
@@ -25,18 +25,17 @@
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
-import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hive.iceberg.com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.hive.iceberg.com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
import org.apache.iceberg.expressions.Literal;
-import org.apache.iceberg.relocated.com.google.common.base.Splitter;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.types.Conversions;
@@ -48,6 +47,8 @@
public final class HiveSchemaUtil {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
private HiveSchemaUtil() {
}
@@ -237,7 +238,7 @@ private static void getDefaultValDiff(Types.NestedField
field, Map<String, Strin
difference.addDefaultChanged(field, expectedDefault);
}
} else if (field.type().isStructType()) {
- Map<String, String> structDefaults = getDefaultValuesMap(defaultStr);
+ Map<String, String> structDefaults =
getDefaultValuesMap(field.type().asStructType().fields(), defaultStr);
for (Types.NestedField nested : field.type().asStructType().fields()) {
getDefaultValDiff(nested, structDefaults, difference);
@@ -413,31 +414,48 @@ public static String convertToTypeString(Type type) {
}
}
- public static void setDefaultValues(Record record, List<Types.NestedField>
fields, Set<String> missingColumns) {
- for (Types.NestedField field : fields) {
- Object fieldValue = record.getField(field.name());
+ public static void setDefaultValues(Record record, List<Types.NestedField>
missingFields) {
+ for (Types.NestedField field : missingFields) {
+ if (field.type().isStructType()) {
+ // Attempt to build the nested struct with its defaults
+ Record nestedRecord =
buildStructWithDefaults(field.type().asStructType());
+ if (nestedRecord != null) {
+ record.setField(field.name(), nestedRecord);
+ }
+ } else if (field.writeDefault() != null) {
+ Object defaultValue = convertToWriteType(field.writeDefault(),
field.type());
+ record.setField(field.name(), defaultValue);
+ }
+ }
+ }
- if (fieldValue == null) {
- boolean isMissing = missingColumns.contains(field.name());
-
- if (isMissing) {
- if (field.type().isStructType()) {
- // Create struct and apply defaults to all nested fields
- Record nestedRecord =
GenericRecord.create(field.type().asStructType());
- record.setField(field.name(), nestedRecord);
- // For nested fields, we consider ALL fields as "missing" to apply
defaults
- setDefaultValuesForNestedStruct(nestedRecord,
field.type().asStructType().fields());
- } else if (field.writeDefault() != null) {
- Object defaultValue = convertToWriteType(field.writeDefault(),
field.type());
- record.setField(field.name(), defaultValue);
- }
+ /**
+ * Recursively builds a struct populated with write defaults.
+ * * @return A populated Record, or null if no nested fields have defaults.
+ */
+ private static Record buildStructWithDefaults(Types.StructType structType) {
+ Record nestedRecord = GenericRecord.create(structType);
+ boolean hasAnyDefault = false;
+
+ for (Types.NestedField field : structType.fields()) {
+ if (field.writeDefault() != null) {
+ Object defaultValue = convertToWriteType(field.writeDefault(),
field.type());
+ nestedRecord.setField(field.name(), defaultValue);
+ hasAnyDefault = true;
+ } else if (field.type().isStructType()) {
+ // Recursively process deeper nested structs
+ Record deeperRecord =
buildStructWithDefaults(field.type().asStructType());
+
+ // If the deeper struct has defaults, attach it and flag this current
struct as populated
+ if (deeperRecord != null) {
+ nestedRecord.setField(field.name(), deeperRecord);
+ hasAnyDefault = true;
}
- // Explicit NULLs remain NULL
- } else if (field.type().isStructType() && fieldValue instanceof Record) {
- // For existing structs, apply defaults to any null nested fields
- setDefaultValuesForNestedStruct((Record) fieldValue,
field.type().asStructType().fields());
}
}
+
+ // If no fields (or nested fields) had defaults, return null to avoid an
empty struct
+ return hasAnyDefault ? nestedRecord : null;
}
/**
@@ -483,11 +501,10 @@ private static void
setDefaultValuesForNestedStruct(Record record, List<Types.Ne
for (Types.NestedField field : fields) {
Object fieldValue = record.getField(field.name());
- if (fieldValue == null && field.writeDefault() != null) {
- // Always apply default to null fields in nested structs
+ if (field.writeDefault() != null) {
Object defaultValue = convertToWriteType(field.writeDefault(),
field.type());
record.setField(field.name(), defaultValue);
- } else if (field.type().isStructType() && fieldValue instanceof Record) {
+ } else if (field.type().isStructType()) {
// Recursively process nested structs
setDefaultValuesForNestedStruct((Record) fieldValue,
field.type().asStructType().fields());
}
@@ -532,12 +549,43 @@ public static Object convertToWriteType(Object value,
Type type) {
return value; // fallback
}
- public static Map<String, String> getDefaultValuesMap(String defaultValue) {
- if (StringUtils.isEmpty(defaultValue)) {
+ public static Map<String, String>
getDefaultValuesMap(List<Types.NestedField> fields, String json) {
+ if (json == null || json.trim().isEmpty()) {
return Collections.emptyMap();
}
- // For Struct, the default value is expected to be in key:value format
- return
Splitter.on(',').trimResults().withKeyValueSeparator(':').split(stripQuotes(defaultValue));
+ // 1. Strip the surrounding single or double quotes passed by Hive
+ String cleanJson = stripQuotes(json);
+
+ // 2. Check for NULL on the cleaned string
+ if (cleanJson.equalsIgnoreCase("NULL")) {
+ if (fields == null || fields.isEmpty()) {
+ return Collections.emptyMap();
+ }
+ return fields.stream().collect(Collectors.toMap(Types.NestedField::name,
f -> "NULL"));
+ }
+
+ Map<String, Object> parsed;
+ Map<String, String> result = Maps.newHashMap();
+ try {
+ // 3. Pass the cleaned JSON to the mapper
+ parsed = MAPPER.readValue(cleanJson, new TypeReference<>() {
+ });
+
+ for (Map.Entry<String, Object> entry : parsed.entrySet()) {
+ Object value = entry.getValue();
+
+ if (value == null) {
+ result.put(entry.getKey(), "NULL");
+ } else if (value instanceof Map || value instanceof List) {
+ result.put(entry.getKey(), MAPPER.writeValueAsString(value));
+ } else {
+ result.put(entry.getKey(), value.toString());
+ }
+ }
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Invalid default values JSON: " +
json, e);
+ }
+ return result;
}
public static String stripQuotes(String val) {
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index 1e197d62e89..58c2d19373d 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -890,7 +890,8 @@ private void handleDefaultValues(Map<String, String>
defaultValues, Map<String,
Type fieldType =
columns.stream().filter(col ->
col.name().equalsIgnoreCase(simpleName)).findFirst().get().type();
if (fieldType.isStructType()) {
- Map<String, String> structDefaults =
HiveSchemaUtil.getDefaultValuesMap(field.getValue());
+ Map<String, String> structDefaults =
+
HiveSchemaUtil.getDefaultValuesMap(fieldType.asStructType().fields(),
field.getValue());
handleDefaultValues(structDefaults, renameMapping,
fieldType.asStructType().fields(), qualifiedName + ".");
} else {
Object defaultValue =
HiveSchemaUtil.getDefaultValue(field.getValue(), fieldType);
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDefaultWriter.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDefaultWriter.java
index 5e21cab1651..59bd3314518 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDefaultWriter.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/writer/HiveIcebergDefaultWriter.java
@@ -42,8 +42,7 @@
abstract class HiveIcebergDefaultWriter extends HiveIcebergWriterBase {
private final int currentSpecId;
- private final Set<String> missingColumns;
- private final List<NestedField> missingOrStructFields;
+ private final List<NestedField> missingFields;
HiveIcebergDefaultWriter(
Table table,
@@ -54,12 +53,12 @@ abstract class HiveIcebergDefaultWriter extends
HiveIcebergWriterBase {
Schema schema = table.schema();
this.currentSpecId = table.spec().specId();
- this.missingColumns =
+ Set<String> missingColumns =
context != null && context.missingColumns() != null ?
context.missingColumns() : Set.of();
- this.missingOrStructFields =
+ this.missingFields =
schema.columns().stream()
- .filter(field -> missingColumns.contains(field.name()) ||
field.type().isStructType())
+ .filter(field -> missingColumns.contains(field.name()))
.toList();
}
@@ -76,7 +75,7 @@ private static PartitioningWriter<Record, DataWriteResult>
newDataWriterWithVari
}
protected final void applyDefaultValues(Record record) {
- HiveSchemaUtil.setDefaultValues(record, missingOrStructFields,
missingColumns);
+ HiveSchemaUtil.setDefaultValues(record, missingFields);
}
protected final void write(Record record) {
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q
index 4ec6abf4faf..b86c2d92ecb 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q
@@ -5,7 +5,7 @@ TBLPROPERTIES ('format-version'='3');
INSERT INTO ice_t (id) VALUES (1);
-ALTER TABLE ice_t ADD COLUMNS (point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ALTER TABLE ice_t ADD COLUMNS (point STRUCT<x:INT, y:INT> DEFAULT
'{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -20,7 +20,7 @@ INSERT INTO ice_t (id) VALUES (2);
SELECT * FROM ice_t ORDER BY id;
ALTER TABLE ice_t REPLACE COLUMNS (id INT,
- point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ point STRUCT<x:INT, y:INT> DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -32,7 +32,7 @@ ALTER TABLE ice_t REPLACE COLUMNS (id INT,
SELECT * FROM ice_t ORDER BY id;
-- change default of a field of Struct column
-ALTER TABLE ice_t CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT
'x:100,y:88';
+ALTER TABLE ice_t CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT
'{"x":100,"y":88}';
-- rename and change default value of age column
ALTER TABLE ice_t CHANGE COLUMN age age_new int DEFAULT 21;
@@ -42,7 +42,7 @@ INSERT INTO ice_t (id) VALUES (3);
SELECT * FROM ice_t ORDER BY id;
-- Rename the struct column with default changes
-ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT<x:INT, y:INT> DEFAULT
'x:55,y:88';
+ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT<x:INT, y:INT> DEFAULT
'{"x":55,"y":88}';
INSERT INTO ice_t (id) VALUES (4);
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_default_column.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_default_column.q
index e8ff63c6aa6..22d57ba0d92 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_default_column.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_default_column.q
@@ -1,6 +1,6 @@
CREATE TABLE t3 (
id INT,
- point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ point STRUCT<x:INT, y:INT> DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -13,7 +13,7 @@ CREATE TABLE t3 (
STORED BY ICEBERG
TBLPROPERTIES ('format-version'='3');
--- Case 1: Partial struct with explicit null field
+-- Case 1: Partial struct with explicit null field, the null should be
preserved
INSERT INTO t3 (id, point) VALUES (2, named_struct('x', CAST(null AS INT),
'y', 7));
-- Case 2: Only ID specified (all defaults should apply)
@@ -25,13 +25,38 @@ INSERT INTO t3 (id, name) VALUES (4, NULL);
-- Case 4: Mixed scenario - some fields provided, some missing
INSERT INTO t3 (id, name, age) VALUES (5, 'custom_name', 30);
--- Case 5: Complex struct with nested nulls
+-- Case 5: Complex struct with nested nulls, the null should be preserved
INSERT INTO t3 (id, point) VALUES (6, named_struct('x', CAST(null AS INT),
'y', CAST(null AS INT)));
-- Case 6: ALTER AND SET NULL and 'null' string
ALTER TABLE t3 CHANGE COLUMN age age int DEFAULT null;
ALTER TABLE t3 CHANGE COLUMN name name string DEFAULT 'null';
+ALTER TABLE t3 CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT null;
INSERT INTO t3 (id) VALUES (7);
+-- Case 7: Add a nested struct field with default value
+ALTER TABLE t3 ADD COLUMNS (person STRUCT<
+ name: STRING,
+ address: STRUCT<
+ street: STRING,
+ city: STRING
+ >
+> DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}'
+);
+
+INSERT INTO t3 (id) VALUES (8);
+-- Partial struct with explicit null field, the null should be preserved
+INSERT INTO t3 (id, person)
+VALUES (
+ 9,
+ named_struct(
+ 'name', CAST(NULL AS STRING),
+ 'address', named_struct(
+ 'street', CAST(NULL AS STRING),
+ 'city', 'Bangalore'
+ )
+ )
+);
+
SELECT * FROM t3 ORDER BY id;
\ No newline at end of file
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_initial_default.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_initial_default.q
index 8d78e671e63..c0c058bcd5d 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_initial_default.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_initial_default.q
@@ -5,7 +5,7 @@ TBLPROPERTIES ('format-version'='3');
INSERT INTO ice_parq (id) VALUES (1);
-ALTER TABLE ice_parq ADD COLUMNS (point STRUCT<x:INT, y:INT> DEFAULT
'x:100,y:99',
+ALTER TABLE ice_parq ADD COLUMNS (point STRUCT<x:INT, y:INT> DEFAULT
'{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -20,7 +20,7 @@ INSERT INTO ice_parq (id) VALUES (2);
SELECT * FROM ice_parq ORDER BY id;
-- change default of a field of Struct column
-ALTER TABLE ice_parq CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT
'x:100,y:88';
+ALTER TABLE ice_parq CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT
'{"x":100,"y":88}';
-- rename and change default value of age column
ALTER TABLE ice_parq CHANGE COLUMN age age_new int DEFAULT 21;
@@ -37,7 +37,7 @@ TBLPROPERTIES ('format-version'='3');
INSERT INTO ice_avro (id) VALUES (1);
-ALTER TABLE ice_avro ADD COLUMNS (point STRUCT<x:INT, y:INT> DEFAULT
'x:100,y:99',
+ALTER TABLE ice_avro ADD COLUMNS (point STRUCT<x:INT, y:INT> DEFAULT
'{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -52,7 +52,7 @@ INSERT INTO ice_avro (id) VALUES (2);
SELECT * FROM ice_avro ORDER BY id;
-- change default of a field of Struct column
-ALTER TABLE ice_avro CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT
'x:100,y:88';
+ALTER TABLE ice_avro CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT
'{"x":100,"y":88}';
-- rename and change default value of age column
ALTER TABLE ice_avro CHANGE COLUMN age age_new int DEFAULT 21;
@@ -69,7 +69,7 @@ TBLPROPERTIES ('format-version'='3');
INSERT INTO ice_orc (id) VALUES (1);
-ALTER TABLE ice_orc ADD COLUMNS (point STRUCT<x:INT, y:INT> DEFAULT
'x:100,y:99',
+ALTER TABLE ice_orc ADD COLUMNS (point STRUCT<x:INT, y:INT> DEFAULT
'{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -84,7 +84,7 @@ INSERT INTO ice_orc (id) VALUES (2);
SELECT * FROM ice_orc ORDER BY id;
-- change default of a field of Struct column
-ALTER TABLE ice_orc CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT
'x:100,y:88';
+ALTER TABLE ice_orc CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT
'{"x":100,"y":88}';
-- rename and change default value of age column
ALTER TABLE ice_orc CHANGE COLUMN age age_new int DEFAULT 21;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out
index ec1aa32117a..bb1ae6aa060 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out
@@ -20,7 +20,7 @@ POSTHOOK: query: INSERT INTO ice_t (id) VALUES (1)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_t
-PREHOOK: query: ALTER TABLE ice_t ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+PREHOOK: query: ALTER TABLE ice_t ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -32,7 +32,7 @@ PREHOOK: query: ALTER TABLE ice_t ADD COLUMNS (point
STRUCT<x:INT, y:INT> DEFAUL
PREHOOK: type: ALTERTABLE_ADDCOLS
PREHOOK: Input: default@ice_t
PREHOOK: Output: default@ice_t
-POSTHOOK: query: ALTER TABLE ice_t ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+POSTHOOK: query: ALTER TABLE ice_t ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -63,7 +63,7 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01
10:00:00 100.00 general
2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
PREHOOK: query: ALTER TABLE ice_t REPLACE COLUMNS (id INT,
- point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ point STRUCT<x:INT, y:INT> DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -75,7 +75,7 @@ PREHOOK: type: ALTERTABLE_REPLACECOLS
PREHOOK: Input: default@ice_t
PREHOOK: Output: default@ice_t
POSTHOOK: query: ALTER TABLE ice_t REPLACE COLUMNS (id INT,
- point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ point STRUCT<x:INT, y:INT> DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -96,11 +96,11 @@ POSTHOOK: Input: default@ice_t
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01
10:00:00 general
2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 general
-PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT '{"x":100,"y":88}'
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@ice_t
PREHOOK: Output: default@ice_t
-POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT '{"x":100,"y":88}'
POSTHOOK: type: ALTERTABLE_RENAMECOL
POSTHOOK: Input: default@ice_t
POSTHOOK: Output: default@ice_t
@@ -131,11 +131,11 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01
10:00:00 general
2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 general
3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01
2024-01-01 10:00:00 general
-PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT<x:INT,
y:INT> DEFAULT 'x:55,y:88'
+PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT<x:INT,
y:INT> DEFAULT '{"x":55,"y":88}'
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@ice_t
PREHOOK: Output: default@ice_t
-POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT<x:INT,
y:INT> DEFAULT 'x:55,y:88'
+POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT<x:INT,
y:INT> DEFAULT '{"x":55,"y":88}'
POSTHOOK: type: ALTERTABLE_RENAMECOL
POSTHOOK: Input: default@ice_t
POSTHOOK: Output: default@ice_t
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_default_column.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_default_column.q.out
index 9874e2911c2..e2e90caed20 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_default_column.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_default_column.q.out
@@ -1,6 +1,6 @@
PREHOOK: query: CREATE TABLE t3 (
id INT,
- point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ point STRUCT<x:INT, y:INT> DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -17,7 +17,7 @@ PREHOOK: Output: database:default
PREHOOK: Output: default@t3
POSTHOOK: query: CREATE TABLE t3 (
id INT,
- point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ point STRUCT<x:INT, y:INT> DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -88,6 +88,14 @@ POSTHOOK: query: ALTER TABLE t3 CHANGE COLUMN name name
string DEFAULT 'null'
POSTHOOK: type: ALTERTABLE_RENAMECOL
POSTHOOK: Input: default@t3
POSTHOOK: Output: default@t3
+PREHOOK: query: ALTER TABLE t3 CHANGE COLUMN point point STRUCT<x:INT, y:INT>
DEFAULT null
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@t3
+PREHOOK: Output: default@t3
+POSTHOOK: query: ALTER TABLE t3 CHANGE COLUMN point point STRUCT<x:INT, y:INT>
DEFAULT null
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@t3
+POSTHOOK: Output: default@t3
PREHOOK: query: INSERT INTO t3 (id) VALUES (7)
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
@@ -96,6 +104,64 @@ POSTHOOK: query: INSERT INTO t3 (id) VALUES (7)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@t3
+PREHOOK: query: ALTER TABLE t3 ADD COLUMNS (person STRUCT<
+ name: STRING,
+ address: STRUCT<
+ street: STRING,
+ city: STRING
+ >
+> DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}'
+)
+PREHOOK: type: ALTERTABLE_ADDCOLS
+PREHOOK: Input: default@t3
+PREHOOK: Output: default@t3
+POSTHOOK: query: ALTER TABLE t3 ADD COLUMNS (person STRUCT<
+ name: STRING,
+ address: STRUCT<
+ street: STRING,
+ city: STRING
+ >
+> DEFAULT '{"name":"John","address":{"street":"Main St","city":"New York"}}'
+)
+POSTHOOK: type: ALTERTABLE_ADDCOLS
+POSTHOOK: Input: default@t3
+POSTHOOK: Output: default@t3
+PREHOOK: query: INSERT INTO t3 (id) VALUES (8)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t3
+POSTHOOK: query: INSERT INTO t3 (id) VALUES (8)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t3
+PREHOOK: query: INSERT INTO t3 (id, person)
+VALUES (
+ 9,
+ named_struct(
+ 'name', CAST(NULL AS STRING),
+ 'address', named_struct(
+ 'street', CAST(NULL AS STRING),
+ 'city', 'Bangalore'
+ )
+ )
+)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@t3
+POSTHOOK: query: INSERT INTO t3 (id, person)
+VALUES (
+ 9,
+ named_struct(
+ 'name', CAST(NULL AS STRING),
+ 'address', named_struct(
+ 'street', CAST(NULL AS STRING),
+ 'city', 'Bangalore'
+ )
+ )
+)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@t3
PREHOOK: query: SELECT * FROM t3 ORDER BY id
PREHOOK: type: QUERY
PREHOOK: Input: default@t3
@@ -104,9 +170,11 @@ POSTHOOK: query: SELECT * FROM t3 ORDER BY id
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t3
POSTHOOK: Output: hdfs://### HDFS PATH ###
-2 {"x":100,"y":7} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
-3 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
-4 {"x":100,"y":99} NULL 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
-5 {"x":100,"y":99} custom_name 30 50000.0 true
2024-01-01 2024-01-01 10:00:00 100.00 general
-6 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
-7 {"x":100,"y":99} null NULL 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
+2 {"x":null,"y":7} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general NULL
+3 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general NULL
+4 {"x":100,"y":99} NULL 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general NULL
+5 {"x":100,"y":99} custom_name 30 50000.0 true
2024-01-01 2024-01-01 10:00:00 100.00 general NULL
+6 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01
10:00:00 100.00 general NULL
+7 NULL null NULL 50000.0 true 2024-01-01 2024-01-01
10:00:00 100.00 general NULL
+8 NULL null NULL 50000.0 true 2024-01-01 2024-01-01
10:00:00 100.00 general {"name":"John","address":{"street":"Main
St","city":"New York"}}
+9 NULL null NULL 50000.0 true 2024-01-01 2024-01-01
10:00:00 100.00 general
{"name":null,"address":{"street":null,"city":"Bangalore"}}
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_initial_default.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_initial_default.q.out
index e6ff59cd2c1..da857354d81 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_initial_default.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_initial_default.q.out
@@ -20,7 +20,7 @@ POSTHOOK: query: INSERT INTO ice_parq (id) VALUES (1)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_parq
-PREHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+PREHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -32,7 +32,7 @@ PREHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point
STRUCT<x:INT, y:INT> DEF
PREHOOK: type: ALTERTABLE_ADDCOLS
PREHOOK: Input: default@ice_parq
PREHOOK: Output: default@ice_parq
-POSTHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+POSTHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -62,11 +62,11 @@ POSTHOOK: Input: default@ice_parq
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01
10:00:00 100.00 general
2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
-PREHOOK: query: ALTER TABLE ice_parq CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+PREHOOK: query: ALTER TABLE ice_parq CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT '{"x":100,"y":88}'
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@ice_parq
PREHOOK: Output: default@ice_parq
-POSTHOOK: query: ALTER TABLE ice_parq CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+POSTHOOK: query: ALTER TABLE ice_parq CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT '{"x":100,"y":88}'
POSTHOOK: type: ALTERTABLE_RENAMECOL
POSTHOOK: Input: default@ice_parq
POSTHOOK: Output: default@ice_parq
@@ -119,7 +119,7 @@ POSTHOOK: query: INSERT INTO ice_avro (id) VALUES (1)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_avro
-PREHOOK: query: ALTER TABLE ice_avro ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+PREHOOK: query: ALTER TABLE ice_avro ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -131,7 +131,7 @@ PREHOOK: query: ALTER TABLE ice_avro ADD COLUMNS (point
STRUCT<x:INT, y:INT> DEF
PREHOOK: type: ALTERTABLE_ADDCOLS
PREHOOK: Input: default@ice_avro
PREHOOK: Output: default@ice_avro
-POSTHOOK: query: ALTER TABLE ice_avro ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+POSTHOOK: query: ALTER TABLE ice_avro ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -161,11 +161,11 @@ POSTHOOK: Input: default@ice_avro
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 NULL unknown 25 50000.0 true 2024-01-01 2024-01-01
10:00:00 100.00 general
2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
-PREHOOK: query: ALTER TABLE ice_avro CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+PREHOOK: query: ALTER TABLE ice_avro CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT '{"x":100,"y":88}'
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@ice_avro
PREHOOK: Output: default@ice_avro
-POSTHOOK: query: ALTER TABLE ice_avro CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+POSTHOOK: query: ALTER TABLE ice_avro CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT '{"x":100,"y":88}'
POSTHOOK: type: ALTERTABLE_RENAMECOL
POSTHOOK: Input: default@ice_avro
POSTHOOK: Output: default@ice_avro
@@ -218,7 +218,7 @@ POSTHOOK: query: INSERT INTO ice_orc (id) VALUES (1)
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@ice_orc
-PREHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+PREHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -230,7 +230,7 @@ PREHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point
STRUCT<x:INT, y:INT> DEFA
PREHOOK: type: ALTERTABLE_ADDCOLS
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+POSTHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT '{"x":100,"y":99}',
name STRING DEFAULT 'unknown',
age INT DEFAULT 25,
salary DOUBLE DEFAULT 50000.0,
@@ -260,11 +260,11 @@ POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 NULL NULL NULL NULL NULL NULL NULL NULL NULL
2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
-PREHOOK: query: ALTER TABLE ice_orc CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+PREHOOK: query: ALTER TABLE ice_orc CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT '{"x":100,"y":88}'
PREHOOK: type: ALTERTABLE_RENAMECOL
PREHOOK: Input: default@ice_orc
PREHOOK: Output: default@ice_orc
-POSTHOOK: query: ALTER TABLE ice_orc CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+POSTHOOK: query: ALTER TABLE ice_orc CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT '{"x":100,"y":88}'
POSTHOOK: type: ALTERTABLE_RENAMECOL
POSTHOOK: Input: default@ice_orc
POSTHOOK: Output: default@ice_orc
