This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f2fdea058f0 HIVE-25948: Optimize Iceberg writes by directing records
either to Clustered or Fanout writer (#5789)
f2fdea058f0 is described below
commit f2fdea058f0124153514d40fa827e91c730c0e51
Author: Denys Kuzmenko <[email protected]>
AuthorDate: Thu May 22 12:02:23 2025 +0300
HIVE-25948: Optimize Iceberg writes by directing records either to
Clustered or Fanout writer (#5789)
---
.../iceberg/mr/hive/HiveIcebergInputFormat.java | 3 +-
.../mr/hive/HiveIcebergOutputCommitter.java | 8 +-
.../apache/iceberg/mr/hive/HiveIcebergSerDe.java | 53 +++---
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 50 +++---
.../mr/hive/IcebergMergeTaskProperties.java | 2 +-
.../positive/ctas_iceberg_partitioned_orc.q | 20 ++-
.../positive/ctas_iceberg_partitioned_orc.q.out | 187 +++++++++++++++++++--
.../hadoop/hive/ql/exec/FileSinkOperator.java | 2 +-
.../hadoop/hive/ql/optimizer/GenMapRedUtils.java | 2 +-
.../hadoop/hive/ql/plan/DynamicPartitionCtx.java | 32 ++--
.../apache/hadoop/hive/ql/plan/FileSinkDesc.java | 13 ++
.../hadoop/hive/ql/plan/MergeTaskProperties.java | 4 +-
12 files changed, 271 insertions(+), 105 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
index a8e0589a900..42522ec0a54 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
@@ -122,7 +122,8 @@ static Expression getFilterExpr(Configuration conf,
ExprNodeGenericFuncDesc expr
try {
return HiveIcebergFilterFactory.generateFilterExpression(sarg);
} catch (UnsupportedOperationException e) {
- LOG.warn("Unable to create Iceberg filter, continuing without filter
(will be applied by Hive later): ", e);
+ LOG.warn("Unable to create Iceberg filter, proceeding without it (will
be applied by Hive later): {}",
+ e.getMessage());
}
}
return null;
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
index ae9d2c75b5a..ef1fc90678d 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
@@ -456,7 +456,7 @@ private void commitTable(FileIO io, ExecutorService
executor, OutputTable output
for (JobContext jobContext : jobContexts) {
JobConf conf = jobContext.getJobConf();
- table = Optional.ofNullable(table).orElse(Catalogs.loadTable(conf,
catalogProperties));
+ table = Optional.ofNullable(table).orElseGet(() ->
Catalogs.loadTable(conf, catalogProperties));
branchName = conf.get(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF);
snapshotId = getSnapshotId(outputTable.table, branchName);
@@ -857,7 +857,6 @@ public List<FileStatus> getOutputFiles(List<JobContext>
jobContexts) throws IOEx
.onFailure((output, exc) -> LOG.warn("Failed to retrieve merge input
file for the table {}", output, exc))
.run(output -> {
for (JobContext jobContext : outputs.get(output)) {
- LOG.info("Cleaning job for jobID: {}, table: {}",
jobContext.getJobID(), output);
Table table = output.table;
FileSystem fileSystem = new
Path(table.location()).getFileSystem(jobConf);
String jobLocation = generateJobLocation(table.location(),
jobConf, jobContext.getJobID());
@@ -897,14 +896,14 @@ public List<FileStatus> getOutputFiles(List<JobContext>
jobContexts) throws IOEx
* @return Returns the list of file statuses of the output files in the
jobContexts
* @throws IOException Throws IOException
*/
- public List<ContentFile> getOutputContentFiles(List<JobContext> jobContexts)
throws IOException {
+ public List<ContentFile<?>> getOutputContentFiles(List<JobContext>
jobContexts) throws IOException {
Multimap<OutputTable, JobContext> outputs = collectOutputs(jobContexts);
JobConf jobConf = jobContexts.get(0).getJobConf();
ExecutorService fileExecutor = fileExecutor(jobConf);
ExecutorService tableExecutor = tableExecutor(jobConf,
outputs.keySet().size());
- Collection<ContentFile> files = new ConcurrentLinkedQueue<>();
+ Collection<ContentFile<?>> files = new ConcurrentLinkedQueue<>();
try {
Tasks.foreach(outputs.keySet())
.suppressFailureWhenFinished()
@@ -912,7 +911,6 @@ public List<ContentFile>
getOutputContentFiles(List<JobContext> jobContexts) thr
.onFailure((output, exc) -> LOG.warn("Failed to retrieve merge input
file for the table {}", output, exc))
.run(output -> {
for (JobContext jobContext : outputs.get(output)) {
- LOG.info("Cleaning job for jobID: {}, table: {}",
jobContext.getJobID(), output);
Table table = output.table;
String jobLocation = generateJobLocation(table.location(),
jobConf, jobContext.getJobID());
// list jobLocation to get number of forCommit files
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java
index b7de4f705de..59372e35731 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSerDe.java
@@ -25,7 +25,6 @@
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
-import javax.annotation.Nullable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
@@ -33,7 +32,6 @@
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -78,9 +76,9 @@ public class HiveIcebergSerDe extends AbstractSerDe {
private final Map<String, String> jobConf = Maps.newHashMap();
@Override
- public void initialize(@Nullable Configuration configuration, Properties
serDeProperties,
- Properties partitionProperties) throws SerDeException
{
- super.initialize(configuration, serDeProperties, partitionProperties);
+ public void initialize(Configuration conf, Properties serDeProperties,
+ Properties partitionProperties) throws SerDeException {
+ super.initialize(conf, serDeProperties, partitionProperties);
// HiveIcebergSerDe.initialize is called multiple places in Hive code:
// - When we are trying to create a table - HiveDDL data is stored at the
serDeProperties, but no Iceberg table
@@ -103,7 +101,7 @@ public void initialize(@Nullable Configuration
configuration, Properties serDePr
}
} else {
try {
- Table table = IcebergTableUtil.getTable(configuration,
serDeProperties);
+ Table table = IcebergTableUtil.getTable(conf, serDeProperties);
// always prefer the original table schema if there is one
this.tableSchema = table.schema();
this.partitionColumns =
table.spec().fields().stream().map(PartitionField::name).collect(Collectors.toList());
@@ -115,7 +113,7 @@ public void initialize(@Nullable Configuration
configuration, Properties serDePr
if (serDeProperties.get("metadata_location") != null) {
// If metadata location is provided, extract the schema details from
it.
- try (FileIO fileIO = new HadoopFileIO(configuration)) {
+ try (FileIO fileIO = new HadoopFileIO(conf)) {
TableMetadata metadata = TableMetadataParser.read(fileIO,
serDeProperties.getProperty("metadata_location"));
this.tableSchema = metadata.schema();
this.partitionColumns =
@@ -126,28 +124,28 @@ public void initialize(@Nullable Configuration
configuration, Properties serDePr
}
}
} else {
- boolean autoConversion =
configuration.getBoolean(InputFormatConfig.SCHEMA_AUTO_CONVERSION, false);
+ boolean autoConversion =
conf.getBoolean(InputFormatConfig.SCHEMA_AUTO_CONVERSION, false);
// If we can not load the table try the provided hive schema
this.tableSchema = hiveSchemaOrThrow(e, autoConversion);
// This is only for table creation, it is ok to have an empty
partition column list
this.partitionColumns = ImmutableList.of();
}
if (e instanceof NoSuchTableException &&
- HiveTableUtil.isCtas(serDeProperties) &&
- !Catalogs.hiveCatalog(configuration, serDeProperties)) {
+ HiveTableUtil.isCtas(serDeProperties) &&
!Catalogs.hiveCatalog(conf, serDeProperties)) {
throw new SerDeException(CTAS_EXCEPTION_MSG);
}
}
}
this.projectedSchema =
- projectedSchema(configuration,
serDeProperties.getProperty(Catalogs.NAME), tableSchema, jobConf);
+ projectedSchema(conf, serDeProperties.getProperty(Catalogs.NAME),
tableSchema, jobConf);
- // Currently ClusteredWriter is used which requires that records are
ordered by partition keys.
- // Here we ensure that SortedDynPartitionOptimizer will kick in and do the
sorting.
- // TODO: remove once we have both Fanout and ClusteredWriter available:
HIVE-25948
- HiveConf.setIntVar(configuration,
HiveConf.ConfVars.HIVE_OPT_SORT_DYNAMIC_PARTITION_THRESHOLD, 1);
- HiveConf.setVar(configuration,
HiveConf.ConfVars.DYNAMIC_PARTITIONING_MODE, "nonstrict");
+ if
(!IcebergTableUtil.isFanoutEnabled(Maps.fromProperties(serDeProperties))) {
+ // ClusteredWriter requires that records are ordered by partition keys.
+ // Here we ensure that SortedDynPartitionOptimizer will kick in and do
the sorting.
+ HiveConf.setIntVar(conf,
HiveConf.ConfVars.HIVE_OPT_SORT_DYNAMIC_PARTITION_THRESHOLD, 1);
+ }
+ HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMIC_PARTITIONING_MODE,
"nonstrict");
try {
this.inspector = IcebergObjectInspector.create(projectedSchema);
} catch (Exception e) {
@@ -155,12 +153,12 @@ public void initialize(@Nullable Configuration
configuration, Properties serDePr
}
}
- private static Schema projectedSchema(Configuration configuration, String
tableName, Schema tableSchema,
- Map<String, String> jobConfs) {
- Context.Operation operation =
HiveCustomStorageHandlerUtils.getWriteOperation(configuration::get, tableName);
+ private static Schema projectedSchema(Configuration conf, String tableName,
Schema tableSchema,
+ Map<String, String> jobConf) {
+ Context.Operation operation =
HiveCustomStorageHandlerUtils.getWriteOperation(conf::get, tableName);
if (operation == null) {
- jobConfs.put(InputFormatConfig.CASE_SENSITIVE, "false");
- String[] selectedColumns =
ColumnProjectionUtils.getReadColumnNames(configuration);
+ jobConf.put(InputFormatConfig.CASE_SENSITIVE, "false");
+ String[] selectedColumns =
ColumnProjectionUtils.getReadColumnNames(conf);
// When same table is joined multiple times, it is possible some
selected columns are duplicated,
// in this case wrong recordStructField position leads wrong value or
ArrayIndexOutOfBoundException
String[] distinctSelectedColumns =
Arrays.stream(selectedColumns).distinct().toArray(String[]::new);
@@ -175,7 +173,7 @@ private static Schema projectedSchema(Configuration
configuration, String tableN
return projectedSchema;
}
}
- boolean isCOW = IcebergTableUtil.isCopyOnWriteMode(operation,
configuration::get);
+ boolean isCOW = IcebergTableUtil.isCopyOnWriteMode(operation, conf::get);
if (isCOW) {
return IcebergAcidUtil.createSerdeSchemaForDelete(tableSchema.columns());
}
@@ -212,16 +210,9 @@ public Writable serialize(Object o, ObjectInspector
objectInspector) {
return row;
}
- @Override
- public SerDeStats getSerDeStats() {
- return null;
- }
-
@Override
public void handleJobLevelConfiguration(HiveConf conf) {
- for (Map.Entry<String, String> confs : jobConf.entrySet()) {
- conf.set(confs.getKey(), confs.getValue());
- }
+ jobConf.forEach(conf::set);
}
@Override
@@ -268,7 +259,7 @@ private Schema hiveSchemaOrThrow(Exception
previousException, boolean autoConver
/**
* If the table already exists then returns the list of the current Iceberg
partition column names.
- * If the table is not partitioned by Iceberg, or the table does not exists
yet then returns an empty list.
+ * If the table is not partitioned by Iceberg, or the table does not exist
yet then returns an empty list.
* @return The name of the Iceberg partition columns.
*/
public Collection<String> partitionColumns() {
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index dd73133876b..73ad248b203 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -41,7 +41,6 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections.MapUtils;
@@ -92,7 +91,6 @@
import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
-import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler;
@@ -194,7 +192,6 @@
import
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.base.Splitter;
-import org.apache.iceberg.relocated.com.google.common.base.Throwables;
import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -348,7 +345,7 @@ public void configureJobConf(TableDesc tableDesc, JobConf
jobConf) {
boolean isMergeTaskEnabled = Boolean.parseBoolean(tableDesc.getProperty(
HiveCustomStorageHandlerUtils.MERGE_TASK_ENABLED + tableName));
if (isMergeTaskEnabled) {
- HiveCustomStorageHandlerUtils.setMergeTaskEnabled(jobConf, tableName,
isMergeTaskEnabled);
+ HiveCustomStorageHandlerUtils.setMergeTaskEnabled(jobConf, tableName,
true);
}
String tables = jobConf.get(InputFormatConfig.OUTPUT_TABLES);
tables = tables == null ? tableName : tables + TABLE_NAME_SEPARATOR +
tableName;
@@ -366,7 +363,7 @@ public void configureJobConf(TableDesc tableDesc, JobConf
jobConf) {
Utilities.addDependencyJars(jobConf, HiveIcebergStorageHandler.class);
}
} catch (IOException e) {
- Throwables.propagate(e);
+ throw new RuntimeException(e);
}
}
@@ -606,14 +603,10 @@ private static Map<String, String>
getPartishSummary(Partish partish, Table tabl
}
private Table getTable(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
- Table table;
- final Optional<QueryState> queryState =
SessionStateUtil.getQueryState(conf);
- if (!queryState.isPresent() || queryState.get().getNumModifiedRows() > 0) {
- table = IcebergTableUtil.getTable(conf, hmsTable.getTTable(), true);
- } else {
- table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
- }
- return table;
+ boolean skipCache = SessionStateUtil.getQueryState(conf)
+ .map(queryState -> queryState.getNumModifiedRows() > 0)
+ .orElse(true);
+ return IcebergTableUtil.getTable(conf, hmsTable.getTTable(), skipCache);
}
@Override
@@ -919,11 +912,10 @@ public DynamicPartitionCtx createDPContext(
DynamicPartitionCtx dpCtx = new
DynamicPartitionCtx(Maps.newLinkedHashMap(),
hiveConf.getVar(ConfVars.DEFAULT_PARTITION_NAME),
hiveConf.getIntVar(ConfVars.DYNAMIC_PARTITION_MAX_PARTS_PER_NODE));
- List<Function<List<ExprNodeDesc>, ExprNodeDesc>> customSortExprs =
Lists.newLinkedList();
- dpCtx.setCustomSortExpressions(customSortExprs);
- if (table.spec().isPartitioned()) {
- addCustomSortExpr(table, hmsTable, writeOperation, customSortExprs,
getPartitionTransformSpec(hmsTable));
+ if (table.spec().isPartitioned() &&
+
hiveConf.getIntVar(ConfVars.HIVE_OPT_SORT_DYNAMIC_PARTITION_THRESHOLD) >= 0) {
+ addCustomSortExpr(table, hmsTable, writeOperation, dpCtx,
getPartitionTransformSpec(hmsTable));
}
SortOrder sortOrder = table.sortOrder();
@@ -951,15 +943,14 @@ public DynamicPartitionCtx createDPContext(
}
}
- addCustomSortExpr(table, hmsTable, writeOperation, customSortExprs,
getSortTransformSpec(table));
+ addCustomSortExpr(table, hmsTable, writeOperation, dpCtx,
getSortTransformSpec(table));
}
- dpCtx.setHasCustomSortExprs(!customSortExprs.isEmpty());
return dpCtx;
}
private void addCustomSortExpr(Table table,
org.apache.hadoop.hive.ql.metadata.Table hmsTable,
- Operation writeOperation, List<Function<List<ExprNodeDesc>,
ExprNodeDesc>> customSortExprs,
+ Operation writeOperation, DynamicPartitionCtx dpCtx,
List<TransformSpec> transformSpecs) {
Map<String, Integer> fieldOrderMap = Maps.newHashMap();
List<Types.NestedField> fields = table.schema().columns();
@@ -970,7 +961,7 @@ private void addCustomSortExpr(Table table,
org.apache.hadoop.hive.ql.metadata.
int offset = (shouldOverwrite(hmsTable, writeOperation) ?
ACID_VIRTUAL_COLS_AS_FIELD_SCHEMA : acidSelectColumns(hmsTable,
writeOperation)).size();
- customSortExprs.addAll(transformSpecs.stream().map(spec ->
+ dpCtx.addCustomSortExpressions(transformSpecs.stream().map(spec ->
IcebergTransformSortFunctionUtil.getCustomSortExprs(spec,
fieldOrderMap.get(spec.getColumnName()) + offset)
).collect(Collectors.toList()));
}
@@ -1511,7 +1502,7 @@ public boolean
addDynamicSplitPruningEdge(org.apache.hadoop.hive.ql.metadata.Tab
ExprNodeDesc
syntheticFilterPredicate) {
try {
Collection<String> partitionColumns = ((HiveIcebergSerDe)
table.getDeserializer()).partitionColumns();
- if (partitionColumns.size() > 0) {
+ if (!partitionColumns.isEmpty()) {
// The filter predicate contains ExprNodeDynamicListDesc object(s) for
places where we will substitute
// dynamic values later during execution. For Example:
// GenericUDFIn(Column[ss_sold_date_sk], RS[5] <-- This is an
ExprNodeDynamicListDesc)
@@ -1805,19 +1796,16 @@ private String
collectColumnAndReplaceDummyValues(ExprNodeDesc node, String foun
* </ul>
* @param tableProps table properties, must be not null
*/
- private static void fallbackToNonVectorizedModeBasedOnProperties(Properties
tableProps) {
+ private void fallbackToNonVectorizedModeBasedOnProperties(Properties
tableProps) {
Schema tableSchema =
SchemaParser.fromJson(tableProps.getProperty(InputFormatConfig.TABLE_SCHEMA));
- String tableMeta =
tableProps.getProperty(IcebergAcidUtil.META_TABLE_PROPERTY);
- boolean isMetaTable = tableMeta != null &&
IcebergMetadataTables.isValidMetaTable(tableMeta);
+
if
(FileFormat.AVRO.name().equalsIgnoreCase(tableProps.getProperty(TableProperties.DEFAULT_FILE_FORMAT))
||
- isMetaTable ||
+
isValidMetadataTable(tableProps.getProperty(IcebergAcidUtil.META_TABLE_PROPERTY))
||
hasOrcTimeInSchema(tableProps, tableSchema) ||
!hasParquetNestedTypeWithinListOrMap(tableProps, tableSchema)) {
- try {
-
Hive.get(false).getConf().setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED,
false);
- } catch (HiveException e) {
- throw new RuntimeException(e);
- }
+ // disable vectorization
+ SessionStateUtil.getQueryState(conf).ifPresent(queryState ->
+ queryState.getConf().setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED,
false));
}
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergMergeTaskProperties.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergMergeTaskProperties.java
index 0e8befac887..788e7b13bf9 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergMergeTaskProperties.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergMergeTaskProperties.java
@@ -49,7 +49,7 @@ public Properties getSplitProperties() throws IOException {
if (jobContextList.isEmpty()) {
return null;
}
- List<ContentFile> contentFiles =
HiveIcebergOutputCommitter.getInstance().getOutputContentFiles(jobContextList);
+ List<ContentFile<?>> contentFiles =
HiveIcebergOutputCommitter.getInstance().getOutputContentFiles(jobContextList);
Properties pathToContentFile = new Properties();
contentFiles.forEach(contentFile ->
pathToContentFile.put(new Path(String.valueOf(contentFile.path())),
contentFile));
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/ctas_iceberg_partitioned_orc.q
b/iceberg/iceberg-handler/src/test/queries/positive/ctas_iceberg_partitioned_orc.q
index 8beebf7214d..be68b81cf38 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/ctas_iceberg_partitioned_orc.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/ctas_iceberg_partitioned_orc.q
@@ -19,12 +19,24 @@ insert into source values (1, 'one', 3);
insert into source values (1, 'two', 4);
explain extended
-create external table tbl_ice partitioned by spec (bucket(16, a), truncate(3,
b)) stored by iceberg stored as orc tblproperties ('format-version'='2') as
-select a, b, c from source;
+create external table tbl_ice partitioned by spec (bucket(16, a), truncate(3,
b)) stored by iceberg stored as orc
+as select a, b, c from source;
-create external table tbl_ice partitioned by spec (bucket(16, a), truncate(3,
b)) stored by iceberg stored as orc tblproperties ('format-version'='2') as
-select a, b, c from source;
+create external table tbl_ice partitioned by spec (bucket(16, a), truncate(3,
b)) stored by iceberg stored as orc
+as select a, b, c from source;
describe formatted tbl_ice;
select * from tbl_ice;
+
+set hive.stats.autogather=false;
+set hive.optimize.sort.dynamic.partition.threshold=-1;
+
+explain
+create external table tbl_ice partitioned by (c) stored by iceberg
+as select * from source;
+
+explain
+create external table tbl_ice partitioned by (c) stored by iceberg
+ tblproperties ('write.fanout.enabled'='false')
+as select * from source;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out
b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out
index ae6f0ae42c4..8cbadc4eace 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out
@@ -29,16 +29,16 @@ POSTHOOK: Lineage: source.a SCRIPT []
POSTHOOK: Lineage: source.b SCRIPT []
POSTHOOK: Lineage: source.c SCRIPT []
PREHOOK: query: explain extended
-create external table tbl_ice partitioned by spec (bucket(16, a), truncate(3,
b)) stored by iceberg stored as orc tblproperties ('format-version'='2') as
-select a, b, c from source
+create external table tbl_ice partitioned by spec (bucket(16, a), truncate(3,
b)) stored by iceberg stored as orc
+as select a, b, c from source
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@source
PREHOOK: Output: database:default
PREHOOK: Output: default@tbl_ice
PREHOOK: Output: hdfs://### HDFS PATH ###
POSTHOOK: query: explain extended
-create external table tbl_ice partitioned by spec (bucket(16, a), truncate(3,
b)) stored by iceberg stored as orc tblproperties ('format-version'='2') as
-select a, b, c from source
+create external table tbl_ice partitioned by spec (bucket(16, a), truncate(3,
b)) stored by iceberg stored as orc
+as select a, b, c from source
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@source
POSTHOOK: Output: database:default
@@ -66,7 +66,6 @@ STAGE PLANS:
storage handler: org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table properties:
EXTERNAL TRUE
- format-version 2
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
isExternal: true
@@ -102,7 +101,6 @@ STAGE PLANS:
columns a,b,c
columns.types int:string:int
created_with_ctas true
- format-version 2
iceberg.mr.operation.type.default.tbl_ice OTHER
iceberg.mr.serialized.table.default.tbl_ice
rO0ABXA=
iceberg.mr.table.identifier default.tbl_ice
@@ -120,7 +118,6 @@ STAGE PLANS:
bucketing_version -1
columns a,b,c
columns.types int:string:int
- format-version 2
iceberg.mr.operation.type.default.tbl_ice OTHER
iceberg.mr.table.partition.spec
{"spec-id":0,"fields":[{"name":"a_bucket","transform":"bucket[16]","source-id":1,"field-id":1000},{"name":"b_trunc","transform":"truncate[3]","source-id":2,"field-id":1001}]}
iceberg.mr.table.schema
{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"a","required":false,"type":"int"},{"id":2,"name":"b","required":false,"type":"string"},{"id":3,"name":"c","required":false,"type":"int"}]}
@@ -252,15 +249,15 @@ STAGE PLANS:
source: hdfs://### HDFS PATH ###
destination: hdfs://### HDFS PATH ###
-PREHOOK: query: create external table tbl_ice partitioned by spec (bucket(16,
a), truncate(3, b)) stored by iceberg stored as orc tblproperties
('format-version'='2') as
-select a, b, c from source
+PREHOOK: query: create external table tbl_ice partitioned by spec (bucket(16,
a), truncate(3, b)) stored by iceberg stored as orc
+as select a, b, c from source
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@source
PREHOOK: Output: database:default
PREHOOK: Output: default@tbl_ice
PREHOOK: Output: hdfs://### HDFS PATH ###
-POSTHOOK: query: create external table tbl_ice partitioned by spec (bucket(16,
a), truncate(3, b)) stored by iceberg stored as orc tblproperties
('format-version'='2') as
-select a, b, c from source
+POSTHOOK: query: create external table tbl_ice partitioned by spec (bucket(16,
a), truncate(3, b)) stored by iceberg stored as orc
+as select a, b, c from source
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@source
POSTHOOK: Output: database:default
@@ -337,3 +334,171 @@ POSTHOOK: Input: default@tbl_ice
POSTHOOK: Output: hdfs://### HDFS PATH ###
1 two 4
1 one 3
+PREHOOK: query: explain
+create external table tbl_ice partitioned by (c) stored by iceberg
+as select * from source
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@source
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl_ice
+PREHOOK: Output: default@tbl_ice
+POSTHOOK: query: explain
+create external table tbl_ice partitioned by (c) stored by iceberg
+as select * from source
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@source
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tbl_ice
+POSTHOOK: Output: default@tbl_ice
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-3
+ Create Table
+ columns: a int, b string
+ name: default.tbl_ice
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ location: hdfs://### HDFS PATH ###
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ partition columns: c int
+ storage handler: org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table properties:
+ storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ isExternal: true
+
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: source
+ Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: a (type: int), b (type: string), c (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 190 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 190 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.tbl_ice
+ Execution mode: vectorized
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ c
+ replace: false
+ table:
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.tbl_ice
+
+PREHOOK: query: explain
+create external table tbl_ice partitioned by (c) stored by iceberg
+ tblproperties ('write.fanout.enabled'='false')
+as select * from source
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@source
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tbl_ice
+PREHOOK: Output: default@tbl_ice
+POSTHOOK: query: explain
+create external table tbl_ice partitioned by (c) stored by iceberg
+ tblproperties ('write.fanout.enabled'='false')
+as select * from source
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@source
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tbl_ice
+POSTHOOK: Output: default@tbl_ice
+STAGE DEPENDENCIES:
+ Stage-3 is a root stage
+ Stage-1 depends on stages: Stage-3
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-3
+ Create Table
+ columns: a int, b string
+ name: default.tbl_ice
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ location: hdfs://### HDFS PATH ###
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ partition columns: c int
+ storage handler: org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table properties:
+ storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ write.fanout.enabled false
+ isExternal: true
+
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: source
+ Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: a (type: int), b (type: string), c (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 190 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 2 Data size: 190 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1 (type:
string)
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type:
string), KEY._col2 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ Dp Sort State: PARTITION_SORTED
+ Statistics: Num rows: 2 Data size: 190 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.tbl_ice
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ c
+ replace: false
+ table:
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.tbl_ice
+
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 44144aae8fb..8047d584e95 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -636,7 +636,7 @@ protected void initializeOp(Configuration hconf) throws
HiveException {
jc = new JobConf(hconf);
setWriteOperation(jc, getConf().getTableInfo().getTableName(),
getConf().getWriteOperation());
setWriteOperationIsSorted(jc, getConf().getTableInfo().getTableName(),
- dpCtx != null && dpCtx.hasCustomSortExprs());
+ dpCtx != null && dpCtx.hasCustomSortExpression());
setMergeTaskEnabled(jc, getConf().getTableInfo().getTableName(),
Boolean.parseBoolean((String)
getConf().getTableInfo().getProperties().get(
MERGE_TASK_ENABLED + getConf().getTableInfo().getTableName())));
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 27ff4769ec5..5af9ff1af4c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -1368,7 +1368,7 @@ public static void
createMRWorkForMergingFiles(FileSinkOperator fsInput,
// NOTE: we should gather stats in MR1 rather than MR2 at merge job since
we don't
// know if merge MR2 will be triggered at execution time
MoveWork dummyMv = null;
- if (srcMmWriteId == null) {
+ if (srcMmWriteId == null && !fsInputDesc.isIcebergTable()) {
// Only create the movework for non-MM table. No action needed for a MM
table.
dummyMv = new MoveWork(null, null, null,
new LoadFileDesc(inputDirName, finalName, true, null, null, false),
false);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java
index 580b4499217..c90f6dfb491 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DynamicPartitionCtx.java
@@ -26,9 +26,10 @@
import java.util.function.Function;
import java.util.regex.Pattern;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -52,7 +53,7 @@ public class DynamicPartitionCtx implements Serializable {
private String defaultPartName; // default partition name in case of null or
empty value
private int maxPartsPerNode; // maximum dynamic partitions created per
mapper/reducer
private Pattern whiteListPattern;
- private boolean hasCustomSortExprs = false;
+ private boolean hasCustomSortExpr = false;
/**
* Expressions describing a custom way of sorting the table before write.
Expressions can reference simple
* column descriptions or a tree of expressions containing more columns and
UDFs.
@@ -90,7 +91,7 @@ public DynamicPartitionCtx(List<String> partColNames, String
defaultPartName,
this.spPath = null;
String confVal;
try {
- confVal =
Hive.get().getMetaConf(ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.varname);
+ confVal =
Hive.get().getMetaConf(ConfVars.PARTITION_NAME_WHITELIST_PATTERN.getHiveName());
} catch (HiveException e) {
throw new SemanticException(e);
}
@@ -103,8 +104,8 @@ public DynamicPartitionCtx(List<String> partColNames,
String defaultPartName,
public DynamicPartitionCtx(Map<String, String> partSpec, String
defaultPartName,
int maxParts) throws SemanticException {
this.partSpec = partSpec;
- this.spNames = new ArrayList<String>();
- this.dpNames = new ArrayList<String>();
+ this.spNames = new ArrayList<>();
+ this.dpNames = new ArrayList<>();
this.numBuckets = 0;
this.maxPartsPerNode = maxParts;
this.defaultPartName = defaultPartName;
@@ -125,7 +126,7 @@ public DynamicPartitionCtx(Map<String, String> partSpec,
String defaultPartName,
}
String confVal;
try {
- confVal =
Hive.get().getMetaConf(ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.varname);
+ confVal =
Hive.get().getMetaConf(ConfVars.PARTITION_NAME_WHITELIST_PATTERN.getHiveName());
} catch (HiveException e) {
throw new SemanticException(e);
}
@@ -147,8 +148,8 @@ public DynamicPartitionCtx(DynamicPartitionCtx dp) {
this.defaultPartName = dp.defaultPartName;
this.maxPartsPerNode = dp.maxPartsPerNode;
this.whiteListPattern = dp.whiteListPattern;
- this.customSortExpressions = dp.customSortExpressions;
- this.hasCustomSortExprs = dp.customSortExpressions != null &&
!dp.customSortExpressions.isEmpty();
+ this.customSortExpressions = new LinkedList<>();
+ addCustomSortExpressions(dp.customSortExpressions);
this.customSortOrder = dp.customSortOrder;
this.customSortNullOrder = dp.customSortNullOrder;
}
@@ -241,8 +242,11 @@ public List<Function<List<ExprNodeDesc>, ExprNodeDesc>>
getCustomSortExpressions
return customSortExpressions;
}
- public void setCustomSortExpressions(List<Function<List<ExprNodeDesc>,
ExprNodeDesc>> customSortExpressions) {
- this.customSortExpressions = customSortExpressions;
+ public void addCustomSortExpressions(List<Function<List<ExprNodeDesc>,
ExprNodeDesc>> customSortExpressions) {
+ if (!CollectionUtils.isEmpty(customSortExpressions)) {
+ this.hasCustomSortExpr = true;
+ this.customSortExpressions.addAll(customSortExpressions);
+ }
}
public List<Integer> getCustomSortOrder() {
@@ -261,11 +265,7 @@ public void setCustomSortNullOrder(List<Integer>
customSortNullOrder) {
this.customSortNullOrder = customSortNullOrder;
}
- public boolean hasCustomSortExprs() {
- return hasCustomSortExprs;
- }
-
- public void setHasCustomSortExprs(boolean hasCustomSortExprs) {
- this.hasCustomSortExprs = hasCustomSortExprs;
+ public boolean hasCustomSortExpression() {
+ return hasCustomSortExpr;
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index d09035576ac..5b5685df164 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -24,11 +24,14 @@
import java.util.Objects;
import java.util.Set;
+import com.google.common.collect.Maps;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.ddl.DDLUtils;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.signature.Signature;
@@ -368,6 +371,15 @@ public void setTemporary(boolean temporary) {
this.temporary = temporary;
}
+ public boolean isIcebergTable() {
+ if (getTable() != null) {
+ return DDLUtils.isIcebergTable(table);
+ } else {
+ return MetaStoreUtils.isIcebergTable(
+ Maps.fromProperties(getTableInfo().getProperties()));
+ }
+ }
+
public boolean isMmTable() {
if (getTable() != null) {
return AcidUtils.isInsertOnlyTable(table.getParameters());
@@ -375,6 +387,7 @@ public boolean isMmTable() {
return AcidUtils.isInsertOnlyTable(getTableInfo().getProperties());
}
}
+
public boolean isFullAcidTable() {
if(getTable() != null) {
return AcidUtils.isFullAcidTable(table);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeTaskProperties.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeTaskProperties.java
index fdf8451d273..ad17dc6126c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeTaskProperties.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeTaskProperties.java
@@ -21,12 +21,10 @@
import org.apache.hadoop.fs.Path;
import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
import java.util.Properties;
public interface MergeTaskProperties {
- public Path getTmpLocation();
+ Path getTmpLocation();
default Properties getSplitProperties() throws IOException {
return null;