This is an automated email from the ASF dual-hosted git repository.
difin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2d967374efb HIVE-30056: Iceberg: Add client side support for REST
catalog server-side scan planning (#6789)
2d967374efb is described below
commit 2d967374efbce8b18a1f7894f725d16e0824b8c1
Author: Dmitriy Fingerman <[email protected]>
AuthorDate: Tue Nov 3 19:25:52 2026 -0600
HIVE-30056: Iceberg: Add client side support for REST catalog server-side
scan planning (#6789)
* HIVE-30056: Iceberg: Add REST catalog server-side scan planning support
---------
Co-authored-by: Dmitriy Fingerman <[email protected]>
Co-authored-by: Dmitriy Fingerman <[email protected]>
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 4 +
.../main/java/org/apache/iceberg/mr/Catalogs.java | 2 +-
.../org/apache/iceberg/mr/InputFormatConfig.java | 11 +
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 36 +-
.../org/apache/iceberg/mr/hive/HiveTableUtil.java | 115 ++++-
.../mr/hive/IcebergVendedCredentialUtil.java | 69 ++-
.../mr/hive/RestCatalogScanPlanningUtil.java | 147 +++++++
.../iceberg/mr/mapreduce/IcebergInputFormat.java | 107 +++--
.../apache/iceberg/mr/TestIcebergInputFormats.java | 5 +-
.../mr/hive/TestHiveIcebergStorageHandler.java | 46 ++
.../mr/hive/TestIcebergVendedCredentialUtil.java | 125 +++---
.../mr/hive/TestRestCatalogScanPlanningUtil.java | 142 ++++++
.../mr/hive/TestServerSideScanSnapshotPinning.java | 243 ++++++++++
.../TestHiveIcebergServerSideScanPlanning.java | 136 ++++++
iceberg/iceberg-rest-catalog-client/pom.xml | 12 -
itests/hive-iceberg-rest-server/pom.xml | 234 ++++++++++
...tHiveIcebergServerSideScanPlanningServerIT.java | 487 +++++++++++++++++++++
.../rest/TestRestCatalogScanPlanningServerIT.java | 171 ++++++++
itests/pom.xml | 23 +
19 files changed, 1960 insertions(+), 155 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 8a5561fab6d..bf6ac91e14d 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2161,6 +2161,10 @@ public static enum ConfVars {
"If this is set to true the URI for auth will have the default
location masked with DEFAULT_TABLE_LOCATION"),
HIVE_ICEBERG_ALLOW_DATAFILES_IN_TABLE_LOCATION_ONLY("hive.iceberg.allow.datafiles.in.table.location.only",
false,
"If this is set to true, then all the data files being read should be
withing the table location"),
+ HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED(
+ "hive.iceberg.rest.server-side-scan-planning.enabled", false,
+ "Whether Hive may use server-side scan planning for Iceberg REST
catalogs whose\n" +
+ "scan-planning-mode is server. When false, Hive plans scans itself."),
HIVE_USE_EXPLICIT_RCFILE_HEADER("hive.exec.rcfile.use.explicit.header",
true,
"If this is set the header for RCFiles will simply be RCF. If this is
not\n" +
"set the header will be that borrowed from sequence files, e.g. SEQ-
followed\n" +
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/Catalogs.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/Catalogs.java
index 7b6f0810461..69a2e155bb2 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/Catalogs.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/Catalogs.java
@@ -228,7 +228,7 @@ public static void renameTable(Configuration conf,
Properties props, TableIdenti
}
}
- static Optional<Catalog> loadCatalog(Configuration conf, String catalogName)
{
+ public static Optional<Catalog> loadCatalog(Configuration conf, String
catalogName) {
String resolvedName = catalogName != null ? catalogName :
configuredDefaultCatalogName(conf);
String catalogType = IcebergCatalogProperties.getCatalogType(conf,
resolvedName);
if (NO_CATALOG_TYPE.equalsIgnoreCase(catalogType)) {
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
index 46c281ce0f0..7a0874c5933 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
@@ -92,6 +92,17 @@ public static String vendedCredentialsKey(String tableName) {
return VENDED_STORAGE_CREDENTIALS + "." + tableName;
}
+ /**
+ * Set by HS2 per table when split generation should reload the table from
the REST catalog and
+ * plan it server-side. Keyed per table because the REST server decides the
planning mode per table
+ * and split generation reuses one job conf for all tables of a vertex.
+ */
+ public static final String SERVER_SIDE_SCAN_PLANNING_PREFIX =
"iceberg.mr.server-side-scan-planning.";
+
+ public static String serverSideScanPlanningKey(String tableIdentifier) {
+ return SERVER_SIDE_SCAN_PLANNING_PREFIX + tableIdentifier;
+ }
+
public static final String SORT_ORDER = "sort.order";
public static final String SORT_COLUMNS = "sort.columns";
public static final String ZORDER = "ZORDER";
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index b30f30fbcb6..0e6d4fab0ae 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -279,16 +279,28 @@ public HiveAuthorizationProvider
getAuthorizationProvider() {
@Override
public void configureInputJobCredentials(TableDesc tableDesc, Map<String,
String> secrets) {
- if
(!IcebergVendedCredentialUtil.requestsVendedCredentials(tableDesc.getProperties(),
conf)) {
+ Properties props = tableDesc.getProperties();
+ if (props == null) {
+ return;
+ }
+ boolean vendedCredentials =
IcebergVendedCredentialUtil.requestsVendedCredentials(props, conf);
+ if (!vendedCredentials &&
!RestCatalogScanPlanningUtil.isServerSideScanPlanningEnabled(conf)) {
return;
}
+ Table table;
try {
- Table table =
-
IcebergVendedCredentialUtil.getTableWithVendedCredentials(tableDesc.getProperties(),
conf);
- String catalogName =
tableDesc.getProperties().getProperty(InputFormatConfig.CATALOG_NAME);
- IcebergVendedCredentialUtil.propagateToJob(table, catalogName, null,
secrets, conf);
- } catch (NoSuchTableException ex) {
- // Table may not exist yet for CTAS; credentials will not be available.
+ table = IcebergVendedCredentialUtil.getTableWithVendedCredentials(props,
conf);
+ } catch (NoSuchTableException e) {
+ // Table may not exist yet for CTAS; there is nothing to plan or vend
credentials for.
+ return;
+ }
+ String catalogName = HiveTableUtil.resolveCatalogName(conf,
props.getProperty(InputFormatConfig.CATALOG_NAME));
+
+ if (RestCatalogScanPlanningUtil.shouldPlanServerSide(conf, table)) {
+ RestCatalogScanPlanningUtil.propagateCatalogSecretsToJob(catalogName,
secrets, conf);
+ }
+ if (vendedCredentials) {
+ secrets.putAll(IcebergVendedCredentialUtil.toJobConf(table, catalogName,
conf).secrets());
}
}
@@ -385,7 +397,7 @@ private static void configureOutputTableJobConf(TableDesc
tableDesc, JobConf job
tables = (tables == null) ? tableName : tables + TABLE_NAME_SEPARATOR +
tableName;
jobConf.set(InputFormatConfig.OUTPUT_TABLES, tables);
- String catalogName =
tableDesc.getProperties().getProperty(InputFormatConfig.CATALOG_NAME);
+ String catalogName = tableDesc.getProperty(InputFormatConfig.CATALOG_NAME);
if (catalogName != null) {
jobConf.set(InputFormatConfig.TABLE_CATALOG_PREFIX + tableName,
catalogName);
}
@@ -1738,6 +1750,8 @@ static void overlayTableProperties(Configuration
configuration, TableDesc tableD
location = table.location();
schema = table.schema();
spec = table.spec();
+ RestCatalogScanPlanningUtil.propagateCatalogPropertiesToJob(
+ configuration, table, props.getProperty(Catalogs.NAME), map);
// For intra-txn read-after-write: if the table has in-memory metadata
with no metadata file
// (i.e. uncommitted changes from a prior statement in the same txn),
write the metadata to
@@ -1753,8 +1767,10 @@ static void overlayTableProperties(Configuration
configuration, TableDesc tableD
}
if (isVendedCredentials) {
- String catalogName = props.getProperty(InputFormatConfig.CATALOG_NAME);
- IcebergVendedCredentialUtil.propagateToJob(table, catalogName, map,
null, configuration);
+ String catalogName =
+ HiveTableUtil.resolveCatalogName(configuration,
props.getProperty(InputFormatConfig.CATALOG_NAME));
+ IcebergVendedCredentialUtil.toJobConf(table, catalogName,
configuration)
+ .properties().forEach(map::putIfAbsent);
// Serializing the table as-is would embed its FileIO, and with it the
vended credentials,
// into plain job properties: ship a copy over a secret-free FileIO
instead; executors
// restore the credentials from the Credentials channel
(setCredentials).
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java
index 94e7217cefc..71b5290933d 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java
@@ -28,6 +28,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.Callable;
@@ -58,9 +59,12 @@
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.SerializableTable;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.SnapshotRef;
import org.apache.iceberg.StaticTableOperations;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TableScan;
import org.apache.iceberg.Transaction;
import org.apache.iceberg.data.TableMigrationUtil;
import org.apache.iceberg.exceptions.NotFoundException;
@@ -69,12 +73,14 @@
import org.apache.iceberg.hadoop.HadoopConfigurable;
import org.apache.iceberg.hadoop.HadoopFileIO;
import org.apache.iceberg.hadoop.Util;
+import org.apache.iceberg.hive.IcebergCatalogProperties;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.OutputFile;
import org.apache.iceberg.mapping.NameMapping;
import org.apache.iceberg.mapping.NameMappingParser;
import org.apache.iceberg.mr.Catalogs;
import org.apache.iceberg.mr.InputFormatConfig;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import
org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.iceberg.util.SerializationUtil;
@@ -259,6 +265,113 @@ public static Table deserializeTable(Configuration
config, String name) {
return table;
}
+ /**
+ * Returns true when HS2 marked the table for server-side scan planning and
the scan can
+ * actually be planned by the REST server. Incremental append scans and
metadata tables are planned
+ * locally even on a {@code RESTTable}, and intra-transaction reads must see
uncommitted metadata,
+ * so these keep using the serialized table.
+ */
+ public static boolean shouldReloadForServerSideScanPlanning(Configuration
conf, Table serializedTable) {
+ if
(StringUtils.isNotBlank(conf.get(InputFormatConfig.TABLE_METADATA_LOCATION)) ||
+ conf.getLong(InputFormatConfig.SNAPSHOT_ID_INTERVAL_FROM, -1) != -1 ||
+ serializedTable instanceof
SerializableTable.SerializableMetadataTable) {
+ return false;
+ }
+ String tableIdentifier = conf.get(InputFormatConfig.TABLE_IDENTIFIER);
+ return StringUtils.isNotBlank(tableIdentifier) &&
+
conf.getBoolean(InputFormatConfig.serverSideScanPlanningKey(tableIdentifier),
false);
+ }
+
+ /**
+ * Returns true when the job conf pins snapshot, ref, or as-of time (no
compile-time snapshot pin).
+ */
+ public static boolean hasExplicitScanSnapshotBinding(Configuration conf) {
+ if (conf == null) {
+ return false;
+ }
+ if (conf.getLong(InputFormatConfig.SNAPSHOT_ID_INTERVAL_FROM, -1) != -1) {
+ return true;
+ }
+ if (conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1) != -1) {
+ return true;
+ }
+ if
(StringUtils.isNotBlank(conf.get(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF)))
{
+ return true;
+ }
+ try {
+ if (conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1) != -1) {
+ return true;
+ }
+ } catch (NumberFormatException e) {
+ String version = conf.get(InputFormatConfig.SNAPSHOT_ID);
+ return StringUtils.isNotBlank(version) && !"-1".equals(version);
+ }
+ return false;
+ }
+
+ /**
+ * Pins a scan on a reloaded REST table to the snapshot of the compile-time
table serialized into
+ * the job conf. {@link TableScan#useSnapshot} also switches the scan to
that snapshot's schema; no
+ * explicit projection is set because Hive applies {@code select} afterwards.
+ */
+ public static TableScan pinServerSideScanToSerializedTable(
+ Configuration conf, Table serializedTable, TableScan scan) {
+ if (serializedTable == null || hasExplicitScanSnapshotBinding(conf)) {
+ return scan;
+ }
+ Snapshot snapshot = serializedTable.currentSnapshot();
+ return snapshot != null ? scan.useSnapshot(snapshot.snapshotId()) : scan;
+ }
+
+ /**
+ * Pins a branch or tag read on a reloaded REST table to the snapshot the
ref pointed at in the
+ * compile-time table. A branch whose head snapshot has an older schema
keeps {@link TableScan#useRef}:
+ * branches read with the table schema, while {@link TableScan#useSnapshot}
would switch to the
+ * snapshot schema and lose columns added since. In that case the live ref
head must still match the
+ * compile-time snapshot id, or the branch moved after compilation and this
method fails fast.
+ *
+ * @throws IllegalStateException if the ref is absent from the serialized
table, or the branch head moved
+ */
+ public static TableScan pinServerSideScanRefToSerializedTable(
+ Table serializedTable, String refName, TableScan scan) {
+ SnapshotRef ref = serializedTable.refs().get(refName);
+ Preconditions.checkState(
+ ref != null,
+ "Cannot find reference %s in compile-time table metadata",
+ refName);
+ Integer refSchemaId =
serializedTable.snapshot(ref.snapshotId()).schemaId();
+ if (ref.isBranch() && !Objects.equals(refSchemaId,
serializedTable.schema().schemaId())) {
+ TableScan pinned = scan.useRef(refName);
+ Preconditions.checkState(
+ pinned.snapshot().snapshotId() == ref.snapshotId(),
+ "Branch %s moved since query compilation", refName);
+ return pinned;
+ }
+ return scan.useSnapshot(ref.snapshotId());
+ }
+
+ /**
+ * Resolves the Iceberg catalog name from {@link
InputFormatConfig#CATALOG_NAME} on the
+ * configuration, or the session default ({@link
IcebergCatalogProperties#getCatalogName(Configuration)}).
+ */
+ public static String resolveCatalogName(Configuration conf) {
+ return resolveCatalogName(conf, conf != null ?
conf.get(InputFormatConfig.CATALOG_NAME) : null);
+ }
+
+ /**
+ * Resolves the Iceberg catalog name from a per-table {@link
InputFormatConfig#CATALOG_NAME} value
+ * (e.g. {@code TableDesc} properties) or the session default when that
value is absent.
+ */
+ public static String resolveCatalogName(Configuration conf, String
catalogNameFromTable) {
+ if (StringUtils.isNotBlank(catalogNameFromTable)) {
+ return catalogNameFromTable;
+ }
+ if (conf == null) {
+ return null;
+ }
+ return IcebergCatalogProperties.getCatalogName(conf);
+ }
+
/**
* If enabled, it populates the FileIO's hadoop configuration with the input
config object.
* This might be necessary when the table object was serialized without the
FileIO config.
@@ -266,7 +379,7 @@ public static Table deserializeTable(Configuration config,
String name) {
* @param config Configuration to set for FileIO, if enabled
* @param table The Iceberg table object
*/
- private static void checkAndSetIoConfig(Configuration config, Table table) {
+ public static void checkAndSetIoConfig(Configuration config, Table table) {
if (table != null &&
config.getBoolean(InputFormatConfig.CONFIG_SERIALIZATION_DISABLED,
InputFormatConfig.CONFIG_SERIALIZATION_DISABLED_DEFAULT) &&
table.io() instanceof HadoopConfigurable fileIO) {
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergVendedCredentialUtil.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergVendedCredentialUtil.java
index dfa4547b182..83e20682552 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergVendedCredentialUtil.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergVendedCredentialUtil.java
@@ -63,40 +63,39 @@ public final class IcebergVendedCredentialUtil {
private IcebergVendedCredentialUtil() {
}
+ /** Vended credentials split per HIVE-20651: plain job properties vs job
secrets. */
+ record VendedJobConf(Map<String, String> properties, Map<String, String>
secrets) {
+ static final VendedJobConf EMPTY = new VendedJobConf(Map.of(), Map.of());
+ }
+
/**
- * Copies vended credentials from the table FileIO into Hive job
configuration.
+ * Maps vended credentials from the table FileIO to Hive job configuration.
*
* <p>Follows the HIVE-20651 split used by {@code JdbcStorageHandler}:
sensitive values (including
- * the serialized {@link StorageCredential} list) go to {@code jobSecrets};
non-secret config such
- * as endpoint and path-style access go to {@code jobProperties}.
+ * the serialized {@link StorageCredential} list) go to {@link
VendedJobConf#secrets()}; non-secret
+ * config such as endpoint and path-style access go to {@link
VendedJobConf#properties()}.
*
* @param table loaded Iceberg table; its catalog-qualified name keys the
credentials blob,
* since REST catalogs mint credentials per table (the table pointer
carries the same
* name, so executors resolve the matching key)
- * @param catalogName Hive catalog name ({@link
InputFormatConfig#CATALOG_NAME})
- * @param jobProperties Tez/MR non-secret job properties; may be {@code null}
- * @param jobSecrets sensitive keys and serialized credentials; may be
{@code null}
+ * @param catalogName Hive catalog name, resolved by {@link
HiveTableUtil#resolveCatalogName}
* @param conf session conf used to preserve host-side endpoint overrides
*/
- public static void propagateToJob(Table table, String catalogName,
Map<String, String> jobProperties,
- Map<String, String> jobSecrets, Configuration conf) {
-
+ static VendedJobConf toJobConf(Table table, String catalogName,
Configuration conf) {
List<StorageCredential> credentials =
withConfigurationOverrides(catalogName, extractCredentials(table),
conf);
-
if (credentials.isEmpty()) {
- return;
+ return VendedJobConf.EMPTY;
}
-
- if (jobSecrets != null) {
- jobSecrets.put(
- InputFormatConfig.vendedCredentialsKey(table.name()),
- serializeToSingleLineBase64(Lists.newArrayList(credentials)));
- }
-
+ Map<String, String> properties = new LinkedHashMap<>();
+ Map<String, String> secrets = new LinkedHashMap<>();
+ secrets.put(
+ InputFormatConfig.vendedCredentialsKey(table.name()),
+ serializeToSingleLineBase64(Lists.newArrayList(credentials)));
for (StorageCredential credential : credentials) {
- addCredentialEntries(catalogName, credential, jobProperties, jobSecrets,
conf);
+ addCredentialEntries(catalogName, credential, properties, secrets, conf);
}
+ return new VendedJobConf(properties, secrets);
}
/**
@@ -125,10 +124,8 @@ private static void addCredentialEntries(String
catalogName, StorageCredential c
addCredentialEntry(
catalogName, scope, mapper, entry.getKey(), entry.getValue(),
jobProperties, jobSecrets, conf);
}
- if (jobProperties != null) {
- Support.additionalNonSecretHadoopProperties(mapper, scope, config)
- .forEach(jobProperties::putIfAbsent);
- }
+ Support.additionalNonSecretHadoopProperties(mapper, scope, config)
+ .forEach(jobProperties::putIfAbsent);
}
/**
@@ -136,8 +133,7 @@ private static void addCredentialEntries(String
catalogName, StorageCredential c
*
* <p>Skips blank values, applies session catalog overrides via {@link
#resolveCredentialValue},
* then sends non-secret keys (endpoint, path-style access, etc.) to {@code
jobProperties} and
- * secret keys (access key, secret key, session token) to {@code
jobSecrets}. Either map may be
- * {@code null} when {@link #propagateToJob} is called for only properties
or only secrets.
+ * secret keys (access key, secret key, session token) to {@code jobSecrets}.
*/
private static void addCredentialEntry(String catalogName, String scope,
HadoopMapper mapper, String icebergKey, String value,
@@ -148,12 +144,10 @@ private static void addCredentialEntry(String
catalogName, String scope,
}
String resolvedValue = resolveCredentialValue(catalogName, icebergKey,
value, conf);
- if (jobProperties != null && !isSecretKey(icebergKey, conf)) {
- addNonSecretCredentialEntry(catalogName, scope, mapper, icebergKey,
resolvedValue, jobProperties);
- }
-
- if (jobSecrets != null && isSecretKey(icebergKey, conf)) {
+ if (isSecretKey(icebergKey, conf)) {
addSecretCredentialEntry(scope, mapper, icebergKey, resolvedValue,
jobSecrets);
+ } else {
+ addNonSecretCredentialEntry(catalogName, scope, mapper, icebergKey,
resolvedValue, jobProperties);
}
}
@@ -294,18 +288,13 @@ static void refreshVendedCredentialsIfMissing(TableDesc
tableDesc, JobConf jobCo
return;
}
- String catalogName = props.getProperty(InputFormatConfig.CATALOG_NAME);
- if (catalogName == null) {
- return;
- }
-
+ String catalogName =
+ HiveTableUtil.resolveCatalogName(configuration,
props.getProperty(InputFormatConfig.CATALOG_NAME));
try {
Table table = getTableWithVendedCredentials(props, configuration);
- Map<String, String> jobProps = new LinkedHashMap<>();
- Map<String, String> secrets = new LinkedHashMap<>();
- propagateToJob(table, catalogName, jobProps, secrets, configuration);
- jobProps.forEach(jobConf::set);
- mergeJobSecrets(tableDesc, secrets);
+ VendedJobConf vended = toJobConf(table, catalogName, configuration);
+ vended.properties().forEach(jobConf::set);
+ mergeJobSecrets(tableDesc, vended.secrets());
} catch (NoSuchTableException ex) {
// Table may not exist yet for CTAS; credentials will not be available.
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/RestCatalogScanPlanningUtil.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/RestCatalogScanPlanningUtil.java
new file mode 100644
index 00000000000..6eaa7649d2b
--- /dev/null
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/RestCatalogScanPlanningUtil.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive;
+
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.hive.IcebergCatalogProperties;
+import org.apache.iceberg.mr.InputFormatConfig;
+import org.apache.iceberg.rest.RESTCatalogProperties;
+
+/**
+ * Utilities for Iceberg REST catalog server-side scan planning configuration.
+ *
+ * <p>When a REST catalog server advertises the scan-planning endpoints and
+ * {@link RESTCatalogProperties#SCAN_PLANNING_MODE} is set to
+ * {@link RESTCatalogProperties.ScanPlanningMode#SERVER}, Iceberg's {@code
RESTSessionCatalog} returns a
+ * {@code RESTTable} that delegates {@code planTasks()} to the server. Hive's
+ * {@code IcebergInputFormat} calls {@code scan.planTasks()} via
+ * {@code IcebergInputFormat#getSplits}, which reloads a live REST catalog
table for {@code POST /plan}
+ * (pinned to the serialized compile-time snapshot). The catalog decides the
scan planning mode;
+ * {@link
HiveConf.ConfVars#HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED} only
controls whether
+ * Hive is allowed to follow it.
+ *
+ * <p>Tests: {@code TestRestCatalogScanPlanningUtil} in {@code
iceberg-handler};
+ * {@code TestHiveIcebergServerSideScanPlanning} in {@code iceberg-handler};
embedded REST server
+ * tests {@code TestRestCatalogScanPlanningServerIT} and {@code
TestHiveIcebergServerSideScanPlanningServerIT}
+ * in {@code itests/hive-iceberg-rest-server}.
+ *
+ * <p>{@link #setScanPlanningMode(Configuration, String, String)},
+ * {@link #getScanPlanningMode(Configuration, String)} and
+ * {@link #propagateCatalogPropertiesToJob(Configuration, Table, String, Map)}
are public so tests in other
+ * Maven modules (for example {@code itests/hive-iceberg-rest-server}) can
configure scan planning; those modules
+ * compile against this artifact as a JAR and cannot call package-private
members. They are not
+ * intended as a general operator or application API.
+ *
+ * @see <a
href="https://iceberg.apache.org/docs/latest/catalog-properties/">REST catalog
properties</a>
+ */
+public final class RestCatalogScanPlanningUtil {
+
+ private static final String REST_TABLE_CLASS_NAME =
"org.apache.iceberg.rest.RESTTable";
+
+ private RestCatalogScanPlanningUtil() {
+ }
+
+ public static void setScanPlanningMode(Configuration conf, String
catalogName, String mode) {
+ conf.set(
+ IcebergCatalogProperties.catalogPropertyConfigKey(
+ catalogName, RESTCatalogProperties.SCAN_PLANNING_MODE),
+ RESTCatalogProperties.ScanPlanningMode.fromString(mode).modeName());
+ }
+
+ public static RESTCatalogProperties.ScanPlanningMode getScanPlanningMode(
+ Configuration conf, String catalogName) {
+ String mode = conf.get(
+ IcebergCatalogProperties.catalogPropertyConfigKey(
+ catalogName, RESTCatalogProperties.SCAN_PLANNING_MODE),
+ RESTCatalogProperties.SCAN_PLANNING_MODE_DEFAULT.modeName());
+ return RESTCatalogProperties.ScanPlanningMode.fromString(mode);
+ }
+
+ /**
+ * Returns true when Hive is allowed to use server-side scan planning (the
Hive-side kill switch).
+ */
+ static boolean isServerSideScanPlanningEnabled(Configuration conf) {
+ return conf != null &&
+ HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED);
+ }
+
+ /**
+ * Returns true when Iceberg resolved the loaded table for server-side scan
planning. Iceberg decides
+ * this per table, letting the mode in the server's {@code
LoadTableResponse} take precedence over the
+ * client {@code scan-planning-mode}, and then returns a package-private
{@code RESTTable}.
+ */
+ static boolean usesServerSideScanPlanning(Table table) {
+ return table != null &&
REST_TABLE_CLASS_NAME.equals(table.getClass().getName());
+ }
+
+ /**
+ * Returns true when split generation should reload the live REST table for
server-side scan planning:
+ * the Hive kill switch is on and Iceberg resolved the loaded table for
server-side planning. Both the
+ * per-table marker and the catalog job secrets are gated on this.
+ */
+ static boolean shouldPlanServerSide(Configuration sessionConf, Table table) {
+ return isServerSideScanPlanningEnabled(sessionConf) &&
usesServerSideScanPlanning(table);
+ }
+
+ /**
+ * Writes the per-table {@link
InputFormatConfig#serverSideScanPlanningKey(String)} marker and the
+ * session default catalog into the table's job properties when {@link
#shouldPlanServerSide} holds.
+ * The {@code iceberg.catalog.<catalog>.*} entries go through job secrets
instead, see
+ * {@link #propagateCatalogSecretsToJob(String, Map, Configuration)}.
+ * Session values replace existing entries so a session {@code SET} wins.
+ */
+ public static void propagateCatalogPropertiesToJob(
+ Configuration sessionConf, Table table, String tableIdentifier,
Map<String, String> jobProperties) {
+ if (jobProperties == null || !shouldPlanServerSide(sessionConf, table)) {
+ return;
+ }
+
jobProperties.put(InputFormatConfig.serverSideScanPlanningKey(tableIdentifier),
Boolean.TRUE.toString());
+
+ String sessionDefaultCatalog = MetastoreConf.getVar(sessionConf,
MetastoreConf.ConfVars.CATALOG_DEFAULT);
+ if (StringUtils.isNotBlank(sessionDefaultCatalog)) {
+ jobProperties.put(MetastoreConf.ConfVars.CATALOG_DEFAULT.getVarname(),
sessionDefaultCatalog);
+ }
+ }
+
+ /**
+ * Copies all {@code iceberg.catalog.<catalog>.*} entries from the HS2
session into job secrets
+ * (HIVE-20651), so catalog credentials never land in plain job properties.
Only split generation in
+ * the AM needs them to load the REST catalog, so the caller ships them only
when
+ * {@link #shouldPlanServerSide} holds for the loaded table, the same
decision as the per-table marker;
+ * {@code HiveInputFormat#addSplitsForGroup} copies job secrets into the job
conf before {@code getSplits}.
+ */
+ static void propagateCatalogSecretsToJob(
+ String catalogName, Map<String, String> jobSecrets, Configuration
sessionConf) {
+ if (StringUtils.isEmpty(catalogName)) {
+ return;
+ }
+ String catalogPrefix = IcebergCatalogProperties.CATALOG_CONFIG_PREFIX +
catalogName + ".";
+ sessionConf.forEach(entry -> {
+ if (entry.getKey().startsWith(catalogPrefix)) {
+ jobSecrets.put(entry.getKey(), entry.getValue());
+ }
+ });
+ }
+}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
index 89f888255b8..3969073c87a 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
@@ -22,7 +22,6 @@
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.List;
-import java.util.Optional;
import java.util.concurrent.ExecutorService;
import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
@@ -53,8 +52,11 @@
import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.TableScan;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableGroup;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.mr.Catalogs;
import org.apache.iceberg.mr.InputFormatConfig;
@@ -72,31 +74,39 @@
*/
public class IcebergInputFormat<T> extends InputFormat<Void, T> {
- private static TableScan createTableScan(Table table, Configuration conf) {
+ private static TableScan createTableScan(Table table, Configuration conf,
Table serverSidePinTable) {
TableScan scan = table.newScan();
- long snapshotId = -1;
- try {
- snapshotId = conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1);
- } catch (NumberFormatException e) {
- String version = conf.get(InputFormatConfig.SNAPSHOT_ID);
- SnapshotRef ref = table.refs().get(version);
- if (ref == null) {
- throw new RuntimeException("Cannot find matching snapshot ID or
reference name for version " + version);
+ if (HiveTableUtil.hasExplicitScanSnapshotBinding(conf)) {
+ long snapshotId = -1;
+ try {
+ snapshotId = conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1);
+ } catch (NumberFormatException e) {
+ String version = conf.get(InputFormatConfig.SNAPSHOT_ID);
+ Table refsTable = serverSidePinTable != null ? serverSidePinTable :
table;
+ SnapshotRef ref = refsTable.refs().get(version);
+ if (ref == null) {
+ throw new RuntimeException("Cannot find matching snapshot ID or
reference name for version " + version);
+ }
+ snapshotId = ref.snapshotId();
+ }
+ String refName = conf.get(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF);
+ if (StringUtils.isNotEmpty(refName)) {
+ String ref = HiveUtils.getTableSnapshotRef(refName);
+ scan = serverSidePinTable != null ?
+
HiveTableUtil.pinServerSideScanRefToSerializedTable(serverSidePinTable, ref,
scan) :
+ scan.useRef(ref);
+ }
+ if (snapshotId != -1) {
+ scan = scan.useSnapshot(snapshotId);
}
- snapshotId = ref.snapshotId();
- }
- String refName = conf.get(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF);
- if (StringUtils.isNotEmpty(refName)) {
- scan = scan.useRef(HiveUtils.getTableSnapshotRef(refName));
- }
- if (snapshotId != -1) {
- scan = scan.useSnapshot(snapshotId);
- }
- long asOfTime = conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1);
- if (asOfTime != -1) {
- scan = scan.asOfTime(asOfTime);
+ long asOfTime = conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1);
+ if (asOfTime != -1) {
+ scan = scan.asOfTime(asOfTime);
+ }
+ } else if (serverSidePinTable != null) {
+ scan = HiveTableUtil.pinServerSideScanToSerializedTable(conf,
serverSidePinTable, scan);
}
return scan;
@@ -157,27 +167,54 @@ private static <T extends Scan<T, FileScanTask,
CombinedScanTask>> T applyConfig
@Override
public List<InputSplit> getSplits(JobContext context) {
Configuration conf = context.getConfiguration();
- Table table = Optional
- .ofNullable(HiveTableUtil.deserializeTable(conf,
conf.get(InputFormatConfig.TABLE_IDENTIFIER)))
- .orElseGet(() -> {
- Table tbl = Catalogs.loadTable(conf);
- conf.set(InputFormatConfig.TABLE_IDENTIFIER, tbl.name());
- // planning-local conf only (never shipped): for credential-vending
catalogs the loaded
- // table's FileIO carries secrets, which must not reach a serialized
job configuration
- conf.set(InputFormatConfig.SERIALIZED_TABLE_PREFIX + tbl.name(),
SerializationUtil.serializeToBase64(tbl));
- return tbl;
- });
+ String tableIdentifier = conf.get(InputFormatConfig.TABLE_IDENTIFIER);
+ Table serializedTable = HiveTableUtil.deserializeTable(conf,
tableIdentifier);
+
final ExecutorService workerPool =
ThreadPools.newFixedThreadPool("iceberg-plan-worker-pool",
conf.getInt(SystemConfigs.WORKER_THREAD_POOL_SIZE.propertyKey(),
ThreadPools.WORKER_THREAD_POOL_SIZE));
try {
- return planInputSplits(table, conf, workerPool);
+ if (HiveTableUtil.shouldReloadForServerSideScanPlanning(conf,
serializedTable)) {
+ Catalog catalog = Catalogs.loadCatalog(conf,
HiveTableUtil.resolveCatalogName(conf))
+ .orElseThrow(() -> new IllegalStateException(
+ "Iceberg catalog is required for REST server-side scan
planning"));
+ try (CloseableGroup closer = new CloseableGroup()) {
+ closer.setSuppressCloseFailure(true);
+ if (catalog instanceof AutoCloseable closeable) {
+ closer.addCloseable(closeable);
+ }
+ Table planningTable =
catalog.loadTable(TableIdentifier.parse(tableIdentifier));
+ HiveTableUtil.checkAndSetIoConfig(conf, planningTable);
+ return planInputSplits(planningTable, conf, workerPool,
tableIdentifier, serializedTable);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+
+ Table table = serializedTable != null ? serializedTable :
Catalogs.loadTable(conf);
+ return planInputSplits(table, conf, workerPool, tableIdentifier, null);
} finally {
workerPool.shutdown();
}
}
- private List<InputSplit> planInputSplits(Table table, Configuration conf,
ExecutorService workerPool) {
+ private List<InputSplit> planInputSplits(
+ Table table,
+ Configuration conf,
+ ExecutorService workerPool,
+ String tableIdentifier,
+ Table serverSidePinTable) {
+ String resolvedTableIdentifier = tableIdentifier;
+ if (StringUtils.isBlank(resolvedTableIdentifier)) {
+ resolvedTableIdentifier = table.name();
+ conf.set(InputFormatConfig.TABLE_IDENTIFIER, resolvedTableIdentifier);
+ }
+ String serializedTableKey = InputFormatConfig.SERIALIZED_TABLE_PREFIX +
resolvedTableIdentifier;
+ if (conf.get(serializedTableKey) == null) {
+ Table toSerialize = serverSidePinTable != null ? serverSidePinTable :
table;
+ conf.set(serializedTableKey,
SerializationUtil.serializeToBase64(toSerialize));
+ }
List<InputSplit> splits = Lists.newArrayList();
boolean applyResidual =
!conf.getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false);
InputFormatConfig.InMemoryDataModel model =
conf.getEnum(InputFormatConfig.IN_MEMORY_DATA_MODEL,
@@ -188,7 +225,7 @@ private List<InputSplit> planInputSplits(Table table,
Configuration conf, Execut
if (fromVersion != -1) {
scan = applyConfig(conf, createIncrementalAppendScan(table, conf));
} else {
- scan = applyConfig(conf, createTableScan(table, conf));
+ scan = applyConfig(conf, createTableScan(table, conf,
serverSidePinTable));
}
scan = scan.planWith(workerPool);
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormats.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormats.java
index 5e59633505e..9ebf2dcadcd 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormats.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormats.java
@@ -438,13 +438,14 @@ private String getUserFromWorkerPool(
UserGroupInformation user, Table table, ExecutorService workerpool)
throws Exception {
Method method =
IcebergInputFormat.class.getDeclaredMethod(
- "planInputSplits", Table.class, Configuration.class,
ExecutorService.class);
+ "planInputSplits", Table.class, Configuration.class,
ExecutorService.class, String.class,
+ Table.class);
method.setAccessible(true);
return user.doAs(
(PrivilegedAction<String>)
() -> {
try {
- method.invoke(new IcebergInputFormat<>(), table, conf,
workerpool);
+ method.invoke(new IcebergInputFormat<>(), table, conf,
workerpool, null, null);
return workerpool
.submit(() ->
UserGroupInformation.getCurrentUser().getUserName())
.get();
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandler.java
index 1a2eddcf5f1..14b60188dcd 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandler.java
@@ -19,17 +19,30 @@
package org.apache.iceberg.mr.hive;
+import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import org.apache.commons.collections4.IterableUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.SnapshotContext;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.iceberg.Schema;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.hive.IcebergCatalogProperties;
+import org.apache.iceberg.mr.Catalogs;
+import org.apache.iceberg.mr.InputFormatConfig;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Types;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -45,6 +58,9 @@
@RunWith(MockitoJUnitRunner.class)
public class TestHiveIcebergStorageHandler {
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
@Mock
private Snapshot anySnapshot;
@Mock
@@ -69,6 +85,36 @@ public void before() {
Mockito.lenient().when(deleteSnapshot.summary()).thenReturn(summary);
}
+ /**
+ * Catalog secrets follow the same decision as the server-side planning
marker: a table Iceberg did
+ * not resolve to a server-planning {@code RESTTable} gets no {@code
iceberg.catalog.<catalog>.*}
+ * secrets, even with the Hive kill switch on. The positive case needs a
REST server and is in
+ * {@code TestHiveIcebergServerSideScanPlanningServerIT}.
+ */
+ @Test
+ public void
configureInputJobCredentialsSkipsCatalogSecretsForTableNotPlannedServerSide()
throws IOException {
+ String location = temp.newFolder().toURI().toString();
+ Configuration conf = new Configuration();
+ new HadoopTables(conf).create(new Schema(Types.NestedField.required(1,
"id", Types.LongType.get())), location);
+ HiveConf.setBoolVar(conf,
HiveConf.ConfVars.HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED, true);
+
conf.set(IcebergCatalogProperties.catalogPropertyConfigKey(Catalogs.ICEBERG_HADOOP_TABLE_NAME,
"credential"),
+ "catalog-credential");
+
+ Properties props = new Properties();
+ props.setProperty(InputFormatConfig.CATALOG_NAME,
Catalogs.ICEBERG_HADOOP_TABLE_NAME);
+ props.setProperty(Catalogs.LOCATION, location);
+ props.setProperty(Catalogs.NAME, "default.tbl");
+ TableDesc tableDesc = new TableDesc();
+ tableDesc.setProperties(props);
+
+ HiveIcebergStorageHandler storageHandler = new HiveIcebergStorageHandler();
+ storageHandler.setConf(conf);
+ Map<String, String> secrets = Maps.newHashMap();
+ storageHandler.configureInputJobCredentials(tableDesc, secrets);
+
+ assertThat(secrets.isEmpty(), is(true));
+ }
+
@Test
public void testHasAppendsOnlyReturnsNullWhenTableIsEmpty() {
SnapshotContext since = new SnapshotContext(42);
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestIcebergVendedCredentialUtil.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestIcebergVendedCredentialUtil.java
index 59dee570c34..30b89034faf 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestIcebergVendedCredentialUtil.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestIcebergVendedCredentialUtil.java
@@ -90,8 +90,8 @@ public void
requestsVendedCredentialsRequiresDelegationHeader() {
}
/**
- * Tests {@link IcebergVendedCredentialUtil#propagateToJob} when both {@code
jobProperties} and
- * {@code jobSecrets} are non-null (full compile-time path after both
storage-handler hooks).
+ * Tests {@link IcebergVendedCredentialUtil#toJobConf} using both its
properties and secrets
+ * (full compile-time path after both storage-handler hooks).
*
* <p><b>Setup:</b> vended credential with internal endpoint {@code
http://minio:9000}; session
* {@code conf} sets {@code iceberg.catalog.ice01.s3.endpoint} to {@code
http://host:9000}.
@@ -106,7 +106,7 @@ public void
requestsVendedCredentialsRequiresDelegationHeader() {
* agree on connectivity while still using vended keys.
*/
@Test
- public void propagateToJobMapsIcebergAndS3aProperties() {
+ public void toJobConfMapsIcebergAndS3aProperties() {
Configuration conf = new Configuration();
conf.set("iceberg.catalog.ice01." + S3FileIOProperties.ENDPOINT,
"http://host:9000");
@@ -126,7 +126,9 @@ public void propagateToJobMapsIcebergAndS3aProperties() {
Map<String, String> jobProps = Maps.newHashMap();
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", jobProps,
jobSecrets, conf);
+ IcebergVendedCredentialUtil.VendedJobConf vended =
IcebergVendedCredentialUtil.toJobConf(table, "ice01", conf);
+ jobProps.putAll(vended.properties());
+ jobSecrets.putAll(vended.secrets());
assertThat(jobProps)
.containsEntry(
@@ -167,10 +169,10 @@ public void propagateToJobMapsIcebergAndS3aProperties() {
}
/**
- * Tests {@link IcebergVendedCredentialUtil#propagateToJob} with {@code
jobSecrets == null}.
+ * Tests {@link IcebergVendedCredentialUtil#toJobConf} using only its
properties.
*
* <p><b>Setup:</b> same mixed secret/non-secret credential as the full
test; only
- * {@code jobProperties} map is passed (models {@code
configureInputJobProperties} /
+ * the properties are applied (models {@code configureInputJobProperties} /
* {@code overlayTableProperties}).
*
* <p><b>Expects:</b> overridden endpoint in {@code jobProps}; no serialized
blob and no Iceberg
@@ -200,7 +202,7 @@ public void propagateNonSecretsOnlyWhenJobSecretsNull() {
new BaseTable(new StaticTableOperations("s3://my-bucket/t", fileIO),
"db.t");
Map<String, String> jobProps = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", jobProps, null,
conf);
+ jobProps.putAll(IcebergVendedCredentialUtil.toJobConf(table, "ice01",
conf).properties());
assertThat(jobProps)
.containsEntry(
@@ -248,10 +250,10 @@ public void
extractCredentialsFromFileIoPropertiesWhenCredentialListEmpty() {
}
/**
- * Tests {@link IcebergVendedCredentialUtil#propagateToJob} with {@code
jobProperties == null}.
+ * Tests {@link IcebergVendedCredentialUtil#toJobConf} using only its
secrets.
*
- * <p><b>Setup:</b> credential with access key and secret only; only {@code
jobSecrets} map is
- * passed (models {@code configureInputJobCredentials} for input and output
tables).
+ * <p><b>Setup:</b> credential with access key and secret only; only the
secrets are
+ * applied (models {@code configureInputJobCredentials} for input and output
tables).
*
* <p><b>Expects:</b> serialized blob and {@code
fs.s3a.bucket.*.access.key/secret.key} in
* {@code jobSecrets}; no catalog-prefixed secret keys, no endpoint in
{@code jobSecrets}.
@@ -274,7 +276,7 @@ public void propagateSecretsOnlyWhenJobPropertiesNull() {
new BaseTable(new StaticTableOperations("s3://my-bucket/t", fileIO),
"db.t");
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", null,
jobSecrets, new HiveConf());
+ jobSecrets.putAll(IcebergVendedCredentialUtil.toJobConf(table, "ice01",
new HiveConf()).secrets());
assertThat(jobSecrets)
.containsEntry("fs.s3a.bucket.my-bucket.access.key", "access")
@@ -299,7 +301,7 @@ public void propagateSecretsOnlyWhenJobPropertiesNull() {
}
@Test
- public void propagateToJobMapsGcsProperties() {
+ public void toJobConfMapsGcsProperties() {
Configuration conf = new HiveConf();
conf.set("iceberg.catalog.ice01.gcs.service.host", "http://host-gcs:4443");
@@ -318,7 +320,9 @@ public void propagateToJobMapsGcsProperties() {
Map<String, String> jobProps = Maps.newHashMap();
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", jobProps,
jobSecrets, conf);
+ IcebergVendedCredentialUtil.VendedJobConf vended =
IcebergVendedCredentialUtil.toJobConf(table, "ice01", conf);
+ jobProps.putAll(vended.properties());
+ jobSecrets.putAll(vended.secrets());
assertThat(jobProps)
.containsEntry("iceberg.catalog.ice01.gcs.project-id", "my-project")
@@ -344,7 +348,7 @@ public void propagateToJobMapsGcsProperties() {
}
@Test
- public void propagateToJobMapsAdlsSasToken() {
+ public void toJobConfMapsAdlsSasToken() {
String prefix =
"abfss://[email protected]/";
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
@@ -360,8 +364,10 @@ public void propagateToJobMapsAdlsSasToken() {
Map<String, String> jobProps = Maps.newHashMap();
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(
- table, "ice01", jobProps, jobSecrets, new HiveConf());
+ IcebergVendedCredentialUtil.VendedJobConf vended =
+ IcebergVendedCredentialUtil.toJobConf(table, "ice01", new HiveConf());
+ jobProps.putAll(vended.properties());
+ jobSecrets.putAll(vended.secrets());
assertThat(jobProps)
.containsEntry(
@@ -381,7 +387,7 @@ public void propagateToJobMapsAdlsSasToken() {
}
@Test
- public void propagateToJobMapsAdlsAccountKey() {
+ public void toJobConfMapsAdlsAccountKey() {
String prefix =
"abfss://[email protected]/";
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
@@ -396,7 +402,7 @@ public void propagateToJobMapsAdlsAccountKey() {
new BaseTable(new StaticTableOperations(prefix + "t", fileIO), "db.t");
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", null,
jobSecrets, new HiveConf());
+ jobSecrets.putAll(IcebergVendedCredentialUtil.toJobConf(table, "ice01",
new HiveConf()).secrets());
assertThat(jobSecrets)
.containsEntry(
@@ -406,7 +412,7 @@ public void propagateToJobMapsAdlsAccountKey() {
}
@Test
- public void propagateToJobMapsOssProperties() {
+ public void toJobConfMapsOssProperties() {
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
List.of(
@@ -423,8 +429,10 @@ public void propagateToJobMapsOssProperties() {
Map<String, String> jobProps = Maps.newHashMap();
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(
- table, "ice01", jobProps, jobSecrets, new HiveConf());
+ IcebergVendedCredentialUtil.VendedJobConf vended =
+ IcebergVendedCredentialUtil.toJobConf(table, "ice01", new HiveConf());
+ jobProps.putAll(vended.properties());
+ jobSecrets.putAll(vended.secrets());
assertThat(jobProps)
.containsEntry("iceberg.catalog.ice01.oss.endpoint",
"oss-cn-hangzhou.aliyuncs.com")
@@ -438,11 +446,11 @@ public void propagateToJobMapsOssProperties() {
}
/**
- * Tests {@link IcebergVendedCredentialUtil#propagateToJob} when a REST
catalog vends OSS keys
+ * Tests {@link IcebergVendedCredentialUtil#toJobConf} when a REST catalog
vends OSS keys
* using Hadoop-style aliases ({@code oss-access-key-id}, etc.) instead of
Iceberg canonical names.
*/
@Test
- public void propagateToJobMapsOssAliasProperties() {
+ public void toJobConfMapsOssAliasProperties() {
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
List.of(
@@ -459,8 +467,10 @@ public void propagateToJobMapsOssAliasProperties() {
Map<String, String> jobProps = Maps.newHashMap();
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(
- table, "ice01", jobProps, jobSecrets, new HiveConf());
+ IcebergVendedCredentialUtil.VendedJobConf vended =
+ IcebergVendedCredentialUtil.toJobConf(table, "ice01", new HiveConf());
+ jobProps.putAll(vended.properties());
+ jobSecrets.putAll(vended.secrets());
assertThat(jobProps)
.containsEntry("iceberg.catalog.ice01.oss-endpoint",
"oss-cn-hangzhou.aliyuncs.com")
@@ -505,7 +515,7 @@ public void
extractCredentialsFromOssAliasFileIoPropertiesWhenCredentialListEmpt
}
@Test
- public void propagateToJobMapsS3SessionToken() {
+ public void toJobConfMapsS3SessionToken() {
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
List.of(
@@ -520,7 +530,7 @@ public void propagateToJobMapsS3SessionToken() {
new BaseTable(new StaticTableOperations("s3://my-bucket/t", fileIO),
"db.t");
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", null,
jobSecrets, new HiveConf());
+ jobSecrets.putAll(IcebergVendedCredentialUtil.toJobConf(table, "ice01",
new HiveConf()).secrets());
assertThat(jobSecrets)
.containsEntry("fs.s3a.bucket.my-bucket.session.token",
"session-token");
@@ -547,7 +557,7 @@ public void
extractCredentialsFromGcsFileIoPropertiesWhenCredentialListEmpty() {
}
@Test
- public void
propagateToJobSerializesBlobWithoutHadoopMappingForUnknownScheme() {
+ public void toJobConfSerializesBlobWithoutHadoopMappingForUnknownScheme() {
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
List.of(
@@ -560,8 +570,10 @@ public void
propagateToJobSerializesBlobWithoutHadoopMappingForUnknownScheme() {
Map<String, String> jobProps = Maps.newHashMap();
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(
- table, "ice01", jobProps, jobSecrets, new HiveConf());
+ IcebergVendedCredentialUtil.VendedJobConf vended =
+ IcebergVendedCredentialUtil.toJobConf(table, "ice01", new HiveConf());
+ jobProps.putAll(vended.properties());
+ jobSecrets.putAll(vended.secrets());
assertThat(jobProps)
.containsEntry("iceberg.catalog.ice01.custom.key", "custom-value");
@@ -592,7 +604,7 @@ public void
applyFromJobConfRestoresGcsCredentialsViaCredentialsChannel() throws
tableDesc.setProperties(tableProps);
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(hs2Table, "ice01", null,
jobSecrets, new HiveConf());
+ jobSecrets.putAll(IcebergVendedCredentialUtil.toJobConf(hs2Table, "ice01",
new HiveConf()).secrets());
tableDesc.setJobSecrets(jobSecrets);
JobConf jobConf = new JobConf();
@@ -635,7 +647,7 @@ public void
applyFromJobConfRestoresGcsCredentialsViaCredentialsChannel() throws
* (here {@code gcs.*}), so GCS Hadoop keys are still emitted.
*/
@Test
- public void propagateToJobSelectsMapperByConfigKeyWhenSchemeUnknown() {
+ public void toJobConfSelectsMapperByConfigKeyWhenSchemeUnknown() {
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
List.of(
@@ -648,7 +660,10 @@ public void
propagateToJobSelectsMapperByConfigKeyWhenSchemeUnknown() {
Map<String, String> jobProps = Maps.newHashMap();
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", jobProps,
jobSecrets, new HiveConf());
+ IcebergVendedCredentialUtil.VendedJobConf vended =
+ IcebergVendedCredentialUtil.toJobConf(table, "ice01", new HiveConf());
+ jobProps.putAll(vended.properties());
+ jobSecrets.putAll(vended.secrets());
assertThat(jobProps).containsEntry("fs.gs.project.id", "my-project");
}
@@ -658,7 +673,7 @@ public void
propagateToJobSelectsMapperByConfigKeyWhenSchemeUnknown() {
* authority), which must still yield the correct account-scoped Hadoop
account-key.
*/
@Test
- public void propagateToJobMapsAdlsAccountKeyForAccountOnlyPrefix() {
+ public void toJobConfMapsAdlsAccountKeyForAccountOnlyPrefix() {
String prefix = "abfss://mystorageaccount.dfs.core.windows.net/";
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
@@ -673,14 +688,14 @@ public void
propagateToJobMapsAdlsAccountKeyForAccountOnlyPrefix() {
new BaseTable(new StaticTableOperations(prefix + "t", fileIO), "db.t");
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", null,
jobSecrets, new HiveConf());
+ jobSecrets.putAll(IcebergVendedCredentialUtil.toJobConf(table, "ice01",
new HiveConf()).secrets());
assertThat(jobSecrets)
.containsEntry("fs.azure.account.key.mystorageaccount.dfs.core.windows.net",
"account-key");
}
@Test
- public void propagateToJobMapsAdlsAccountKeyForWasbPrefix() {
+ public void toJobConfMapsAdlsAccountKeyForWasbPrefix() {
String prefix =
"wasb://[email protected]/";
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
@@ -693,7 +708,7 @@ public void propagateToJobMapsAdlsAccountKeyForWasbPrefix()
{
new BaseTable(new StaticTableOperations(prefix + "t", fileIO), "db.t");
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", null,
jobSecrets, new HiveConf());
+ jobSecrets.putAll(IcebergVendedCredentialUtil.toJobConf(table, "ice01",
new HiveConf()).secrets());
assertThat(jobSecrets)
.containsEntry("fs.azure.account.key.devstoreaccount1.blob.core.windows.net",
"account-key");
@@ -705,7 +720,7 @@ public void propagateToJobMapsAdlsAccountKeyForWasbPrefix()
{
* key nor the Hadoop key) — it belongs solely in the secrets channel.
*/
@Test
- public void propagateToJobKeepsOssAccessKeyIdOutOfJobProperties() {
+ public void toJobConfKeepsOssAccessKeyIdOutOfJobProperties() {
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
List.of(
@@ -719,7 +734,7 @@ public void
propagateToJobKeepsOssAccessKeyIdOutOfJobProperties() {
new BaseTable(new StaticTableOperations("oss://my-bucket/t", fileIO),
"db.t");
Map<String, String> jobProps = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", jobProps, null,
new HiveConf());
+ jobProps.putAll(IcebergVendedCredentialUtil.toJobConf(table, "ice01", new
HiveConf()).properties());
assertThat(jobProps)
.doesNotContainKey("fs.oss.accessKeyId")
@@ -728,7 +743,7 @@ public void
propagateToJobKeepsOssAccessKeyIdOutOfJobProperties() {
}
@Test
- public void propagateToJobKeepsOssAliasAccessKeyIdOutOfJobProperties() {
+ public void toJobConfKeepsOssAliasAccessKeyIdOutOfJobProperties() {
CredentialFileIO fileIO = new CredentialFileIO();
fileIO.setCredentials(
List.of(
@@ -742,7 +757,7 @@ public void
propagateToJobKeepsOssAliasAccessKeyIdOutOfJobProperties() {
new BaseTable(new StaticTableOperations("oss://my-bucket/t", fileIO),
"db.t");
Map<String, String> jobProps = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(table, "ice01", jobProps, null,
new HiveConf());
+ jobProps.putAll(IcebergVendedCredentialUtil.toJobConf(table, "ice01", new
HiveConf()).properties());
assertThat(jobProps)
.doesNotContainKey("fs.oss.accessKeyId")
@@ -822,7 +837,7 @@ public void
applyFromJobConfOverridesEndpointOnExistingCredentials() {
/**
* Tests {@link IcebergVendedCredentialUtil#applyFromJobConf} on a task with
an empty FileIO.
*
- * <p><b>Setup:</b> {@code propagateToJob} fills job maps on HS2; keys are
copied into a task
+ * <p><b>Setup:</b> {@code toJobConf} fills job maps on HS2; keys are copied
into a task
* {@code Configuration} as on Tez/LLAP; executor table is deserialized with
a fresh FileIO (no
* credentials).
*
@@ -847,8 +862,10 @@ public void
applyFromJobConfRestoresCredentialsOnExecutor() {
Map<String, String> jobProps = Maps.newHashMap();
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(
- table, "ice01", jobProps, jobSecrets, new Configuration());
+ IcebergVendedCredentialUtil.VendedJobConf vended =
+ IcebergVendedCredentialUtil.toJobConf(table, "ice01", new
Configuration());
+ jobProps.putAll(vended.properties());
+ jobSecrets.putAll(vended.secrets());
Configuration taskConf = new Configuration();
jobProps.forEach(taskConf::set);
@@ -869,7 +886,7 @@ public void applyFromJobConfRestoresCredentialsOnExecutor()
{
}
/**
- * Tests the full HIVE-20651 secret round trip: {@code propagateToJob} →
+ * Tests the full HIVE-20651 secret round trip: {@code toJobConf} →
* {@link org.apache.hadoop.hive.ql.plan.PlanUtils#configureJobConf}
(secrets → Hadoop
* {@code Credentials}, cleared from the TableDesc) → task UGI →
* {@code Utilities#copyJobSecretToTableProperties} → {@code
Utilities#copyTablePropertiesToConf}
@@ -905,7 +922,7 @@ public void
applyFromJobConfRestoresCredentialsViaCredentialsChannel() throws Ex
tableDesc.setProperties(tableProps);
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(hs2Table, "ice01", null,
jobSecrets, new Configuration());
+ jobSecrets.putAll(IcebergVendedCredentialUtil.toJobConf(hs2Table, "ice01",
new Configuration()).secrets());
tableDesc.setJobSecrets(jobSecrets);
JobConf jobConf = new JobConf();
@@ -993,7 +1010,7 @@ public void configureJobConfKeepsSecretsOutOfJobConf() {
* Tests per-table credential resolution when two vended tables publish into
one configuration.
*
* <p><b>Setup:</b> REST catalogs mint credentials per table; two tables on
different buckets go
- * through the full chain — {@code propagateToJob} → {@code
PlanUtils#configureJobConf} → one
+ * through the full chain — {@code toJobConf} → {@code
PlanUtils#configureJobConf} → one
* task UGI → per-table restore — and both publish into a <em>shared</em>
conf the way merged
* map-works do. {@code Utilities#copyTableJobPropertiesToConf} is
skip-if-present for table
* properties, so with a single global blob key the second table would read
the first table's
@@ -1009,14 +1026,14 @@ public void configureJobConfKeepsSecretsOutOfJobConf() {
public void applyFromJobConfResolvesPerTableCredentialsInSharedConf() throws
Exception {
TableDesc tableDesc1 = tableDescNamed("db.t1");
Map<String, String> secrets1 = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(
- tableWithCredential("s3://bucket-a/", "access-a", "db.t1"), "ice01",
null, secrets1, new Configuration());
+ secrets1.putAll(IcebergVendedCredentialUtil.toJobConf(
+ tableWithCredential("s3://bucket-a/", "access-a", "db.t1"), "ice01",
new Configuration()).secrets());
tableDesc1.setJobSecrets(secrets1);
TableDesc tableDesc2 = tableDescNamed("db.t2");
Map<String, String> secrets2 = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(
- tableWithCredential("s3://bucket-b/", "access-b", "db.t2"), "ice01",
null, secrets2, new Configuration());
+ secrets2.putAll(IcebergVendedCredentialUtil.toJobConf(
+ tableWithCredential("s3://bucket-b/", "access-b", "db.t2"), "ice01",
new Configuration()).secrets());
tableDesc2.setJobSecrets(secrets2);
JobConf jobConf = new JobConf();
@@ -1085,7 +1102,7 @@ private static Table tableWithCredential(String prefix,
String accessKey, String
* <p><b>Setup:</b> HS2-side table with a credentialed FileIO (secret in the
credential list and
* in the FileIO properties) over a real metadata file on local disk. {@code
secretFreeCopy}
* rebuilds the table over a fresh FileIO keeping only allowlisted
non-secret properties before
- * {@code serializeTable}; credentials travel separately via {@code
propagateToJob} (modelling
+ * {@code serializeTable}; credentials travel separately via {@code
toJobConf} (modelling
* the HIVE-20651 restore); {@code HiveTableUtil#deserializeTable} restores
them on the
* deserialized FileIO.
*
@@ -1131,7 +1148,7 @@ public void
vendedSerializedTableShipsSecretFreeAndRestoresCredentials() throws
.contains("http://minio:9000");
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(hs2Table, "ice01", null,
jobSecrets, new Configuration());
+ jobSecrets.putAll(IcebergVendedCredentialUtil.toJobConf(hs2Table, "ice01",
new Configuration()).secrets());
JobConf taskConf = new JobConf();
taskConf.set(InputFormatConfig.SERIALIZED_TABLE_PREFIX + "db.t",
serialized);
@@ -1191,7 +1208,7 @@ public void
vendedMetadataTableSerializesSecretFreeAndRestoresCredentials() thro
assertThat(rawBytes).doesNotContain("cred-secret-3fd2").doesNotContain("prop-secret-3fd2");
Map<String, String> jobSecrets = Maps.newHashMap();
- IcebergVendedCredentialUtil.propagateToJob(filesTable, "ice01", null,
jobSecrets, new Configuration());
+ jobSecrets.putAll(IcebergVendedCredentialUtil.toJobConf(filesTable,
"ice01", new Configuration()).secrets());
JobConf taskConf = new JobConf();
taskConf.set(InputFormatConfig.SERIALIZED_TABLE_PREFIX + "db.t.files",
serialized);
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestRestCatalogScanPlanningUtil.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestRestCatalogScanPlanningUtil.java
new file mode 100644
index 00000000000..38be67e57e5
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestRestCatalogScanPlanningUtil.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive;
+
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.hive.IcebergCatalogProperties;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.rest.RESTCatalogProperties;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Unit tests for {@link RestCatalogScanPlanningUtil}: Hive configuration
keys, server-mode detection,
+ * and copying REST catalog settings from the HS2 session into Tez/MR job
configuration. End-to-end
+ * {@code POST /plan} behavior is in {@code
TestRestCatalogScanPlanningServerIT}
+ * ({@code itests/hive-iceberg-rest-server}).
+ */
+public class TestRestCatalogScanPlanningUtil {
+
+ /**
+ * {@link RestCatalogScanPlanningUtil#setScanPlanningMode} must use the
standard
+ * {@code iceberg.catalog.<name>.*} prefix so {@code scan-planning-mode} is
read/written consistently
+ * with other Iceberg catalog properties in {@code hive-site.xml} and
session {@code SET}.
+ */
+ @Test
+ void setScanPlanningModeUsesCatalogPropertyConfigKey() {
+ Configuration conf = new Configuration();
+ RestCatalogScanPlanningUtil.setScanPlanningMode(conf, "ice01", "server");
+ assertThat(conf.get(
+ IcebergCatalogProperties.catalogPropertyConfigKey(
+ "ice01", RESTCatalogProperties.SCAN_PLANNING_MODE)))
+ .isEqualTo("server");
+ }
+
+ /** The Hive kill switch is off by default. */
+ @Test
+ void serverSideScanPlanningIsDisabledByDefault() {
+ Configuration conf = new Configuration();
+
assertThat(RestCatalogScanPlanningUtil.isServerSideScanPlanningEnabled(conf)).isFalse();
+ enableHiveServerMode(conf);
+
assertThat(RestCatalogScanPlanningUtil.isServerSideScanPlanningEnabled(conf)).isTrue();
+ }
+
+ /**
+ * Job propagation uses the per-table catalog name when present; otherwise
the session default
+ * catalog from {@link MetastoreConf.ConfVars#CATALOG_DEFAULT}.
+ */
+ @Test
+ void resolveCatalogNameUsesSessionDefaultWhenTablePropertyMissing() {
+ Configuration conf = new Configuration();
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT,
"ice01");
+ assertThat(HiveTableUtil.resolveCatalogName(conf,
null)).isEqualTo("ice01");
+ assertThat(HiveTableUtil.resolveCatalogName(conf,
"ice02")).isEqualTo("ice02");
+ }
+
+ /**
+ * Server-side planning needs both the Hive kill switch and a table Iceberg
resolved for server-side
+ * planning; a table that is not a {@code RESTTable} never qualifies, even
in client {@code server} mode.
+ */
+ @Test
+ void shouldPlanServerSideRequiresServerPlanningTable() {
+ Table table = Mockito.mock(Table.class);
+
assertThat(RestCatalogScanPlanningUtil.shouldPlanServerSide(restSessionConf(),
table)).isFalse();
+
assertThat(RestCatalogScanPlanningUtil.shouldPlanServerSide(restSessionConf(),
null)).isFalse();
+ }
+
+ /**
+ * The marker follows the table Iceberg loaded, not the client {@code
scan-planning-mode}: a table
+ * that is not a server-planning {@code RESTTable} gets no marker even in
client {@code server} mode.
+ * The positive case needs a REST server and is in {@code
TestHiveIcebergServerSideScanPlanningServerIT}.
+ */
+ @Test
+ void propagateCatalogPropertiesToJobSkipsTableNotPlannedServerSide() {
+ Map<String, String> jobProperties = Maps.newHashMap();
+ RestCatalogScanPlanningUtil.propagateCatalogPropertiesToJob(
+ restSessionConf(), Mockito.mock(Table.class), "db.tbl", jobProperties);
+ assertThat(jobProperties).isEmpty();
+ }
+
+ /**
+ * Every {@code iceberg.catalog.<catalog>.*} entry of the given catalog goes
to job secrets, whatever
+ * its name, so custom auth properties such as {@code header.Authorization}
are never exposed in plain
+ * job properties. Other catalogs and unrelated keys are not copied.
+ */
+ @Test
+ void propagateCatalogSecretsToJobCopiesOnlyTheCatalogProperties() {
+ Configuration sessionConf = restSessionConf();
+ String headerKey =
IcebergCatalogProperties.catalogPropertyConfigKey("ice01",
"header.Authorization");
+ sessionConf.set(headerKey, "Bearer token");
+ String otherCatalogKey =
IcebergCatalogProperties.catalogPropertyConfigKey("ice02", "uri");
+ sessionConf.set(otherCatalogKey, "http://other:8181");
+ sessionConf.set("unrelated.key", "skip");
+
+ Map<String, String> jobSecrets = Maps.newHashMap();
+ RestCatalogScanPlanningUtil.propagateCatalogSecretsToJob("ice01",
jobSecrets, sessionConf);
+
+ assertThat(jobSecrets)
+ .containsEntry(headerKey, "Bearer token")
+
.containsEntry(IcebergCatalogProperties.catalogPropertyConfigKey("ice01",
"uri"), "http://localhost:8181")
+ .doesNotContainKeys(otherCatalogKey, "unrelated.key");
+ }
+
+ private static Configuration restSessionConf() {
+ Configuration sessionConf = new Configuration(false);
+ sessionConf.set(
+ IcebergCatalogProperties.catalogPropertyConfigKey("ice01",
CatalogUtil.ICEBERG_CATALOG_TYPE),
+ CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+ sessionConf.set(
+ IcebergCatalogProperties.catalogPropertyConfigKey("ice01", "uri"),
"http://localhost:8181");
+ RestCatalogScanPlanningUtil.setScanPlanningMode(sessionConf, "ice01",
"server");
+ enableHiveServerMode(sessionConf);
+ return sessionConf;
+ }
+
+ private static void enableHiveServerMode(Configuration conf) {
+ HiveConf.setBoolVar(conf,
HiveConf.ConfVars.HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED, true);
+ }
+}
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestServerSideScanSnapshotPinning.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestServerSideScanSnapshotPinning.java
new file mode 100644
index 00000000000..082e8d9c620
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestServerSideScanSnapshotPinning.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive;
+
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableScan;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.hive.IcebergCatalogProperties;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.mr.InputFormatConfig;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class TestServerSideScanSnapshotPinning {
+
+ private static final String CATALOG_NAME = "ice01";
+ private static final Schema SCHEMA =
+ new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+ private static final PartitionSpec SPEC = PartitionSpec.unpartitioned();
+ private static final String BRANCH = "b";
+
+ @TempDir
+ private Path warehouse;
+
+ @Test
+ void pinsScanToSerializedSnapshotWhenCatalogHasAdvanced() throws Exception {
+ Table table = new HadoopTables().create(SCHEMA, SPEC,
warehouse.toString());
+ appendFile(table, "data-a.parquet");
+ long pinnedSnapshotId = table.currentSnapshot().snapshotId();
+
+ Configuration conf = buildServerModeConf(table);
+ Table serializedTable = HiveTableUtil.deserializeTable(conf, table.name());
+
+ appendFile(table, "data-b.parquet");
+
assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(pinnedSnapshotId);
+
+ TableScan scan =
+ HiveTableUtil.pinServerSideScanToSerializedTable(conf,
serializedTable, table.newScan());
+ List<String> plannedPaths = Lists.newArrayList();
+ try (CloseableIterable<FileScanTask> tasks = scan.planFiles()) {
+ tasks.forEach(task -> plannedPaths.add(task.file().location()));
+ }
+ assertThat(plannedPaths)
+ .anyMatch(path -> path.contains("data-a.parquet"))
+ .noneMatch(path -> path.contains("data-b.parquet"));
+
assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(pinnedSnapshotId);
+ }
+
+ @Test
+ void pinnedScanAllowsColumnSelection() {
+ Table table = new HadoopTables().create(SCHEMA, SPEC,
warehouse.toString());
+ appendFile(table, "data-a.parquet");
+ Configuration conf = buildServerModeConf(table);
+ Table serializedTable = HiveTableUtil.deserializeTable(conf, table.name());
+
+ TableScan scan =
+ HiveTableUtil.pinServerSideScanToSerializedTable(conf,
serializedTable, table.newScan())
+ .select("id");
+
assertThat(scan.schema().columns()).extracting(Types.NestedField::name).containsExactly("id");
+ }
+
+ @Test
+ void failsWhenRefMissingFromSerializedTable() {
+ Table table = new HadoopTables().create(SCHEMA, SPEC,
warehouse.toString());
+ appendFile(table, "data-a.parquet");
+ Configuration conf = buildServerModeConf(table);
+ Table serializedTable = HiveTableUtil.deserializeTable(conf, table.name());
+ table.manageSnapshots().createBranch(BRANCH,
table.currentSnapshot().snapshotId()).commit();
+ TableScan scan = table.newScan();
+
+ assertThatThrownBy(() ->
HiveTableUtil.pinServerSideScanRefToSerializedTable(serializedTable, BRANCH,
scan))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Cannot find reference " + BRANCH);
+ }
+
+ @Test
+ void pinsOlderSchemaBranchWithUseRefWhenBranchHasNotMoved() throws Exception
{
+ Table table = new HadoopTables().create(SCHEMA, SPEC,
warehouse.toString());
+ appendFile(table, "data-a.parquet");
+ long branchSnapshotId = table.currentSnapshot().snapshotId();
+ table.manageSnapshots().createBranch(BRANCH, branchSnapshotId).commit();
+
+ table.updateSchema().addColumn("extra", Types.StringType.get()).commit();
+ appendFile(table, "data-b.parquet");
+
+ Configuration conf = buildServerModeConf(table);
+ Table serializedTable = HiveTableUtil.deserializeTable(conf, table.name());
+ assertThat(serializedTable.snapshot(branchSnapshotId).schemaId())
+ .isNotEqualTo(serializedTable.schema().schemaId());
+
+ TableScan scan =
+ HiveTableUtil.pinServerSideScanRefToSerializedTable(serializedTable,
BRANCH, table.newScan());
+ assertThat(scan.snapshot().snapshotId()).isEqualTo(branchSnapshotId);
+ assertThat(scan.schema().columns())
+ .extracting(Types.NestedField::name)
+ .containsExactly("id", "extra");
+
+ List<String> plannedPaths = Lists.newArrayList();
+ try (CloseableIterable<FileScanTask> tasks = scan.planFiles()) {
+ tasks.forEach(task -> plannedPaths.add(task.file().location()));
+ }
+ assertThat(plannedPaths)
+ .anyMatch(path -> path.contains("data-a.parquet"))
+ .noneMatch(path -> path.contains("data-b.parquet"));
+ }
+
+ @Test
+ void failsWhenBranchMovedSinceCompilation() {
+ Table table = new HadoopTables().create(SCHEMA, SPEC,
warehouse.toString());
+ appendFile(table, "data-a.parquet");
+ long branchSnapshotId = table.currentSnapshot().snapshotId();
+ table.manageSnapshots().createBranch(BRANCH, branchSnapshotId).commit();
+
+ table.updateSchema().addColumn("extra", Types.StringType.get()).commit();
+ appendFile(table, "data-b.parquet");
+
+ Configuration conf = buildServerModeConf(table);
+ Table serializedTable = HiveTableUtil.deserializeTable(conf, table.name());
+ SnapshotRef compileTimeRef = serializedTable.refs().get(BRANCH);
+ assertThat(compileTimeRef).isNotNull();
+ assertThat(compileTimeRef.snapshotId()).isEqualTo(branchSnapshotId);
+ assertThat(serializedTable.snapshot(branchSnapshotId).schemaId())
+ .isNotEqualTo(serializedTable.schema().schemaId());
+
+ appendFileOnBranch(table, BRANCH, "data-c.parquet");
+
assertThat(table.refs().get(BRANCH).snapshotId()).isNotEqualTo(branchSnapshotId);
+ TableScan scan = table.newScan();
+
+ assertThatThrownBy(() ->
HiveTableUtil.pinServerSideScanRefToSerializedTable(serializedTable, BRANCH,
scan))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Branch " + BRANCH + " moved since query
compilation");
+ }
+
+ @Test
+ void pinsTagWithUseSnapshotWhenServerSideTableReloaded() throws Exception {
+ Table table = new HadoopTables().create(SCHEMA, SPEC,
warehouse.toString());
+ appendFile(table, "data-a.parquet");
+ long taggedSnapshot = table.currentSnapshot().snapshotId();
+ String tag = "t1";
+ table.manageSnapshots().createTag(tag, taggedSnapshot).commit();
+
+ Configuration conf = buildServerModeConf(table);
+ Table serializedTable = HiveTableUtil.deserializeTable(conf, table.name());
+
+ appendFile(table, "data-b.parquet");
+
assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(taggedSnapshot);
+
+ TableScan scan =
+ HiveTableUtil.pinServerSideScanRefToSerializedTable(serializedTable,
tag, table.newScan());
+ List<String> plannedPaths = Lists.newArrayList();
+ try (CloseableIterable<FileScanTask> tasks = scan.planFiles()) {
+ tasks.forEach(task -> plannedPaths.add(task.file().location()));
+ }
+ assertThat(plannedPaths)
+ .anyMatch(path -> path.contains("data-a.parquet"))
+ .noneMatch(path -> path.contains("data-b.parquet"));
+ }
+
+ @Test
+ void doesNotPinWhenExplicitSnapshotVersionIsSet() {
+ Table table = new HadoopTables().create(SCHEMA, SPEC,
warehouse.toString());
+ Configuration conf = buildServerModeConf(table);
+ conf.setLong(InputFormatConfig.SNAPSHOT_ID, 42L);
+ Table serializedTable = HiveTableUtil.deserializeTable(conf, table.name());
+
+ TableScan scan = table.newScan();
+ TableScan pinned =
+ HiveTableUtil.pinServerSideScanToSerializedTable(conf,
serializedTable, scan);
+ assertThat(pinned).isSameAs(scan);
+ }
+
+ private void appendFile(Table table, String fileName) {
+ table.newAppend()
+ .appendFile(
+ DataFiles.builder(table.spec())
+ .withPath(warehouse + "/" + fileName)
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .build())
+ .commit();
+ }
+
+ private void appendFileOnBranch(Table table, String branch, String fileName)
{
+ table.newAppend()
+ .toBranch(branch)
+ .appendFile(
+ DataFiles.builder(table.spec())
+ .withPath(warehouse + "/" + fileName)
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .build())
+ .commit();
+ }
+
+ private Configuration buildServerModeConf(Table table) {
+ Configuration conf = new Configuration();
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT,
CATALOG_NAME);
+ conf.set(InputFormatConfig.TABLE_IDENTIFIER, table.name());
+ conf.set(InputFormatConfig.CATALOG_NAME, CATALOG_NAME);
+ conf.set(
+ IcebergCatalogProperties.catalogPropertyConfigKey(CATALOG_NAME,
CatalogUtil.ICEBERG_CATALOG_TYPE),
+ CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+ RestCatalogScanPlanningUtil.setScanPlanningMode(conf, CATALOG_NAME,
"server");
+ HiveConf.setBoolVar(conf,
HiveConf.ConfVars.HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED, true);
+ conf.set(
+ InputFormatConfig.SERIALIZED_TABLE_PREFIX + table.name(),
+ HiveTableUtil.serializeTable(table, conf, null, null));
+ return conf;
+ }
+}
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/rest/TestHiveIcebergServerSideScanPlanning.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/rest/TestHiveIcebergServerSideScanPlanning.java
new file mode 100644
index 00000000000..f7a6f81145a
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/rest/TestHiveIcebergServerSideScanPlanning.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.rest;
+
+import java.nio.file.Path;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.MetadataTableType;
+import org.apache.iceberg.MetadataTableUtils;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SerializableTable;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.hive.IcebergCatalogProperties;
+import org.apache.iceberg.mr.InputFormatConfig;
+import org.apache.iceberg.mr.hive.HiveTableUtil;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Unit tests for the reload guard {@link
HiveTableUtil#shouldReloadForServerSideScanPlanning}. The
+ * Tez AM reloads from the REST catalog only when HS2 set the per-table marker
+ * {@link InputFormatConfig#serverSideScanPlanningKey(String)} and the scan
can be planned by the
+ * server. End-to-end behavior is in {@code
TestHiveIcebergServerSideScanPlanningServerIT}.
+ */
+class TestHiveIcebergServerSideScanPlanning {
+
+ private static final String CATALOG_NAME = "ice01";
+ private static final Schema SCHEMA =
+ new Schema(Types.NestedField.required(1, "id", Types.LongType.get()));
+
+ @TempDir
+ private Path warehouse;
+
+ private Table table;
+
+ @BeforeEach
+ void before() {
+ table = new HadoopTables().create(SCHEMA, PartitionSpec.unpartitioned(),
warehouse.toString());
+ }
+
+ @Test
+ void reloadsWhenHs2SetMarker() {
+ Configuration conf = buildTableConf(true);
+ assertThat(HiveTableUtil.shouldReloadForServerSideScanPlanning(conf,
serialized(table))).isTrue();
+ }
+
+ /**
+ * The AM does not re-derive the decision from catalog type, {@code
scan-planning-mode} or the Hive
+ * switch: without the HS2 marker it keeps the serialized table.
+ */
+ @Test
+ void doesNotReloadWithoutMarker() {
+ Configuration conf = buildTableConf(false);
+ assertThat(HiveTableUtil.shouldReloadForServerSideScanPlanning(conf,
serialized(table))).isFalse();
+ }
+
+ /**
+ * The marker is per table: split generation reuses one job conf for all
tables of a vertex, so the
+ * marker of one table must not make another table in the same conf reload.
+ */
+ @Test
+ void doesNotReloadOtherTableInSameConf() {
+ Configuration conf = buildTableConf(true);
+ conf.set(InputFormatConfig.TABLE_IDENTIFIER, "db.other");
+ assertThat(HiveTableUtil.shouldReloadForServerSideScanPlanning(conf,
serialized(table))).isFalse();
+ }
+
+ /**
+ * Intra-transaction read-after-write: {@link
InputFormatConfig#TABLE_METADATA_LOCATION} points at
+ * uncommitted metadata that only exists in the job conf; reloading would
lose it.
+ */
+ @Test
+ void doesNotReloadForIntraTxnMetadata() {
+ Configuration conf = buildTableConf(true);
+ conf.set(InputFormatConfig.TABLE_METADATA_LOCATION, warehouse +
"/metadata/snapshot.metadata.json");
+ assertThat(HiveTableUtil.shouldReloadForServerSideScanPlanning(conf,
serialized(table))).isFalse();
+ }
+
+ /** {@code RESTTable} plans incremental append scans locally; reloading
would only add drift. */
+ @Test
+ void doesNotReloadForIncrementalAppendScan() {
+ Configuration conf = buildTableConf(true);
+ conf.setLong(InputFormatConfig.SNAPSHOT_ID_INTERVAL_FROM, 1L);
+ assertThat(HiveTableUtil.shouldReloadForServerSideScanPlanning(conf,
serialized(table))).isFalse();
+ }
+
+ /** Metadata tables ({@code tbl.files}, {@code tbl.snapshots}, ...) are
planned locally. */
+ @Test
+ void doesNotReloadForMetadataTable() {
+ Configuration conf = buildTableConf(true);
+ Table filesTable = MetadataTableUtils.createMetadataTableInstance(table,
MetadataTableType.FILES);
+ assertThat(HiveTableUtil.shouldReloadForServerSideScanPlanning(conf,
serialized(filesTable))).isFalse();
+ }
+
+ private static Table serialized(Table source) {
+ return SerializableTable.copyOf(source);
+ }
+
+ private Configuration buildTableConf(boolean marker) {
+ Configuration conf = new Configuration();
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT,
CATALOG_NAME);
+ conf.set(InputFormatConfig.TABLE_IDENTIFIER, table.name());
+ conf.set(InputFormatConfig.CATALOG_NAME, CATALOG_NAME);
+ conf.set(
+ IcebergCatalogProperties.catalogPropertyConfigKey(CATALOG_NAME,
CatalogUtil.ICEBERG_CATALOG_TYPE),
+ CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+ if (marker) {
+
conf.setBoolean(InputFormatConfig.serverSideScanPlanningKey(table.name()),
true);
+ }
+ return conf;
+ }
+}
diff --git a/iceberg/iceberg-rest-catalog-client/pom.xml
b/iceberg/iceberg-rest-catalog-client/pom.xml
index a41437f881f..cb73dbabb3e 100644
--- a/iceberg/iceberg-rest-catalog-client/pom.xml
+++ b/iceberg/iceberg-rest-catalog-client/pom.xml
@@ -75,17 +75,5 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-core</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
</dependencies>
</project>
diff --git a/itests/hive-iceberg-rest-server/pom.xml
b/itests/hive-iceberg-rest-server/pom.xml
new file mode 100644
index 00000000000..5c49a4676e4
--- /dev/null
+++ b/itests/hive-iceberg-rest-server/pom.xml
@@ -0,0 +1,234 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-it</artifactId>
+ <version>4.3.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <artifactId>hive-it-iceberg-rest-server</artifactId>
+ <packaging>jar</packaging>
+ <name>Hive Iceberg Integration - In-process REST catalog server tests</name>
+ <description>
+ Integration tests that use Iceberg's in-process REST catalog server (for
example
+ TestBaseWithRESTServer and the iceberg-core REST test harness) together
with Hive
+ Iceberg executor code. Test classpath uses unshaded Iceberg and unpacks
only
+ org.apache.iceberg.mr classes from hive-iceberg-handler so the REST
harness is not
+ mixed with the shaded handler jar. For Hive metastore REST catalog client
tests
+ (Keycloak, Testcontainers, and similar), see hive-it-iceberg.
+ </description>
+ <properties>
+ <hive.path.to.root>../..</hive.path.to.root>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <!-- Jetty 12 for Iceberg REST embedded-server tests; Hive uses
jetty.version 9.x elsewhere. -->
+ <iceberg.rest.test.jetty.version>12.1.8</iceberg.rest.test.jetty.version>
+ </properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ <version>${iceberg.rest.test.jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-io</artifactId>
+ <version>${iceberg.rest.test.jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ <version>${iceberg.rest.test.jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${iceberg.rest.test.jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ <version>${iceberg.rest.test.jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty.ee10</groupId>
+ <artifactId>jetty-ee10-servlet</artifactId>
+ <version>${iceberg.rest.test.jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty.compression</groupId>
+ <artifactId>jetty-compression-server</artifactId>
+ <version>${iceberg.rest.test.jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty.compression</groupId>
+ <artifactId>jetty-compression-gzip</artifactId>
+ <version>${iceberg.rest.test.jetty.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-iceberg-rest-catalog-client</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-iceberg-shading</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-iceberg-handler</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>patched-iceberg-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>patched-iceberg-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-core</artifactId>
+ <version>${iceberg.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>${junit.jupiter.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-io</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty.ee10</groupId>
+ <artifactId>jetty-ee10-servlet</artifactId>
+ <scope>test</scope>
+ <exclusions>
+ <!-- Avoid compile-scoped jetty-security 12 on the module classpath
(breaks root javadoc:aggregate). -->
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-security</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty.compression</groupId>
+ <artifactId>jetty-compression-server</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty.compression</groupId>
+ <artifactId>jetty-compression-gzip</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>unpack-handler-mr-classes</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>unpack</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-iceberg-handler</artifactId>
+ <version>${project.version}</version>
+ <includes>org/apache/iceberg/mr/**</includes>
+
<outputDirectory>${project.build.directory}/handler-mr-classes</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- Iceberg REST test harness needs unshaded Jackson; handler jar
embeds shaded REST. -->
+ <classpathDependencyExcludes>
+
<classpathDependencyExclude>org.apache.hive:hive-iceberg-shading</classpathDependencyExclude>
+
<classpathDependencyExclude>org.apache.hive:hive-iceberg-handler</classpathDependencyExclude>
+ </classpathDependencyExcludes>
+ <additionalClasspathElements>
+
<additionalClasspathElement>${project.build.directory}/handler-mr-classes</additionalClasspathElement>
+ </additionalClasspathElements>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git
a/itests/hive-iceberg-rest-server/src/test/java/org/apache/iceberg/rest/TestHiveIcebergServerSideScanPlanningServerIT.java
b/itests/hive-iceberg-rest-server/src/test/java/org/apache/iceberg/rest/TestHiveIcebergServerSideScanPlanningServerIT.java
new file mode 100644
index 00000000000..1592efea2d6
--- /dev/null
+++
b/itests/hive-iceberg-rest-server/src/test/java/org/apache/iceberg/rest/TestHiveIcebergServerSideScanPlanningServerIT.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.rest;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.function.Consumer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.Scan;
+import org.apache.iceberg.SerializableTable;
+import org.apache.iceberg.SnapshotRef;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hive.IcebergCatalogProperties;
+import org.apache.iceberg.mr.Catalogs;
+import org.apache.iceberg.mr.hive.RestCatalogScanPlanningUtil;
+import org.apache.iceberg.mr.InputFormatConfig;
+import org.apache.iceberg.mr.hive.HiveIcebergStorageHandler;
+import org.apache.iceberg.mr.hive.HiveTableUtil;
+import org.apache.iceberg.mr.mapreduce.IcebergInputFormat;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.rest.requests.PlanTableScanRequest;
+import org.apache.iceberg.rest.responses.ErrorResponse;
+import org.apache.iceberg.rest.responses.LoadTableResponse;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import static org.apache.iceberg.TestBase.FILE_A;
+import static org.apache.iceberg.TestBase.SCHEMA;
+import static org.apache.iceberg.TestBase.SPEC;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Integration tests for Hive server-side REST catalog scan planning via
+ * {@code IcebergInputFormat#getSplits}: with catalog settings propagated from
HS2 into the executor
+ * job conf, split generation reloads a live {@link RESTTable} from the
embedded REST server (then
+ * closes the catalog) instead of planning only from the serialized snapshot
({@link SerializableTable}).
+ *
+ * <p>Without propagation, or when reload guards apply, executors keep the
snapshot; those cases are
+ * unit-tested in {@code TestHiveIcebergServerSideScanPlanning} ({@code
iceberg-handler}). Iceberg
+ * REST client {@code POST /plan} behavior is in {@code
TestRestCatalogScanPlanningServerIT}.
+ */
+class TestHiveIcebergServerSideScanPlanningServerIT extends
TestBaseWithRESTServer {
+
+ private static final String CATALOG_NAME = "hive-iceberg-scan-planning";
+ private static final TableIdentifier TABLE_ID = TableIdentifier.of(NS,
"file_planning_table");
+
+ @Override
+ protected RESTCatalogAdapter createAdapterForServer() {
+ return spy(
+ new RESTCatalogAdapter(backendCatalog) {
+ @Override
+ protected <T extends RESTResponse> T execute(
+ HTTPRequest request,
+ Class<T> responseType,
+ Consumer<ErrorResponse> errorHandler,
+ Consumer<Map<String, String>> responseHeaders) {
+ Object body = roundTripSerialize(request.body(), "request");
+ HTTPRequest req =
ImmutableHTTPRequest.builder().from(request).body(body).build();
+ T response = super.execute(req, responseType, errorHandler,
responseHeaders);
+
+ if (response instanceof LoadTableResponse) {
+ return RESTCatalogAdapter.castResponse(
+ responseType,
+ withPlanningMode(
+ (LoadTableResponse) response,
+
RESTCatalogProperties.ScanPlanningMode.SERVER.modeName()));
+ }
+
+ if (req.body() instanceof PlanTableScanRequest) {
+ return response;
+ }
+
+ return roundTripSerialize(response, "response");
+ }
+ });
+ }
+
+ @Override
+ protected String catalogName() {
+ return CATALOG_NAME;
+ }
+
+ @Override
+ protected Map<String, String> additionalCatalogProperties() {
+ Configuration conf = new Configuration();
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT,
CATALOG_NAME);
+ RestCatalogScanPlanningUtil.setScanPlanningMode(conf, CATALOG_NAME,
"server");
+ HiveConf.setBoolVar(conf,
HiveConf.ConfVars.HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED, true);
+ return IcebergCatalogProperties.getCatalogProperties(conf, CATALOG_NAME);
+ }
+
+ @BeforeEach
+ @Override
+ public void before() throws Exception {
+ super.before();
+ adapterForRESTServer.setPlanningBehavior(
+ new RESTCatalogAdapter.PlanningBehavior() {
+ @Override
+ public boolean shouldPlanTableScanAsync(Scan<?, FileScanTask, ?>
scan) {
+ return false;
+ }
+
+ @Override
+ public int numberFileScanTasksPerPlanTask() {
+ return 100;
+ }
+ });
+ }
+
+ /**
+ * Negative path: without the HS2 per-table marker and catalog settings in
the job conf, split generation
+ * plans from the serialized table snapshot and never sends {@code POST
/plan}.
+ */
+ @Test
+ void executorJobConfWithoutPropagationUsesSerializedTable() {
+ Table table = createTableWithData();
+ Configuration jobConf = executorJobConf(table);
+ jobConf.set(InputFormatConfig.TABLE_LOCATION, table.location());
+
+ assertThat(planRequestsDuringGetSplits(jobConf)).isEmpty();
+ }
+
+ /**
+ * Positive path: when catalog properties are propagated from HS2 into the
executor job conf,
+ * split generation reloads a live {@link RESTTable} from the REST catalog
instead of using the
+ * serialized metadata snapshot.
+ */
+ @Test
+ void propagatedJobConfReloadsRestTableFromCatalog() throws IOException {
+ Table table = createTableWithData();
+ Configuration jobConf = executorJobConf(table);
+ propagateFromHs2(sessionConf(), table, jobConf);
+
+ ArgumentCaptor<HTTPRequest> requestCaptor =
ArgumentCaptor.forClass(HTTPRequest.class);
+ try (RESTCatalog catalog = loadRestCatalog(jobConf)) {
+ Table resolved = catalog.loadTable(TABLE_ID);
+ assertThat(resolved).isInstanceOf(RESTTable.class);
+ assertThat(resolved.newScan()).isInstanceOf(RESTTableScan.class);
+ }
+
+ verify(adapterForRESTServer, atLeastOnce())
+ .execute(requestCaptor.capture(), any(), any(), any(), any());
+ assertThat(
+ requestCaptor.getAllValues().stream()
+ .anyMatch(req -> req.path().contains("/tables/")))
+ .as("Expected Hive split planning to reload the table from the REST
catalog")
+ .isTrue();
+ }
+
+ /**
+ * The REST server returns {@code scan-planning-mode=server} in {@code
LoadTableResponse} while the
+ * client {@code scan-planning-mode} is unset: Iceberg loads a
server-planning {@link RESTTable}, and
+ * Hive follows it instead of planning locally from manifests.
+ */
+ @Test
+ void serverDecidedPlanningModeIsFollowedWithoutClientProperty() throws
IOException {
+ createTableWithData();
+ Configuration sessionConf = sessionConf();
+ sessionConf.unset(
+ IcebergCatalogProperties.catalogPropertyConfigKey(CATALOG_NAME,
RESTCatalogProperties.SCAN_PLANNING_MODE));
+
+ try (RESTCatalog catalog = loadRestCatalog(sessionConf)) {
+ Table table = catalog.loadTable(TABLE_ID);
+ assertThat(table).isInstanceOf(RESTTable.class);
+ assertThat(planRequestsDuringGetSplits(propagatedJobConf(sessionConf,
table))).hasSize(1);
+ }
+ }
+
+ /** With the Hive kill switch off, a table the server plans is still planned
locally by Hive. */
+ @Test
+ void hiveSwitchOffKeepsLocalPlanningForServerPlannedTable() {
+ Table table = createTableWithData();
+ Configuration sessionConf = sessionConf();
+ HiveConf.setBoolVar(sessionConf,
HiveConf.ConfVars.HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED, false);
+
+ assertThat(table).isInstanceOf(RESTTable.class);
+ assertThat(catalogSecrets(sessionConf)).isEmpty();
+ assertThat(planRequestsDuringGetSplits(propagatedJobConf(sessionConf,
table))).isEmpty();
+ }
+
+ /**
+ * For a table Iceberg resolved for server-side planning, every {@code
iceberg.catalog.<catalog>.*}
+ * entry of the table's catalog goes to job secrets (HIVE-20651), whatever
its name, so custom auth
+ * properties such as {@code header.Authorization} are never exposed in
plain job properties. Other
+ * catalogs and unrelated keys are not copied.
+ */
+ @Test
+ void catalogSecretsCopyAllCatalogPropertiesForServerPlannedTable() {
+ Table table = createTableWithData();
+ Configuration sessionConf = sessionConf();
+ String headerKey =
IcebergCatalogProperties.catalogPropertyConfigKey(CATALOG_NAME,
"header.Authorization");
+ sessionConf.set(headerKey, "Bearer token");
+ String otherCatalogKey =
IcebergCatalogProperties.catalogPropertyConfigKey("other",
CatalogProperties.URI);
+ sessionConf.set(otherCatalogKey, "http://other:8181");
+ sessionConf.set("unrelated.key", "skip");
+
+ assertThat(table).isInstanceOf(RESTTable.class);
+ assertThat(catalogSecrets(sessionConf))
+ .containsEntry(headerKey, "Bearer token")
+ .containsEntry(
+ IcebergCatalogProperties.catalogPropertyConfigKey(CATALOG_NAME,
CatalogProperties.URI),
+ httpServer.getURI().toString())
+ .containsEntry(
+ IcebergCatalogProperties.catalogPropertyConfigKey(CATALOG_NAME,
CatalogUtil.ICEBERG_CATALOG_TYPE),
+ CatalogUtil.ICEBERG_CATALOG_TYPE_REST)
+ .doesNotContainKey(otherCatalogKey)
+ .doesNotContainKey("unrelated.key");
+ }
+
+ /**
+ * Compile-time snapshot is serialized into the job conf; if the catalog
advances before split
+ * generation, server-side planning must still plan the serialized snapshot,
not {@code HEAD}.
+ */
+ @Test
+ void serverSidePlanningPinsSerializedSnapshotAfterCatalogAdvances() {
+ Table table = createTableWithData();
+ long pinnedSnapshotId = table.currentSnapshot().snapshotId();
+
+ Configuration jobConf = propagatedJobConf(table);
+
+ DataFile fileB =
+ DataFiles.builder(SPEC)
+ .withPath("/path/to/data-b.parquet")
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .build();
+ restCatalog.loadTable(TABLE_ID).newAppend().appendFile(fileB).commit();
+ assertThat(restCatalog.loadTable(TABLE_ID).currentSnapshot().snapshotId())
+ .isNotEqualTo(pinnedSnapshotId);
+
+ assertThat(planRequestsDuringGetSplits(jobConf))
+ .singleElement()
+ .extracting(PlanTableScanRequest::snapshotId)
+ .isEqualTo(pinnedSnapshotId);
+ }
+
+ /** Time travel by snapshot id is sent to the REST server as the plan
request snapshot. */
+ @Test
+ void serverSidePlanningUsesExplicitSnapshotId() {
+ Table table = createTableWithData();
+ long firstSnapshotId = table.currentSnapshot().snapshotId();
+ appendFileB();
+
+ Configuration jobConf = propagatedJobConf(restCatalog.loadTable(TABLE_ID));
+ jobConf.setLong(InputFormatConfig.SNAPSHOT_ID, firstSnapshotId);
+
+ assertThat(planRequestsDuringGetSplits(jobConf))
+ .singleElement()
+ .extracting(PlanTableScanRequest::snapshotId)
+ .isEqualTo(firstSnapshotId);
+ }
+
+ /**
+ * Time travel by tag name resolves the ref against the compile-time table,
so moving the ref in
+ * the catalog after compilation does not change what is planned.
+ */
+ @Test
+ void serverSidePlanningResolvesRefAgainstSerializedTable() {
+ Table table = createTableWithData();
+ long taggedSnapshotId = table.currentSnapshot().snapshotId();
+ table.manageSnapshots().createTag("v1", taggedSnapshotId).commit();
+
+ Configuration jobConf = propagatedJobConf(restCatalog.loadTable(TABLE_ID));
+ jobConf.set(InputFormatConfig.SNAPSHOT_ID, "v1");
+
+ appendFileB();
+ Table live = restCatalog.loadTable(TABLE_ID);
+ live.manageSnapshots().replaceTag("v1",
live.currentSnapshot().snapshotId()).commit();
+
+ assertThat(planRequestsDuringGetSplits(jobConf))
+ .singleElement()
+ .extracting(PlanTableScanRequest::snapshotId)
+ .isEqualTo(taggedSnapshotId);
+ }
+
+ /**
+ * Reading {@code t.branch_b1} while another writer commits to {@code b1}
between query compilation
+ * and split planning: the scan must use the {@code b1} head captured at
compile time, not the new
+ * head the reloaded REST table sees.
+ */
+ @Test
+ void serverSidePlanningPinsBranchToSerializedTable() {
+ Table table = createTableWithData();
+ long branchHeadId = table.currentSnapshot().snapshotId();
+ table.manageSnapshots().createBranch("b1", branchHeadId).commit();
+
+ Configuration jobConf = propagatedJobConf(restCatalog.loadTable(TABLE_ID));
+ jobConf.set(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF, "branch_b1");
+
+ appendFileB("b1");
+
assertThat(restCatalog.loadTable(TABLE_ID).refs().get("b1").snapshotId()).isNotEqualTo(branchHeadId);
+
+ assertThat(planRequestsDuringGetSplits(jobConf))
+ .singleElement()
+ .extracting(PlanTableScanRequest::snapshotId)
+ .isEqualTo(branchHeadId);
+ }
+
+ /**
+ * Reading {@code t.branch_b1} after a column was added to the table but no
new data was written to
+ * {@code b1}: Iceberg reads branches with the current table schema, so
{@code new_col} must still be
+ * selected. The scan therefore uses the ref instead of pinning the snapshot
id, which would switch to
+ * the older snapshot schema without {@code new_col}.
+ */
+ @Test
+ void serverSidePlanningKeepsTableSchemaForBranchWithOlderSchema() {
+ Table table = createTableWithData();
+ long branchHeadId = table.currentSnapshot().snapshotId();
+ table.manageSnapshots().createBranch("b1", branchHeadId).commit();
+ restCatalog.loadTable(TABLE_ID).updateSchema().addColumn("new_col",
Types.StringType.get()).commit();
+
+ Configuration jobConf = propagatedJobConf(restCatalog.loadTable(TABLE_ID));
+ jobConf.set(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF, "branch_b1");
+
+ PlanTableScanRequest planRequest =
planRequestsDuringGetSplits(jobConf).get(0);
+ assertThat(planRequest.snapshotId()).isEqualTo(branchHeadId);
+ assertThat(planRequest.useSnapshotSchema()).isFalse();
+ assertThat(planRequest.select()).contains("new_col");
+ }
+
+ /** Incremental append scans are planned locally from the serialized table,
without POST /plan. */
+ @Test
+ void incrementalAppendScanDoesNotUseServerSidePlanning() {
+ Table table = createTableWithData();
+ long firstSnapshotId = table.currentSnapshot().snapshotId();
+ appendFileB();
+
+ Configuration jobConf = propagatedJobConf(restCatalog.loadTable(TABLE_ID));
+ jobConf.setLong(InputFormatConfig.SNAPSHOT_ID_INTERVAL_FROM,
firstSnapshotId);
+
+ assertThat(planRequestsDuringGetSplits(jobConf)).isEmpty();
+ }
+
+ private static RESTCatalog loadRestCatalog(Configuration conf) {
+ return (RESTCatalog) Catalogs.loadCatalog(conf,
CATALOG_NAME).orElseThrow();
+ }
+
+ private Configuration propagatedJobConf(Table table) {
+ return propagatedJobConf(sessionConf(), table);
+ }
+
+ private Configuration propagatedJobConf(Configuration sessionConf, Table
table) {
+ Configuration jobConf = executorJobConf(table);
+ jobConf.set(InputFormatConfig.TABLE_LOCATION, table.location());
+ propagateFromHs2(sessionConf, table, jobConf);
+ return jobConf;
+ }
+
+ /**
+ * Mimics what split generation in the AM sees: the per-table marker from
the table's job
+ * properties, decided on the table HS2 loaded, and the catalog properties
from job secrets; both are
+ * copied into the conf by {@code HiveInputFormat#addSplitsForGroup}.
+ */
+ private void propagateFromHs2(Configuration sessionConf, Table table,
Configuration jobConf) {
+ Map<String, String> jobProperties = Maps.newHashMap();
+ RestCatalogScanPlanningUtil.propagateCatalogPropertiesToJob(
+ sessionConf, table, TABLE_ID.toString(), jobProperties);
+ jobProperties.forEach(jobConf::set);
+ catalogSecrets(sessionConf).forEach(jobConf::set);
+ }
+
+ /** Job secrets HS2 builds for the table in {@code
HiveIcebergStorageHandler#configureInputJobCredentials}. */
+ private static Map<String, String> catalogSecrets(Configuration sessionConf)
{
+ TableDesc tableDesc = new TableDesc();
+ Properties tableProps = new Properties();
+ tableProps.setProperty(InputFormatConfig.CATALOG_NAME, CATALOG_NAME);
+ tableProps.setProperty(Catalogs.NAME, TABLE_ID.toString());
+ tableDesc.setProperties(tableProps);
+ HiveIcebergStorageHandler storageHandler = new HiveIcebergStorageHandler();
+ storageHandler.setConf(sessionConf);
+ Map<String, String> jobSecrets = Maps.newHashMap();
+ storageHandler.configureInputJobCredentials(tableDesc, jobSecrets);
+ return jobSecrets;
+ }
+
+ private void appendFileB() {
+ appendFileB(SnapshotRef.MAIN_BRANCH);
+ }
+
+ private void appendFileB(String branch) {
+ DataFile fileB =
+ DataFiles.builder(SPEC)
+ .withPath("/path/to/data-b.parquet")
+ .withFileSizeInBytes(10)
+ .withRecordCount(1)
+ .build();
+
restCatalog.loadTable(TABLE_ID).newAppend().appendFile(fileB).toBranch(branch).commit();
+ }
+
+ private List<PlanTableScanRequest> planRequestsDuringGetSplits(Configuration
jobConf) {
+ Mockito.clearInvocations(adapterForRESTServer);
+ JobContext jobContext = new JobContextImpl(jobConf, new
JobID("server_side_scan", 1));
+ new IcebergInputFormat<>().getSplits(jobContext);
+
+ ArgumentCaptor<HTTPRequest> requestCaptor =
ArgumentCaptor.forClass(HTTPRequest.class);
+ verify(adapterForRESTServer, Mockito.atLeast(0))
+ .execute(requestCaptor.capture(), any(), any(), any(), any());
+ return requestCaptor.getAllValues().stream()
+ .map(HTTPRequest::body)
+ .filter(PlanTableScanRequest.class::isInstance)
+ .map(PlanTableScanRequest.class::cast)
+ .toList();
+ }
+
+ private Table createTableWithData() {
+ restCatalog.createNamespace(NS);
+ Table table =
+ restCatalog.buildTable(TABLE_ID,
SCHEMA).withPartitionSpec(SPEC).create();
+ table.newAppend().appendFile(FILE_A).commit();
+ return table;
+ }
+
+ private Configuration sessionConf() {
+ Configuration sessionConf = new Configuration();
+ MetastoreConf.setVar(sessionConf, MetastoreConf.ConfVars.CATALOG_DEFAULT,
CATALOG_NAME);
+ sessionConf.set(
+ IcebergCatalogProperties.catalogPropertyConfigKey(CATALOG_NAME,
CatalogUtil.ICEBERG_CATALOG_TYPE),
+ CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+ restCatalog.properties().forEach(
+ (key, value) ->
+ sessionConf.set(
+
IcebergCatalogProperties.catalogPropertyConfigKey(CATALOG_NAME, key), value));
+ sessionConf.set(
+ IcebergCatalogProperties.catalogPropertyConfigKey(CATALOG_NAME,
CatalogProperties.URI),
+ httpServer.getURI().toString());
+ RestCatalogScanPlanningUtil.setScanPlanningMode(sessionConf, CATALOG_NAME,
"server");
+ HiveConf.setBoolVar(sessionConf,
HiveConf.ConfVars.HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED, true);
+ return sessionConf;
+ }
+
+ private Configuration executorJobConf(Table table) {
+ Configuration jobConf = new Configuration();
+ jobConf.set(InputFormatConfig.TABLE_IDENTIFIER, TABLE_ID.toString());
+ jobConf.set(InputFormatConfig.CATALOG_NAME, CATALOG_NAME);
+ jobConf.set(
+ InputFormatConfig.SERIALIZED_TABLE_PREFIX + TABLE_ID.toString(),
+ HiveTableUtil.serializeTable(table, jobConf, null, null));
+ return jobConf;
+ }
+
+ private static LoadTableResponse withPlanningMode(LoadTableResponse
response, String mode) {
+ return LoadTableResponse.builder()
+ .withTableMetadata(response.tableMetadata())
+ .addAllConfig(response.config())
+ .addConfig(RESTCatalogProperties.SCAN_PLANNING_MODE, mode)
+ .addAllCredentials(response.credentials())
+ .build();
+ }
+}
diff --git
a/itests/hive-iceberg-rest-server/src/test/java/org/apache/iceberg/rest/TestRestCatalogScanPlanningServerIT.java
b/itests/hive-iceberg-rest-server/src/test/java/org/apache/iceberg/rest/TestRestCatalogScanPlanningServerIT.java
new file mode 100644
index 00000000000..e43ebbb3844
--- /dev/null
+++
b/itests/hive-iceberg-rest-server/src/test/java/org/apache/iceberg/rest/TestRestCatalogScanPlanningServerIT.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.rest;
+
+import java.util.Map;
+import java.util.function.Consumer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.Scan;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hive.IcebergCatalogProperties;
+import org.apache.iceberg.mr.hive.RestCatalogScanPlanningUtil;
+import org.apache.iceberg.rest.requests.PlanTableScanRequest;
+import org.apache.iceberg.rest.responses.ErrorResponse;
+import org.apache.iceberg.rest.responses.LoadTableResponse;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import static org.apache.iceberg.TestBase.FILE_A;
+import static org.apache.iceberg.TestBase.SCHEMA;
+import static org.apache.iceberg.TestBase.SPEC;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Embedded REST server tests for {@link RestCatalogScanPlanningUtil}: when
Hive-style configuration sets
+ * {@code scan-planning-mode=server}, {@link RESTTable} / {@link
RESTTableScan} delegate split
+ * planning to the catalog server ({@code POST /plan}).
+ */
+class TestRestCatalogScanPlanningServerIT extends TestBaseWithRESTServer {
+
+ private static final String CATALOG_NAME = "hive-rest-scan-planning";
+
+ @Override
+ protected RESTCatalogAdapter createAdapterForServer() {
+ return spy(
+ new RESTCatalogAdapter(backendCatalog) {
+ @Override
+ protected <T extends RESTResponse> T execute(
+ HTTPRequest request,
+ Class<T> responseType,
+ Consumer<ErrorResponse> errorHandler,
+ Consumer<Map<String, String>> responseHeaders) {
+ Object body = roundTripSerialize(request.body(), "request");
+ HTTPRequest req =
ImmutableHTTPRequest.builder().from(request).body(body).build();
+ T response = super.execute(req, responseType, errorHandler,
responseHeaders);
+
+ if (response instanceof LoadTableResponse) {
+ return RESTCatalogAdapter.castResponse(
+ responseType,
+ withPlanningMode(
+ (LoadTableResponse) response,
+
RESTCatalogProperties.ScanPlanningMode.SERVER.modeName()));
+ }
+
+ if (req.body() instanceof PlanTableScanRequest) {
+ return response;
+ }
+
+ return roundTripSerialize(response, "response");
+ }
+ });
+ }
+
+ @Override
+ protected String catalogName() {
+ return CATALOG_NAME;
+ }
+
+ @Override
+ protected Map<String, String> additionalCatalogProperties() {
+ Configuration conf = new Configuration();
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT,
CATALOG_NAME);
+ RestCatalogScanPlanningUtil.setScanPlanningMode(conf, CATALOG_NAME,
"server");
+ return IcebergCatalogProperties.getCatalogProperties(conf, CATALOG_NAME);
+ }
+
+ @BeforeEach
+ @Override
+ public void before() throws Exception {
+ super.before();
+ adapterForRESTServer.setPlanningBehavior(
+ new RESTCatalogAdapter.PlanningBehavior() {
+ @Override
+ public boolean shouldPlanTableScanAsync(Scan<?, FileScanTask, ?>
scan) {
+ return false;
+ }
+
+ @Override
+ public int numberFileScanTasksPerPlanTask() {
+ return 100;
+ }
+ });
+ }
+
+ /**
+ * Positive path: REST catalog loaded with server planning mode returns a
{@link RESTTable}; calling
+ * {@code planTasks()} on a scan issues a {@link PlanTableScanRequest} to
the embedded REST server.
+ */
+ @Test
+ void hiveCatalogConfigurationIssuesPlanTableScanRequest() {
+ Configuration conf = new Configuration();
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT,
CATALOG_NAME);
+ RestCatalogScanPlanningUtil.setScanPlanningMode(conf, CATALOG_NAME,
"server");
+ HiveConf.setBoolVar(conf,
HiveConf.ConfVars.HIVE_ICEBERG_REST_SERVER_SIDE_SCAN_PLANNING_ENABLED, true);
+ assertThat(IcebergCatalogProperties.getCatalogProperties(conf,
CATALOG_NAME))
+ .containsEntry(RESTCatalogProperties.SCAN_PLANNING_MODE, "server");
+ assertThat(RestCatalogScanPlanningUtil.getScanPlanningMode(conf,
CATALOG_NAME))
+ .isEqualTo(RESTCatalogProperties.ScanPlanningMode.SERVER);
+
+ restCatalog.createNamespace(NS);
+ Table table =
+ restCatalog.buildTable(TableIdentifier.of(NS, "scan_planning_table"),
SCHEMA)
+ .withPartitionSpec(SPEC)
+ .create();
+ table.newAppend().appendFile(FILE_A).commit();
+
+ parserContext =
+ ParserContext.builder()
+ .add("specsById", table.specs())
+ .add("caseSensitive", false)
+ .build();
+
+ assertThat(table).isInstanceOf(RESTTable.class);
+ assertThat(table.newScan()).isInstanceOf(RESTTableScan.class);
+
+ ArgumentCaptor<HTTPRequest> requestCaptor =
ArgumentCaptor.forClass(HTTPRequest.class);
+ assertThat(table.newScan().planTasks()).isNotEmpty();
+
+ verify(adapterForRESTServer, atLeastOnce())
+ .execute(requestCaptor.capture(), any(), any(), any(), any());
+ assertThat(
+ requestCaptor.getAllValues().stream()
+ .anyMatch(req -> req.body() instanceof PlanTableScanRequest))
+ .as("Expected server-side scan planning via POST /plan
(PlanTableScanRequest)")
+ .isTrue();
+ }
+
+ private static LoadTableResponse withPlanningMode(LoadTableResponse
response, String mode) {
+ return LoadTableResponse.builder()
+ .withTableMetadata(response.tableMetadata())
+ .addAllConfig(response.config())
+ .addConfig(RESTCatalogProperties.SCAN_PLANNING_MODE, mode)
+ .addAllCredentials(response.credentials())
+ .build();
+ }
+}
diff --git a/itests/pom.xml b/itests/pom.xml
index 5a2716d1dd2..448f4d29885 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -31,6 +31,8 @@
<hive.path.to.root>..</hive.path.to.root>
<wiremock-jre8.version>2.32.0</wiremock-jre8.version>
<assertj.version>3.27.3</assertj.version>
+ <!-- Jetty 12 / EE10 REST catalog ITs (hive-it-iceberg-rest-server) -->
+ <jakarta.servlet-api.version>6.0.0</jakarta.servlet-api.version>
</properties>
<modules>
<module>custom-serde</module>
@@ -50,6 +52,7 @@
<module>qtest-iceberg</module>
<module>test-docker</module>
<module>hive-iceberg</module>
+ <module>hive-iceberg-rest-server</module>
<module>tez-yarn-it</module>
</modules>
<dependencyManagement>
@@ -492,6 +495,21 @@
<artifactId>hive-iceberg-handler</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>patched-iceberg-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>patched-iceberg-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>jakarta.servlet</groupId>
+ <artifactId>jakarta.servlet-api</artifactId>
+ <version>${jakarta.servlet-api.version}</version>
+ </dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
@@ -503,6 +521,11 @@
<artifactId>awaitility</artifactId>
<version>${awaitility.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito-core.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
<build>