This is an automated email from the ASF dual-hosted git repository.
okumin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 55be6f58fa0 HIVE-28932: Remove llap-external-client cloud deployment
(#6724)
55be6f58fa0 is described below
commit 55be6f58fa02298ed7251c0b60dd612a23a9e39d
Author: Shohei Okumiya <[email protected]>
AuthorDate: Thu Oct 29 19:16:00 2026 +0900
HIVE-28932: Remove llap-external-client cloud deployment (#6724)
* HIVE-28932: Remove llap-external-client cloud deployment
* Apply cosmetic changes
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 28 ------
.../hadoop/hive/llap/ext/TestLlapInputSplit.java | 3 +-
.../apache/hadoop/hive/llap/LlapInputSplit.java | 20 ++---
.../hive/llap/registry/LlapServiceInstance.java | 22 -----
.../registry/impl/InactiveServiceInstance.java | 10 ---
.../llap/registry/impl/LlapFixedRegistryImpl.java | 18 ----
.../registry/impl/LlapZookeeperRegistryImpl.java | 49 +---------
llap-common/pom.xml | 12 ---
.../java/org/apache/hadoop/hive/llap/LlapUtil.java | 20 -----
.../security/DefaultJwtSharedSecretProvider.java | 100 ---------------------
.../hive/llap/security/JwtSecretProvider.java | 73 ---------------
.../hive/llap/security/LlapExtClientJwtHelper.java | 71 ---------------
llap-common/src/protobuf/LlapDaemonProtocol.proto | 7 +-
.../hadoop/hive/llap/LlapBaseInputFormat.java | 3 -
.../llap/cli/service/AsyncTaskCopyLocalJars.java | 3 -
.../hadoop/hive/llap/daemon/impl/AMReporter.java | 26 +-----
.../hive/llap/daemon/impl/ContainerRunnerImpl.java | 40 ---------
.../hadoop/hive/llap/daemon/impl/LlapDaemon.java | 25 +-----
.../llap/daemon/impl/LlapProtocolServerImpl.java | 33 ++-----
.../hive/llap/daemon/impl/TaskRunnerCallable.java | 2 +-
.../hive/llap/daemon/LlapDaemonExtension.java | 2 +-
.../hadoop/hive/llap/daemon/MiniLlapCluster.java | 9 +-
.../hive/llap/daemon/impl/TestLlapDaemon.java | 6 +-
.../impl/TestLlapDaemonProtocolServerImpl.java | 6 +-
.../daemon/impl/comparator/TestAMReporter.java | 4 +-
.../llap/io/encoded/TestLlapOrcCacheLoader.java | 14 +--
pom.xml | 16 ----
.../hive/ql/udf/generic/GenericUDTFGetSplits.java | 27 ++----
28 files changed, 50 insertions(+), 599 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index a4f8f91b3f8..8a5561fab6d 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -5432,34 +5432,6 @@ public static enum ConfVars {
LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT("hive.llap.external.splits.temp.table.storage.format",
"orc", new StringSet("default", "text", "orc"),
"Storage format for temp tables created using LLAP external client"),
-
LLAP_EXTERNAL_CLIENT_USE_HYBRID_CALENDAR("hive.llap.external.client.use.hybrid.calendar",
- false,
- "Whether to use hybrid calendar for parsing of data/timestamps."),
-
- // ====== confs for llap-external-client cloud deployment ======
- LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED(
- "hive.llap.external.client.cloud.deployment.setup.enabled", false,
- "Tells whether to enable additional RPC port, auth mechanism for llap
external clients. This is meant"
- + "for cloud based deployments. When true, it has following
effects - \n"
- + "1. Enables an extra RPC port on LLAP daemon to accept fragments
from external clients. See"
- + "hive.llap.external.client.cloud.rpc.port\n"
- + "2. Uses external hostnames of LLAP in splits, so that clients
can submit from outside of cloud. "
- + "Env variable PUBLIC_HOSTNAME should be available on LLAP
machines.\n"
- + "3. Uses JWT based authentication for splits to be validated at
LLAP. See "
- + "hive.llap.external.client.cloud.jwt.shared.secret.provider"),
-
LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT("hive.llap.external.client.cloud.rpc.port",
30004,
- "The LLAP daemon RPC port for external clients when llap is running in
cloud environment."),
-
LLAP_EXTERNAL_CLIENT_CLOUD_OUTPUT_SERVICE_PORT("hive.llap.external.client.cloud.output.service.port",
30005,
- "LLAP output service port when llap is running in cloud
environment"),
- LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER(
- "hive.llap.external.client.cloud.jwt.shared.secret.provider",
- "org.apache.hadoop.hive.llap.security.DefaultJwtSharedSecretProvider",
- "Shared secret provider to be used to sign JWT"),
-
LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET("hive.llap.external.client.cloud.jwt.shared.secret",
- "",
- "The LLAP daemon RPC port for external clients when llap is running in
cloud environment. "
- + "Length of the secret should be >= 32 bytes"),
- // ====== confs for llap-external-client cloud deployment ======
LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap",
false,
"Override if grace join should be allowed to run in llap."),
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java
index 9ee65a1c314..d0afa3e9a9f 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java
@@ -58,7 +58,7 @@ public void testWritable() throws Exception {
byte[] tokenBytes = new byte[] { 1 };
LlapInputSplit split1 = new LlapInputSplit(splitNum, planBytes,
fragmentBytes, null,
- locations, llapDaemonInfos, schema, "hive", tokenBytes,
"some-dummy-jwt");
+ locations, llapDaemonInfos, schema, "hive", tokenBytes);
ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(byteOutStream);
split1.write(dataOut);
@@ -89,7 +89,6 @@ static void checkLlapSplits(LlapInputSplit split1,
LlapInputSplit split2) throws
assertArrayEquals(split1.getLocations(), split2.getLocations());
assertEquals(split1.getSchema().toString(), split2.getSchema().toString());
assertEquals(split1.getLlapUser(), split2.getLlapUser());
- assertEquals(split1.getJwt(), split2.getJwt());
assertArrayEquals(split1.getLlapDaemonInfos(),
split2.getLlapDaemonInfos());
}
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
index 619346fd968..f52618c4e21 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
@@ -37,10 +37,6 @@ public class LlapInputSplit implements
InputSplitWithLocationInfo {
private String llapUser;
private byte[] fragmentBytesSignature;
private byte[] tokenBytes;
- //only needed in cloud deployments for llap server to validate request from
external llap clients.
- //HS2 generates a JWT and populates this field while get_splits() call, this
jwt gets validated at LLAP server
- //when LlapInputSplit is submitted.
- private String jwt;
public LlapInputSplit() {
}
@@ -48,7 +44,7 @@ public LlapInputSplit() {
public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes,
byte[] fragmentBytesSignature, SplitLocationInfo[]
locations,
LlapDaemonInfo[] llapDaemonInfos, Schema schema,
- String llapUser, byte[] tokenBytes, String jwt) {
+ String llapUser, byte[] tokenBytes) {
this.planBytes = planBytes;
this.fragmentBytes = fragmentBytes;
this.fragmentBytesSignature = fragmentBytesSignature;
@@ -58,7 +54,6 @@ public LlapInputSplit(int splitNum, byte[] planBytes, byte[]
fragmentBytes,
this.splitNum = splitNum;
this.llapUser = llapUser;
this.tokenBytes = tokenBytes;
- this.jwt = jwt;
}
public Schema getSchema() {
@@ -107,10 +102,6 @@ public void setSchema(Schema schema) {
this.schema = schema;
}
- public String getJwt() {
- return jwt;
- }
-
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(splitNum);
@@ -145,9 +136,8 @@ public void write(DataOutput out) throws IOException {
out.writeInt(0);
}
- if (jwt != null) {
- out.writeUTF(jwt);
- }
+ // Retain the retired JWT field in the wire format for older readers.
+ out.writeUTF("");
}
@Override
@@ -187,7 +177,9 @@ public void readFields(DataInput in) throws IOException {
tokenBytes = new byte[length];
in.readFully(tokenBytes);
}
- jwt = in.readUTF();
+
+ // Discard the retired JWT field kept for wire compatibility.
+ in.readUTF();
}
@Override
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java
index 75c00970e8d..aff0ecbcbd9 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/LlapServiceInstance.java
@@ -19,9 +19,6 @@
package org.apache.hadoop.hive.llap.registry;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.registry.ServiceInstance;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -55,25 +52,6 @@ public interface LlapServiceInstance extends ServiceInstance
{
*/
public int getOutputFormatPort();
- /**
- * External host, usually needed in cloud envs where we cannot access
internal host from outside
- *
- * @return
- */
- String getExternalHostname();
-
- /**
- * RPC endpoint for external clients - tcp traffic on this port should be
opened on cloud.
- *
- * @return
- */
- int getExternalClientsRpcPort();
-
-
- default void ensureCloudEnv(Configuration conf) {
- Preconditions.checkState(LlapUtil.isCloudDeployment(conf), "Only supported
in cloud based deployments");
- }
-
/**
* Memory and Executors available for the LLAP tasks
*
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
index 58948fa68d6..48e44b8741d 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/InactiveServiceInstance.java
@@ -56,16 +56,6 @@ public int getShufflePort() {
throw new UnsupportedOperationException();
}
- @Override
- public String getExternalHostname() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int getExternalClientsRpcPort() {
- throw new UnsupportedOperationException();
- }
-
@Override
public String getServicesAddress() {
throw new UnsupportedOperationException();
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
index af65301a219..75c3ad9ea59 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
@@ -40,11 +40,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet;
import org.apache.hadoop.hive.llap.registry.ServiceRegistry;
-import org.apache.hadoop.hive.registry.ServiceInstance;
import org.apache.hadoop.hive.registry.ServiceInstanceStateChangeListener;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.StringUtils;
@@ -65,9 +63,7 @@ public class LlapFixedRegistryImpl implements
ServiceRegistry<LlapServiceInstanc
private final int shuffle;
private final int mngPort;
private final int webPort;
- private Configuration conf;
private final int outputFormatPort;
- private final int externalClientsRpcPort;
private final String webScheme;
private final String[] hosts;
private final int memory;
@@ -83,10 +79,8 @@ public LlapFixedRegistryImpl(String hosts, Configuration
conf) {
this.resolveHosts = conf.getBoolean(FIXED_REGISTRY_RESOLVE_HOST_NAMES,
true);
this.mngPort = HiveConf.getIntVar(conf, ConfVars.LLAP_MANAGEMENT_RPC_PORT);
this.outputFormatPort = HiveConf.getIntVar(conf,
ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT);
- this.externalClientsRpcPort = HiveConf.getIntVar(conf,
ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT);
this.webPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_WEB_PORT);
- this.conf = conf;
boolean isSsl = HiveConf.getBoolVar(conf, ConfVars.LLAP_DAEMON_WEB_SSL);
this.webScheme = isSsl ? "https" : "http";
@@ -199,18 +193,6 @@ public int getOutputFormatPort() {
return LlapFixedRegistryImpl.this.outputFormatPort;
}
- @Override
- public String getExternalHostname() {
- ensureCloudEnv(LlapFixedRegistryImpl.this.conf);
- return LlapUtil.getPublicHostname();
- }
-
- @Override
- public int getExternalClientsRpcPort() {
- ensureCloudEnv(LlapFixedRegistryImpl.this.conf);
- return LlapFixedRegistryImpl.this.externalClientsRpcPort;
- }
-
@Override
public String getServicesAddress() {
return serviceAddress;
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
index b9a9c36339d..5584a21e77c 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
@@ -42,7 +42,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet;
@@ -73,7 +72,6 @@ public class LlapZookeeperRegistryImpl
private static final String IPC_SHUFFLE = "shuffle";
private static final String IPC_LLAP = "llap";
private static final String IPC_OUTPUTFORMAT = "llapoutputformat";
- private static final String IPC_EXTERNAL_LLAP = "externalllap";
private final static String NAMESPACE_PREFIX = "llap-";
private static final String SLOT_PREFIX = "slot-";
private static final String SASL_LOGIN_CONTEXT_NAME = "LlapZooKeeperClient";
@@ -137,12 +135,6 @@ public Endpoint getOutputFormatEndpoint() {
HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT)));
}
- private Endpoint getExternalRpcEndpoint() {
- int port = HiveConf.getIntVar(conf,
ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT);
- String host = LlapUtil.getPublicHostname();
- return RegistryTypeUtils.ipcEndpoint(IPC_EXTERNAL_LLAP, new
InetSocketAddress(host, port));
- }
-
@Override
public String register() throws IOException {
daemonZkRecord = new ServiceRecord();
@@ -152,11 +144,6 @@ public String register() throws IOException {
daemonZkRecord.addInternalEndpoint(getShuffleEndpoint());
daemonZkRecord.addExternalEndpoint(getServicesEndpoint());
daemonZkRecord.addInternalEndpoint(getOutputFormatEndpoint());
- Endpoint externalRpcEndpoint = null;
- if (LlapUtil.isCloudDeployment(conf)) {
- externalRpcEndpoint = getExternalRpcEndpoint();
- daemonZkRecord.addExternalEndpoint(externalRpcEndpoint);
- }
populateConfigValues(this.conf);
Map<String, String> capacityValues = new HashMap<>(2);
@@ -196,15 +183,9 @@ public String register() throws IOException {
}
registerServiceRecord(daemonZkRecord, uniqueId);
- if (LlapUtil.isCloudDeployment(conf)) {
- LOG.info("Registered node. Created a znode on ZooKeeper for LLAP
instance: rpc: {}, external client rpc : {} "
- + "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}",
rpcEndpoint, externalRpcEndpoint,
- getShuffleEndpoint(), getServicesEndpoint(), getMngEndpoint(),
getRegistrationZnodePath());
- } else {
- LOG.info("Registered node. Created a znode on ZooKeeper for LLAP
instance: rpc: {}, "
- + "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}",
rpcEndpoint, getShuffleEndpoint(),
- getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath());
- }
+ LOG.info("Registered node. Created a znode on ZooKeeper for LLAP instance:
rpc: {}, "
+ + "shuffle: {}, webui: {}, mgmt: {}, znodePath: {}", rpcEndpoint,
getShuffleEndpoint(),
+ getServicesEndpoint(), getMngEndpoint(), getRegistrationZnodePath());
return uniqueId;
}
@@ -242,9 +223,6 @@ public class DynamicServiceInstance
private final int outputFormatPort;
private final String serviceAddress;
- private String externalHost;
- private int externalClientsRpcPort;
-
private final Resource resource;
public DynamicServiceInstance(ServiceRecord srv) throws IOException {
@@ -267,15 +245,6 @@ public DynamicServiceInstance(ServiceRecord srv) throws
IOException {
this.serviceAddress =
RegistryTypeUtils.getAddressField(services.addresses.get(0),
AddressTypes.ADDRESS_URI);
- if (LlapUtil.isCloudDeployment(conf)) {
- final Endpoint externalRpc =
srv.getExternalEndpoint(IPC_EXTERNAL_LLAP);
- this.externalHost =
RegistryTypeUtils.getAddressField(externalRpc.addresses.get(0),
- AddressTypes.ADDRESS_HOSTNAME_FIELD);
- this.externalClientsRpcPort = Integer.parseInt(
- RegistryTypeUtils.getAddressField(externalRpc.addresses.get(0),
- AddressTypes.ADDRESS_PORT_FIELD));
- }
-
String memStr =
srv.get(ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, "");
String coreStr =
srv.get(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, "");
try {
@@ -296,18 +265,6 @@ public String getServicesAddress() {
return serviceAddress;
}
- @Override
- public String getExternalHostname() {
- ensureCloudEnv(LlapZookeeperRegistryImpl.this.conf);
- return externalHost;
- }
-
- @Override
- public int getExternalClientsRpcPort() {
- ensureCloudEnv(LlapZookeeperRegistryImpl.this.conf);
- return externalClientsRpcPort;
- }
-
@Override
public Resource getResource() {
return resource;
diff --git a/llap-common/pom.xml b/llap-common/pom.xml
index 36468f8f79f..3812e77041c 100644
--- a/llap-common/pom.xml
+++ b/llap-common/pom.xml
@@ -48,18 +48,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
- <dependency>
- <groupId>io.jsonwebtoken</groupId>
- <artifactId>jjwt-api</artifactId>
- </dependency>
- <dependency>
- <groupId>io.jsonwebtoken</groupId>
- <artifactId>jjwt-impl</artifactId>
- </dependency>
- <dependency>
- <groupId>io.jsonwebtoken</groupId>
- <artifactId>jjwt-jackson</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
index ce8e188f6e8..c35840e649b 100644
--- a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
@@ -297,24 +297,4 @@ public static Credentials credentialsFromByteArray(byte[]
binaryCredentials)
credentials.readTokenStorageStream(dib);
return credentials;
}
-
- /**
- * @return returns the value of
LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED
- * @param conf
- */
- public static boolean isCloudDeployment(Configuration conf) {
- return HiveConf.getBoolVar(conf,
ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED, false);
- }
-
- /**
- * @return returns the value of PUBLIC_HOSTNAME from either environment
variable or system properties
- */
- public static String getPublicHostname() {
- String publicHostname = System.getenv("PUBLIC_HOSTNAME");
- if (publicHostname == null) {
- publicHostname = System.getProperty("PUBLIC_HOSTNAME");
- }
- return publicHostname;
- }
-
}
diff --git
a/llap-common/src/java/org/apache/hadoop/hive/llap/security/DefaultJwtSharedSecretProvider.java
b/llap-common/src/java/org/apache/hadoop/hive/llap/security/DefaultJwtSharedSecretProvider.java
deleted file mode 100644
index 1e04c2a58bf..00000000000
---
a/llap-common/src/java/org/apache/hadoop/hive/llap/security/DefaultJwtSharedSecretProvider.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.llap.security;
-
-import com.google.common.base.Preconditions;
-import io.jsonwebtoken.security.Keys;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.StandardCharsets;
-import java.security.Key;
-
-import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED;
-import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET;
-
-/**
- * Default implementation of {@link JwtSecretProvider}.
- *
- * 1. It first tries to get shared secret from conf {@link
HiveConf.ConfVars#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET}
- * using {@link Configuration#getPassword(String)}.
- *
- * 2. If not found, it tries to read from env var {@link
#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR}.
- *
- * If secret is not found even after 1) and 2), {@link #init(Configuration)}
methods throws {@link IllegalStateException}.
- *
- * Length of shared secret provided in 1) or 2) should be > 32 bytes.
- *
- * It uses the same encryption and decryption secret which can be used to sign
and verify JWT.
- */
-public class DefaultJwtSharedSecretProvider implements JwtSecretProvider {
-
- public static final String
LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR =
- "LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR";
-
- private Key jwtEncryptionKey;
-
- @Override public Key getEncryptionSecret() {
- return jwtEncryptionKey;
- }
-
- @Override public Key getDecryptionSecret() {
- return jwtEncryptionKey;
- }
-
- @Override public void init(final Configuration conf) {
- char[] sharedSecret;
- byte[] sharedSecretBytes = null;
-
- // try getting secret from conf first
- // if not found, get from env var -
LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR
- try {
- sharedSecret =
conf.getPassword(LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET.varname);
- } catch (IOException e) {
- throw new RuntimeException("Unable to get password
[hive.llap.external.client.cloud.jwt.shared.secret] - "
- + e.getMessage(), e);
- }
- if (sharedSecret != null) {
- ByteBuffer bb =
StandardCharsets.UTF_8.encode(CharBuffer.wrap(sharedSecret));
- sharedSecretBytes = new byte[bb.remaining()];
- bb.get(sharedSecretBytes);
- } else {
- String sharedSecredFromEnv =
System.getenv(LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR);
- if (StringUtils.isNotBlank(sharedSecredFromEnv)) {
- sharedSecretBytes = sharedSecredFromEnv.getBytes();
- }
- }
-
- Preconditions.checkState(sharedSecretBytes != null,
- "With: " + LLAP_EXTERNAL_CLIENT_CLOUD_DEPLOYMENT_SETUP_ENABLED.varname
+ " = true, \n"
- + "To use:
org.apache.hadoop.hive.llap.security.DefaultJwtSharedSecretProvider, \n"
- + "1. a non-null value of
'hive.llap.external.client.cloud.jwt.shared.secret' must be provided OR \n"
- + "2. alternatively environment variable "
- + "LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_ENV_VAR can also
be set. \n"
- + "Length of the secret provided in 1) or 2) should be > 32
bytes.");
-
- this.jwtEncryptionKey = Keys.hmacShaKeyFor(sharedSecretBytes);
- }
-
-}
diff --git
a/llap-common/src/java/org/apache/hadoop/hive/llap/security/JwtSecretProvider.java
b/llap-common/src/java/org/apache/hadoop/hive/llap/security/JwtSecretProvider.java
deleted file mode 100644
index c401f182c64..00000000000
---
a/llap-common/src/java/org/apache/hadoop/hive/llap/security/JwtSecretProvider.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.llap.security;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-import java.security.Key;
-
-/**
- * JwtSecretProvider
- *
- * - provides encryption and decryption secrets for generating and parsing
JWTs.
- *
- * - Hive internally uses method initAndGet() which initializes providers
based on the value of config
- * {@link
HiveConf.ConfVars#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER}.
- * It expects implementations to provide default constructor and {@link
#init(Configuration)} method.
- */
-public interface JwtSecretProvider {
-
- /**
- * returns secret for signing JWT.
- */
- Key getEncryptionSecret();
-
- /**
- * returns secret for parsing JWT.
- */
- Key getDecryptionSecret();
-
- /**
- * Initializes the provider.
- * Should also contain any validations that we want to put on secret, helps
us to fail fast.
- * @param conf configuration
- */
- void init(Configuration conf);
-
- /**
- * Hive internally uses this method to obtain instance of {@link
JwtSecretProvider}
- *
- * @param conf configuration
- * @return implementation of {@link
HiveConf.ConfVars#LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER}
- */
- static JwtSecretProvider initAndGet(Configuration conf) {
- final String providerClass =
- HiveConf.getVar(conf,
HiveConf.ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_JWT_SHARED_SECRET_PROVIDER);
- JwtSecretProvider provider;
- try {
- provider = (JwtSecretProvider)
Class.forName(providerClass).newInstance();
- provider.init(conf);
- } catch (InstantiationException | IllegalAccessException |
ClassNotFoundException e) {
- throw new RuntimeException("Unable to instantiate provider: " +
providerClass, e);
- }
- return provider;
- }
-}
diff --git
a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapExtClientJwtHelper.java
b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapExtClientJwtHelper.java
deleted file mode 100644
index 55500546a9d..00000000000
---
a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapExtClientJwtHelper.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.llap.security;
-
-import io.jsonwebtoken.Claims;
-import io.jsonwebtoken.Jws;
-import io.jsonwebtoken.Jwts;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-
-import java.util.Date;
-import java.util.UUID;
-
-/**
- * Contains helper methods for generating and verifying JWTs for external llap
clients.
- * Initializes and uses {@link JwtSecretProvider} to obtain encryption and
decryption secret.
- */
-public class LlapExtClientJwtHelper {
-
- public static final String LLAP_JWT_SUBJECT = "llap";
- public static final String LLAP_EXT_CLIENT_APP_ID = "llap_ext_client_app_id";
- private final JwtSecretProvider jwtSecretProvider;
-
- public LlapExtClientJwtHelper(Configuration conf) {
- this.jwtSecretProvider = JwtSecretProvider.initAndGet(conf);
- }
-
- /**
- * @param extClientAppId application Id - application Id injected by
get_splits
- * @return JWT signed with {@link JwtSecretProvider#getEncryptionSecret()}.
- * As of now this JWT contains extClientAppId in claims.
- */
- public String buildJwtForLlap(ApplicationId extClientAppId) {
- return Jwts.builder()
- .setSubject(LLAP_JWT_SUBJECT)
- .setIssuedAt(new Date())
- .setId(UUID.randomUUID().toString())
- .claim(LLAP_EXT_CLIENT_APP_ID, extClientAppId.toString())
- .signWith(jwtSecretProvider.getEncryptionSecret())
- .compact();
- }
-
- /**
- *
- * @param jwt signed JWT String
- * @return claims present in JWT, this method parses jwt using {@link
JwtSecretProvider#getDecryptionSecret()}
- */
- public Jws<Claims> parseClaims(String jwt) {
- return Jwts.parser()
- .setSigningKey(jwtSecretProvider.getDecryptionSecret())
- .parseClaimsJws(jwt);
- }
-
-}
diff --git a/llap-common/src/protobuf/LlapDaemonProtocol.proto
b/llap-common/src/protobuf/LlapDaemonProtocol.proto
index 8b15f4392eb..4359fbc5b70 100644
--- a/llap-common/src/protobuf/LlapDaemonProtocol.proto
+++ b/llap-common/src/protobuf/LlapDaemonProtocol.proto
@@ -132,8 +132,11 @@ message SubmitWorkRequestProto {
optional bytes initial_event_signature = 11;
optional bool is_guaranteed = 12 [default = false];
- optional string jwt = 13;
- optional bool is_external_client_request = 14 [default = false];
+
+ // HIVE-28932: We used 13, 14, jwt, is_external_client_request to let
external clients directly access LLAP daemons
+ // in a cloud environment.
+ reserved 13, 14;
+ reserved "jwt", "is_external_client_request";
}
message RegisterDagRequestProto {
diff --git
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
index 7ab8032889d..5772e391d2b 100644
---
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
+++
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
@@ -50,7 +50,6 @@
import org.apache.hadoop.hive.llap.ext.LlapDaemonInfo;
import org.apache.hadoop.hive.llap.ext.LlapTaskUmbilicalExternalClient;
import
org.apache.hadoop.hive.llap.ext.LlapTaskUmbilicalExternalClient.LlapTaskUmbilicalExternalResponder;
-import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
import org.apache.hadoop.hive.llap.tez.Converters;
import org.apache.hadoop.io.BytesWritable;
@@ -416,8 +415,6 @@ private SubmitWorkRequestProto
constructSubmitWorkRequestProto(SubmitWorkInfo su
if (fragmentBytesSignature != null) {
builder.setInitialEventSignature(ByteString.copyFrom(fragmentBytesSignature));
}
- builder.setJwt(llapInputSplit.getJwt());
- builder.setIsExternalClientRequest(true);
return builder.build();
}
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java
index cc87e79dbf4..f4819c586f8 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java
@@ -68,9 +68,6 @@ public Void call() throws Exception {
io.netty.handler.codec.http.HttpObjectAggregator.class, // netty-all
com.google.flatbuffers.Table.class, //flatbuffers
com.carrotsearch.hppc.ByteArrayDeque.class, //hppc
- io.jsonwebtoken.security.Keys.class, //jjwt-api
- io.jsonwebtoken.impl.DefaultJws.class, //jjwt-impl
- io.jsonwebtoken.io.JacksonSerializer.class, //jjwt-jackson
};
for (Class<?> c : dependencies) {
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
index 2c7b729f698..0dadbf982b0 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/AMReporter.java
@@ -19,14 +19,11 @@
package org.apache.hadoop.hive.llap.daemon.impl;
-import org.apache.hadoop.hive.llap.LlapUtil;
import
org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol.BooleanArray;
import
org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol.TezAttemptArray;
import java.util.ArrayList;
import java.util.List;
-import java.util.HashSet;
-import java.util.Set;
import javax.net.SocketFactory;
@@ -198,7 +195,7 @@ public void serviceStop() {
}
}
- public AMNodeInfo registerTask(boolean externalClientRequest, String
amLocation, int port, String umbilicalUser,
+ public AMNodeInfo registerTask(String amLocation, int port, String
umbilicalUser,
Token<JobTokenIdentifier> jobToken,
QueryIdentifier queryIdentifier,
TezTaskAttemptID attemptId, boolean
isGuaranteed) {
if (LOG.isTraceEnabled()) {
@@ -220,7 +217,6 @@ public AMNodeInfo registerTask(boolean
externalClientRequest, String amLocation,
if (amNodeInfo == null) {
amNodeInfo = new AMNodeInfo(amNodeId, umbilicalUser, jobToken,
queryIdentifier, retryPolicy,
retryTimeout, socketFactory, conf);
- amNodeInfo.setIsExternalClientRequest(externalClientRequest);
amNodeInfoPerQuery.put(amNodeId, amNodeInfo);
// Add to the queue only the first time this is registered, and on
// subsequent instances when it's taken off the queue.
@@ -413,15 +409,8 @@ protected Void callInternal() {
BooleanArray guaranteed = new BooleanArray();
guaranteed.set(tasks.guaranteed.toArray(new
BooleanWritable[tasks.guaranteed.size()]));
- if (LlapUtil.isCloudDeployment(conf) &&
amNodeInfo.isExternalClientRequest()) {
- String hostname = amNodeInfo.amNodeId.getHostname();
- int externalClientCloudRpcPort = amNodeInfo.amNodeId.getPort();
- amNodeInfo.getUmbilical().nodeHeartbeat(new Text(hostname),
- new Text(daemonId.getUniqueNodeIdInCluster()),
externalClientCloudRpcPort, aw, guaranteed);
- } else {
- amNodeInfo.getUmbilical().nodeHeartbeat(new
Text(nodeId.getHostname()),
- new Text(daemonId.getUniqueNodeIdInCluster()),
nodeId.getPort(), aw, guaranteed);
- }
+ amNodeInfo.getUmbilical().nodeHeartbeat(new Text(nodeId.getHostname()),
+ new Text(daemonId.getUniqueNodeIdInCluster()), nodeId.getPort(),
aw, guaranteed);
} catch (IOException e) {
QueryIdentifier currentQueryIdentifier =
amNodeInfo.getQueryIdentifier();
amNodeInfo.setAmFailed(true);
@@ -480,7 +469,6 @@ protected class AMNodeInfo implements Delayed {
private LlapTaskUmbilicalProtocol umbilical;
private long nextHeartbeatTime;
private final AtomicBoolean isDone = new AtomicBoolean(false);
- private final AtomicBoolean isExternalClientRequest = new
AtomicBoolean(false);
public AMNodeInfo(LlapNodeId amNodeId, String umbilicalUser,
@@ -552,14 +540,6 @@ boolean isDone() {
return isDone.get();
}
- void setIsExternalClientRequest(boolean val) {
- isExternalClientRequest.set(val);
- }
-
- boolean isExternalClientRequest() {
- return isExternalClientRequest.get();
- }
-
/**
* @return A snapshot of the tasks running at this daemon from this AM.
* Doesn't have to be consistent between multiple tasks; whether some task
makes it into
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
index d9be6c0b097..fb25bc5c8c3 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
@@ -40,10 +40,6 @@
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
-import io.jsonwebtoken.Claims;
-import io.jsonwebtoken.Jws;
-import io.jsonwebtoken.JwtException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.llap.LlapUgiManager;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -83,7 +79,6 @@
import
org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SetCapacityResponseProto;
import
org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.VertexOrBinary;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics;
-import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper;
import org.apache.hadoop.hive.llap.security.LlapSignerImpl;
import org.apache.hadoop.hive.llap.tez.Converters;
import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
@@ -241,8 +236,6 @@ public SubmitWorkResponseProto
submitWork(SubmitWorkRequestProto request) throws
QueryIdentifierProto qIdProto = vertex.getQueryIdentifier();
- verifyJwtForExternalClient(request, qIdProto.getApplicationIdString(),
fragmentIdString);
-
LOG.info("Queueing container for execution: fragemendId={}, {}",
fragmentIdString,
stringifySubmitRequest(request, vertex));
@@ -351,39 +344,6 @@ public SubmitWorkResponseProto
submitWork(SubmitWorkRequestProto request) throws
.build();
}
- // if request is coming from llap external client, verify the JWT
- // as of now, JWT contains applicationId
- private void verifyJwtForExternalClient(SubmitWorkRequestProto request,
String extClientAppIdFromSplit,
- String fragmentIdString) {
- LOG.info("Checking if request[{}] is from llap external client in a cloud
based deployment",
- extClientAppIdFromSplit);
- if (request.getIsExternalClientRequest() &&
LlapUtil.isCloudDeployment(getConfig())) {
- LOG.info("Llap external client request - {}, verifying JWT",
extClientAppIdFromSplit);
- Preconditions.checkState(request.hasJwt(), "JWT not found in request,
fragmentId: " + fragmentIdString);
-
- LlapExtClientJwtHelper llapExtClientJwtHelper = new
LlapExtClientJwtHelper(getConfig());
- Jws<Claims> claimsJws;
- try {
- claimsJws = llapExtClientJwtHelper.parseClaims(request.getJwt());
- } catch (JwtException e) {
- LOG.error("Cannot verify JWT provided with the request, fragmentId:
{}, {}", fragmentIdString, e);
- throw e;
- }
-
- String extClientAppIdFromJwt = (String)
claimsJws.getBody().get(LlapExtClientJwtHelper.LLAP_EXT_CLIENT_APP_ID);
-
- // this should never happen ideally.
- // extClientAppId is injected in JWT and fragment request by initial
get_splits() call.
- // so both of these - extClientAppIdFromJwt and extClientAppIdFromSplit
should be equal eventually if the signed JWT is valid for this request.
- // In get_splits, this extClientAppId is obtained via
LlapCoordinator#createExtClientAppId which generates a
- // application Id to be used by external clients.
-
Preconditions.checkState(extClientAppIdFromJwt.equals(extClientAppIdFromSplit),
- String.format("applicationId[%s] in request does not match to
applicationId[%s] in JWT",
- extClientAppIdFromSplit, extClientAppIdFromJwt));
- LOG.info("Llap external client request - {}, JWT verification
successful", extClientAppIdFromSplit);
- }
- }
-
private SignableVertexSpec extractVertexSpec(SubmitWorkRequestProto request,
LlapTokenInfo tokenInfo) throws InvalidProtocolBufferException,
IOException {
VertexOrBinary vob = request.getWorkSpec();
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index 4d3a5d06e6c..71c77289318 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -76,7 +76,6 @@
import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem;
import org.apache.hadoop.hive.llap.metrics.MetricsUtils;
import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
-import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper;
import org.apache.hadoop.hive.llap.security.SecretManager;
import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler;
import org.apache.hadoop.hive.ql.ServiceContext;
@@ -141,7 +140,6 @@ public class LlapDaemon extends CompositeService implements
ContainerRunner, Lla
public LlapDaemon(Configuration daemonConf, int numExecutors, long
executorMemoryBytes,
boolean ioEnabled, boolean isDirectCache, long ioMemoryBytes, String[]
localDirs, int srvPort,
- boolean externalClientCloudSetupEnabled, int
externalClientsRpcPort,
int mngPort, int shufflePort, int webPort, String appName) {
super("LlapDaemon");
@@ -150,11 +148,6 @@ public LlapDaemon(Configuration daemonConf, int
numExecutors, long executorMemor
Preconditions.checkArgument(numExecutors > 0);
Preconditions.checkArgument(srvPort == 0 || (srvPort > 1024 && srvPort <
65536),
"Server RPC Port must be between 1025 and 65535, or 0 automatic
selection");
- if (externalClientCloudSetupEnabled) {
- Preconditions.checkArgument(
- externalClientsRpcPort == 0 || (externalClientsRpcPort > 1024 &&
externalClientsRpcPort < 65536),
- "Server RPC port for external clients must be between 1025 and
65535, or 0 automatic selection");
- }
Preconditions.checkArgument(mngPort == 0 || (mngPort > 1024 && mngPort <
65536),
"Management RPC Port must be between 1025 and 65535, or 0 automatic
selection");
@@ -241,8 +234,6 @@ public LlapDaemon(Configuration daemonConf, int
numExecutors, long executorMemor
", llapIoEnabled=" + ioEnabled +
", llapIoCacheIsDirect=" + isDirectCache +
", rpcListenerPort=" + srvPort +
- ", externalClientCloudSetupEnabled=" + externalClientCloudSetupEnabled +
- ", rpcListenerPortForExternalClients=" + externalClientsRpcPort +
", mngListenerPort=" + mngPort +
", webPort=" + webPort +
", outputFormatSvcPort=" + outputFormatServicePort +
@@ -340,7 +331,7 @@ public LlapDaemon(Configuration daemonConf, int
numExecutors, long executorMemor
this.secretManager = sm;
this.server = new LlapProtocolServerImpl(secretManager, numHandlers, this,
srvAddress, mngAddress, srvPort,
- externalClientsRpcPort, mngPort, daemonId,
metrics).withTokenManager(this.llapTokenManager);
+ mngPort, metrics).withTokenManager(this.llapTokenManager);
LlapUgiManager llapUgiManager = LlapUgiManager.getInstance(daemonConf);
@@ -502,15 +493,6 @@ public void serviceStart() throws Exception {
getConfig().setInt(ConfVars.LLAP_DAEMON_WEB_PORT.varname,
webServices.getPort());
}
getConfig().setInt(ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT.varname,
LlapOutputFormatService.get().getPort());
- if (LlapUtil.isCloudDeployment(getConfig())) {
-
- // this invokes JWT secret provider and tries to get shared secret.
- // meant to validate shared secret as well.
- new LlapExtClientJwtHelper(getConfig());
-
- getConfig().setInt(ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT.varname,
- server.getExternalClientsRpcServerBindAddress().getPort());
- }
// Ensure this is set in the config so that the AM can read it.
getConfig()
@@ -624,8 +606,6 @@ public static void main(String[] args) throws Exception {
String[] localDirs = (localDirList == null || localDirList.isEmpty()) ?
new String[0] : StringUtils.getTrimmedStrings(localDirList);
int rpcPort = HiveConf.getIntVar(daemonConf,
ConfVars.LLAP_DAEMON_RPC_PORT);
- int externalClientCloudRpcPort = HiveConf.getIntVar(daemonConf,
ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT);
- boolean externalClientCloudSetupEnabled =
LlapUtil.isCloudDeployment(daemonConf);
int mngPort = HiveConf.getIntVar(daemonConf,
ConfVars.LLAP_MANAGEMENT_RPC_PORT);
int shufflePort = HiveConf.getIntVar(daemonConf,
ConfVars.LLAP_DAEMON_YARN_SHUFFLE_PORT);
int webPort = HiveConf.getIntVar(daemonConf,
ConfVars.LLAP_DAEMON_WEB_PORT);
@@ -643,8 +623,7 @@ public static void main(String[] args) throws Exception {
LlapDaemon.initializeLogging(daemonConf);
llapDaemon =
new LlapDaemon(daemonConf, numExecutors, executorMemoryBytes,
isLlapIo, isDirectCache,
- ioMemoryBytes, localDirs, rpcPort,
externalClientCloudSetupEnabled,
- externalClientCloudRpcPort, mngPort, shufflePort, webPort,
appName);
+ ioMemoryBytes, localDirs, rpcPort, mngPort, shufflePort,
webPort, appName);
LOG.info("Adding shutdown hook for LlapDaemon");
ShutdownHookManager.addShutdownHook(new
CompositeServiceShutdownHook(llapDaemon), 1);
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
index b1e8db5287d..9b13eb269ca 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
@@ -42,7 +42,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.llap.DaemonId;
import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos;
import
org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.CacheEntryList;
@@ -83,8 +82,10 @@ private enum TokenRequiresSigning {
private final int numHandlers;
private final ContainerRunner containerRunner;
- private final int srvPort, mngPort, externalClientsRpcPort;
- private RPC.Server server, mngServer, externalClientsRpcServer;
+ private final int srvPort;
+ private final int mngPort;
+ private RPC.Server server;
+ private RPC.Server mngServer;
private final AtomicReference<InetSocketAddress> srvAddress, mngAddress;
private final SecretManager secretManager;
private String clusterUser = null;
@@ -93,11 +94,9 @@ private enum TokenRequiresSigning {
private TokenRequiresSigning isSigningRequiredConfig =
TokenRequiresSigning.TRUE;
private LlapTokenManager llapTokenManager = new DummyTokenManager();
- public LlapProtocolServerImpl(SecretManager secretManager, int numHandlers,
- ContainerRunner containerRunner, AtomicReference<InetSocketAddress>
srvAddress,
- AtomicReference<InetSocketAddress> mngAddress, int srvPort, int
externalClientsRpcPort,
- int mngPort, DaemonId daemonId,
- LlapDaemonExecutorMetrics executorMetrics) {
+ public LlapProtocolServerImpl(SecretManager secretManager, int numHandlers,
ContainerRunner containerRunner,
+ AtomicReference<InetSocketAddress> srvAddress,
AtomicReference<InetSocketAddress> mngAddress, int srvPort,
+ int mngPort, LlapDaemonExecutorMetrics executorMetrics) {
super("LlapDaemonProtocolServerImpl");
this.numHandlers = numHandlers;
this.containerRunner = containerRunner;
@@ -105,7 +104,6 @@ public LlapProtocolServerImpl(SecretManager secretManager,
int numHandlers,
this.srvAddress = srvAddress;
this.srvPort = srvPort;
this.mngAddress = mngAddress;
- this.externalClientsRpcPort = externalClientsRpcPort;
this.mngPort = mngPort;
this.executorMetrics = executorMetrics;
LOG.info("Creating: " + LlapProtocolServerImpl.class.getSimpleName() +
@@ -241,15 +239,6 @@ private void startProtocolServers(
server = LlapUtil.startProtocolServer(srvPort, numHandlers, srvAddress,
conf, daemonImpl,
LlapProtocolBlockingPB.class, secretManager, pp,
ConfVars.LLAP_SECURITY_ACL,
ConfVars.LLAP_SECURITY_ACL_DENY);
- // for cloud deployments, start a separate RPC server on the port
- // which we can open to accept requests from external clients.
- if (LlapUtil.isCloudDeployment(conf)) {
- externalClientsRpcServer =
LlapUtil.startProtocolServer(externalClientsRpcPort, numHandlers, null, conf,
daemonImpl,
- LlapProtocolBlockingPB.class, secretManager, pp,
ConfVars.LLAP_SECURITY_ACL,
- ConfVars.LLAP_SECURITY_ACL_DENY);
-
- LOG.info("Started externalClientsRpcServer for cloud based deployments :
{}, {}", externalClientsRpcServer.getListenerAddress(),
externalClientsRpcServer);
- }
mngServer = LlapUtil.startProtocolServer(mngPort, 2, mngAddress, conf,
managementImpl,
LlapManagementProtocolPB.class, secretManager, pp,
ConfVars.LLAP_MANAGEMENT_ACL,
ConfVars.LLAP_MANAGEMENT_ACL_DENY);
@@ -261,9 +250,6 @@ public void serviceStop() {
if (server != null) {
server.stop();
}
- if (externalClientsRpcServer != null) {
- externalClientsRpcServer.stop();
- }
if (mngServer != null) {
mngServer.stop();
}
@@ -279,11 +265,6 @@ InetSocketAddress getManagementBindAddress() {
return mngAddress.get();
}
- @InterfaceAudience.Private
- InetSocketAddress getExternalClientsRpcServerBindAddress() {
- return externalClientsRpcServer.getListenerAddress();
- }
-
@Override
public GetTokenResponseProto getDelegationToken(RpcController controller,
GetTokenRequestProto request) throws ServiceException {
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
index 9f6ff74ecc9..eddf2834ece 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
@@ -160,7 +160,7 @@ public TaskRunnerCallable(SubmitWorkRequestProto request,
QueryFragmentInfo frag
this.amReporter = amReporter;
// Register with the AMReporter when the callable is setup. Unregister
once it starts running.
if (amReporter != null && jobToken != null) {
- this.amNodeInfo =
amReporter.registerTask(request.getIsExternalClientRequest(),
request.getAmHost(), request.getAmPort(),
+ this.amNodeInfo = amReporter.registerTask(request.getAmHost(),
request.getAmPort(),
vertex.getTokenIdentifier(), jobToken,
fragmentInfo.getQueryInfo().getQueryIdentifier(),
attemptId, isGuaranteed);
} else {
diff --git
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java
index 0b6ddb25dae..0d1afc569b9 100644
---
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java
+++
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/LlapDaemonExtension.java
@@ -64,7 +64,7 @@ public void beforeEach(ExtensionContext context) throws
Exception {
HiveConf.setVar(conf, HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS, "llap");
LlapDaemonInfo.initialize(appName, conf);
daemon =
- new LlapDaemon(conf, 1, LlapDaemon.getTotalHeapSize(), false, false,
-1, new String[1], 0, false, 0, 0, 0, 0,
+ new LlapDaemon(conf, 1, LlapDaemon.getTotalHeapSize(), false, false,
-1, new String[1], 0, 0, 0, 0,
appName);
daemon.init(conf);
daemon.start();
diff --git
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
index f8cd79d6826..47cadddf5ce 100644
---
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
+++
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
@@ -24,7 +24,6 @@
import java.io.File;
import java.io.IOException;
-import org.apache.hadoop.hive.llap.LlapUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -37,8 +36,6 @@
import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.Service;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hive.testutils.MiniZooKeeperCluster;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
@@ -142,7 +139,6 @@ private MiniLlapCluster(String clusterName, @Nullable
MiniZooKeeperCluster miniZ
@Override
public void serviceInit(Configuration conf) throws IOException,
InterruptedException {
int rpcPort = 0;
- int externalClientCloudRpcPort = 0;
int mngPort = 0;
int shufflePort = 0;
int webPort = 0;
@@ -151,7 +147,6 @@ public void serviceInit(Configuration conf) throws
IOException, InterruptedExcep
LOG.info("MiniLlap configured to use ports from conf: {}",
usePortsFromConf);
if (usePortsFromConf) {
rpcPort = HiveConf.getIntVar(conf,
HiveConf.ConfVars.LLAP_DAEMON_RPC_PORT);
- externalClientCloudRpcPort = HiveConf.getIntVar(conf,
ConfVars.LLAP_EXTERNAL_CLIENT_CLOUD_RPC_PORT);
mngPort = HiveConf.getIntVar(conf,
HiveConf.ConfVars.LLAP_MANAGEMENT_RPC_PORT);
shufflePort = conf.getInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY,
ShuffleHandler.DEFAULT_SHUFFLE_PORT);
webPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_WEB_PORT);
@@ -174,12 +169,10 @@ public void serviceInit(Configuration conf) throws
IOException, InterruptedExcep
clusterSpecificConfiguration.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname,
"localhost");
clusterSpecificConfiguration.setInt(ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT.varname,
miniZooKeeperCluster.getClientPort());
- boolean externalClientCloudSetupEnabled = LlapUtil.isCloudDeployment(conf);
-
LOG.info("Initializing {} llap instances for MiniLlapCluster with
name={}", numInstances, clusterNameTrimmed);
for (int i = 0 ;i < numInstances ; i++) {
llapDaemons[i] = new LlapDaemon(conf, numExecutorsPerService,
execBytesPerService, llapIoEnabled,
- ioIsDirect, ioBytesPerService, localDirs, rpcPort,
externalClientCloudSetupEnabled, externalClientCloudRpcPort,
+ ioIsDirect, ioBytesPerService, localDirs, rpcPort,
mngPort, shufflePort, webPort, clusterNameTrimmed);
llapDaemons[i].init(new Configuration(conf));
}
diff --git
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java
index 44351b9d419..7b3b629ea35 100644
---
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java
+++
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java
@@ -106,7 +106,7 @@ public void testEnforceProperNumberOfIOThreads() throws
IOException {
HiveConf.setIntVar(hiveConf, HiveConf.ConfVars.LLAP_IO_THREADPOOL_SIZE, 3);
daemon = new LlapDaemon(hiveConf, 4, LlapDaemon.getTotalHeapSize(), true,
false,
- -1, new String[1], 0, false, 0,0, 0, defaultWebPort,
"TestLlapDaemon");
+ -1, new String[1], 0, 0, 0, defaultWebPort, "TestLlapDaemon");
}
@Test
@@ -120,7 +120,7 @@ public void testLocalDirCleaner() throws IOException,
InterruptedException {
createFile(localDirs[0] + "/file3");
daemon = new LlapDaemon(hiveConf, 1, LlapDaemon.getTotalHeapSize(), false,
false,
- -1, localDirs, 0, false, 0,0, 0, defaultWebPort, "TestLlapDaemon");
+ -1, localDirs, 0, 0, 0, defaultWebPort, "TestLlapDaemon");
daemon.init(hiveConf);
assertFileExists(localDirs[0] + "/hive/appcache/file1", true);
@@ -155,7 +155,7 @@ public void testUpdateRegistration() throws IOException {
int enabledQueue = 2;
daemon = new LlapDaemon(hiveConf, 1, LlapDaemon.getTotalHeapSize(), false,
false,
- -1, new String[1], 0, false, 0,0, 0, defaultWebPort, "TestLlapDaemon");
+ -1, new String[1], 0, 0, 0, defaultWebPort, "TestLlapDaemon");
trySetMock(daemon, LlapRegistryService.class, mockRegistry);
diff --git
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
index a3802ecda44..f0577a14e03 100644
---
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
+++
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemonProtocolServerImpl.java
@@ -60,7 +60,7 @@ public void testSimpleCall() throws ServiceException,
IOException {
LlapProtocolServerImpl server =
new LlapProtocolServerImpl(null, numHandlers, containerRunnerMock,
new AtomicReference<InetSocketAddress>(), new
AtomicReference<InetSocketAddress>(),
- 0, 0, 0, null, null);
+ 0, 0, null);
when(containerRunnerMock.submitWork(any(SubmitWorkRequestProto.class))).thenReturn(
SubmitWorkResponseProto
.newBuilder()
@@ -95,7 +95,7 @@ public void testGetDaemonMetrics() throws ServiceException,
IOException {
LlapProtocolServerImpl server =
new LlapProtocolServerImpl(null, numHandlers, null,
new AtomicReference<InetSocketAddress>(), new
AtomicReference<InetSocketAddress>(),
- 0, 0, 0, null, executorMetrics);
+ 0, 0, executorMetrics);
executorMetrics.addMetricsFallOffFailedTimeLost(10);
executorMetrics.addMetricsFallOffKilledTimeLost(11);
executorMetrics.addMetricsFallOffSuccessTimeLost(12);
@@ -166,7 +166,7 @@ public void testSetCapacity() throws ServiceException,
IOException {
LlapProtocolServerImpl server =
new LlapProtocolServerImpl(null, numHandlers, containerRunnerMock,
new AtomicReference<InetSocketAddress>(), new
AtomicReference<InetSocketAddress>(),
- 0, 0, 0, null, executorMetrics);
+ 0, 0, executorMetrics);
try {
server.init(new Configuration());
diff --git
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java
index 5070dfc62a1..56b3e8b6939 100644
---
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java
+++
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/comparator/TestAMReporter.java
@@ -72,9 +72,9 @@ public void testMultipleAM() throws InterruptedException {
String am2Location = "am2";
String umbilicalUser = "user";
QueryIdentifier queryId = new QueryIdentifier("app", 0);
- amReporter.registerTask(false,am1Location, am1Port, umbilicalUser, null,
queryId,
+ amReporter.registerTask(am1Location, am1Port, umbilicalUser, null, queryId,
mock(TezTaskAttemptID.class), false);
- amReporter.registerTask(false,am2Location, am2Port, umbilicalUser, null,
queryId,
+ amReporter.registerTask(am2Location, am2Port, umbilicalUser, null, queryId,
mock(TezTaskAttemptID.class), false);
Thread.currentThread().sleep(2000);
diff --git
a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java
index fa0e59b04a2..f8a0b369f7d 100644
---
a/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java
+++
b/llap-server/src/test/org/apache/hadoop/hive/llap/io/encoded/TestLlapOrcCacheLoader.java
@@ -18,7 +18,6 @@
*/
package org.apache.hadoop.hive.llap.io.encoded;
-import io.jsonwebtoken.lang.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@@ -35,6 +34,7 @@
import org.apache.hadoop.hive.ql.io.SyntheticFileId;
import org.apache.hadoop.hive.ql.io.orc.encoded.IoTrace;
import org.apache.hive.common.util.FixedSizedObjectPool;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -89,7 +89,7 @@ public void testLoadFooter() throws IOException {
loader.loadFileFooter();
}
MetadataCache.LlapBufferOrBuffers metadata =
metaCache.getFileMetadata(key);
- Assert.notNull(metadata);
+ Assert.assertNotNull(metadata);
}
@@ -108,7 +108,7 @@ public void testLoadUncompressedRanges() throws IOException
{
cache.getFileData(key, range, 0,
mockDiskRangeListFactory, null, gotAllData);
- Assert.isTrue(gotAllData.value);
+ Assert.assertTrue(gotAllData.value);
}
}
@@ -126,7 +126,7 @@ public void testLoadValidRanges() throws IOException {
cache.getFileData(key, range, 0,
mockDiskRangeListFactory, null, gotAllData);
- Assert.isTrue(gotAllData.value);
+ Assert.assertTrue(gotAllData.value);
}
@Test
@@ -143,7 +143,7 @@ public void testLoadAlreadyLoadedRange() throws IOException
{
DataCache.BooleanRef gotAllData = new DataCache.BooleanRef();
cache.getFileData(key, range, 0,
mockDiskRangeListFactory, null, gotAllData);
- Assert.isTrue(gotAllData.value);
+ Assert.assertTrue(gotAllData.value);
DiskRangeList range2 = new DiskRangeList(ORC_PADDING,14);
try(LlapOrcCacheLoader loader = new LlapOrcCacheLoader(path, key, conf,
mockDataCache, metaCache,
@@ -154,7 +154,7 @@ public void testLoadAlreadyLoadedRange() throws IOException
{
gotAllData.value = false;
cache.getFileData(key, range, 0,
mockDiskRangeListFactory, null, gotAllData);
- Assert.isTrue(gotAllData.value);
+ Assert.assertTrue(gotAllData.value);
}
@Test
@@ -171,7 +171,7 @@ public void testLoadBadlyEstimatedRanges() throws
IOException {
cache.getFileData(key, range, 0,
mockDiskRangeListFactory, null, gotAllData);
- Assert.isTrue(!gotAllData.value);
+ Assert.assertFalse(gotAllData.value);
}
diff --git a/pom.xml b/pom.xml
index a287aecc376..dc70c8b6da8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -232,7 +232,6 @@
<curator.version>5.7.1</curator.version>
<jsr305.version>3.0.0</jsr305.version>
<gson.version>2.9.0</gson.version>
- <jjwt.version>0.10.5</jjwt.version>
<re2j.version>1.2</re2j.version>
<rs-api.version>2.0.1</rs-api.version>
<json-path.version>2.9.0</json-path.version>
@@ -462,21 +461,6 @@
<artifactId>truffle-runtime</artifactId>
<version>${graalvm.version}</version>
</dependency>
- <dependency>
- <groupId>io.jsonwebtoken</groupId>
- <artifactId>jjwt-api</artifactId>
- <version>${jjwt.version}</version>
- </dependency>
- <dependency>
- <groupId>io.jsonwebtoken</groupId>
- <artifactId>jjwt-impl</artifactId>
- <version>${jjwt.version}</version>
- </dependency>
- <dependency>
- <groupId>io.jsonwebtoken</groupId>
- <artifactId>jjwt-jackson</artifactId>
- <version>${jjwt.version}</version>
- </dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
index d135dc2f9fc..df856d80484 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
@@ -47,7 +47,6 @@
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.FieldDesc;
import org.apache.hadoop.hive.llap.LlapInputSplit;
-import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.NotTezEventHelper;
import org.apache.hadoop.hive.llap.Schema;
import org.apache.hadoop.hive.llap.SubmitWorkInfo;
@@ -58,7 +57,6 @@
import org.apache.hadoop.hive.llap.ext.LlapDaemonInfo;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet;
-import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper;
import org.apache.hadoop.hive.llap.security.LlapSigner;
import org.apache.hadoop.hive.llap.security.LlapSigner.Signable;
import org.apache.hadoop.hive.llap.security.LlapSigner.SignedMessage;
@@ -422,7 +420,7 @@ private SplitResult getSplits(JobConf job, TezWork work,
Schema schema, Applicat
SplitResult splitResult = new SplitResult();
splitResult.schemaSplit = new LlapInputSplit(
0, new byte[0], new byte[0], new byte[0],
- new SplitLocationInfo[0], new LlapDaemonInfo[0], schema, "", new
byte[0], "");
+ new SplitLocationInfo[0], new LlapDaemonInfo[0], schema, "", new
byte[0]);
if (schemaSplitOnly) {
// schema only
return splitResult;
@@ -542,7 +540,7 @@ private SplitResult getSplits(JobConf job, TezWork work,
Schema schema, Applicat
if (generateLightWeightSplits) {
splitResult.planSplit = new LlapInputSplit(
0, submitWorkBytes, new byte[0], new byte[0],
- new SplitLocationInfo[0], new LlapDaemonInfo[0], new Schema(),
"", new byte[0], "");
+ new SplitLocationInfo[0], new LlapDaemonInfo[0], new Schema(),
"", new byte[0]);
}
}
@@ -555,22 +553,12 @@ private SplitResult getSplits(JobConf job, TezWork work,
Schema schema, Applicat
// 5. populate info about llap daemons(to help client submit request
and read data)
LlapDaemonInfo[] llapDaemonInfos = populateLlapDaemonInfos(job,
locations);
- // 6. Generate JWT for external clients if it's a cloud deployment
- // we inject extClientAppId in JWT which is same as what fragment
contains.
- // extClientAppId in JWT and in fragment are compared on LLAP when a
fragment is submitted.
- // see method ContainerRunnerImpl#verifyJwtForExternalClient
- String jwt = "";
- if (LlapUtil.isCloudDeployment(job)) {
- LlapExtClientJwtHelper llapExtClientJwtHelper = new
LlapExtClientJwtHelper(job);
- jwt = llapExtClientJwtHelper.buildJwtForLlap(extClientAppId);
- }
-
if (generateLightWeightSplits) {
result[i] = new LlapInputSplit(i, emptySubmitWorkBytes,
eventBytes.message,
- eventBytes.signature, locations, llapDaemonInfos, emptySchema,
llapUser, tokenBytes, jwt);
+ eventBytes.signature, locations, llapDaemonInfos, emptySchema,
llapUser, tokenBytes);
} else {
result[i] = new LlapInputSplit(i, submitWorkBytes,
eventBytes.message,
- eventBytes.signature, locations, llapDaemonInfos, schema,
llapUser, tokenBytes, jwt);
+ eventBytes.signature, locations, llapDaemonInfos, schema,
llapUser, tokenBytes);
}
}
splitResult.actualSplits = result;
@@ -668,12 +656,7 @@ private LlapDaemonInfo[] populateLlapDaemonInfos(JobConf
job, SplitLocationInfo[
LlapDaemonInfo[] llapDaemonInfos = new
LlapDaemonInfo[llapServiceInstances.size()];
int count = 0;
for (LlapServiceInstance inst : llapServiceInstances) {
- LlapDaemonInfo info;
- if (LlapUtil.isCloudDeployment(job)) {
- info = new LlapDaemonInfo(inst.getExternalHostname(),
inst.getExternalClientsRpcPort(), inst.getOutputFormatPort());
- } else {
- info = new LlapDaemonInfo(inst.getHost(), inst.getRpcPort(),
inst.getOutputFormatPort());
- }
+ LlapDaemonInfo info = new LlapDaemonInfo(inst.getHost(),
inst.getRpcPort(), inst.getOutputFormatPort());
llapDaemonInfos[count++] = info;
}
return llapDaemonInfos;