[hadoop] branch trunk updated: HADOOP-18870. CURATOR-599 change broke functionality introduced in HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9342ecf6ccd HADOOP-18870. CURATOR-599 change broke functionality
introduced in HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi
9342ecf6ccd is described below
commit 9342ecf6ccd5c7ef443a0eb722852d2addc1d5db
Author: Szilard Nemeth
AuthorDate: Wed Sep 6 21:32:36 2023 -0400
HADOOP-18870. CURATOR-599 change broke functionality introduced in
HADOOP-18139 and HADOOP-18709. Contributed by Ferenc Erdelyi
---
.../hadoop/util/curator/ZKCuratorManager.java | 12 +++-
.../hadoop/util/curator/TestZKCuratorManager.java | 35 ++
2 files changed, 46 insertions(+), 1 deletion(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
index 81ee4663e1c..4df79774329 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
@@ -549,7 +549,17 @@ public final class ZKCuratorManager {
public ZooKeeper newZooKeeper(String connectString, int sessionTimeout,
Watcher watcher, boolean canBeReadOnly
) throws Exception {
- ZKClientConfig zkClientConfig = new ZKClientConfig();
+ return this.newZooKeeper(connectString, sessionTimeout,
+ watcher, canBeReadOnly, new ZKClientConfig());
+}
+
+@Override
+public ZooKeeper newZooKeeper(String connectString, int sessionTimeout,
+Watcher watcher, boolean canBeReadOnly, ZKClientConfig zkClientConfig
+) throws Exception {
+ if (zkClientConfig == null) {
+zkClientConfig = new ZKClientConfig();
+ }
if (zkPrincipal != null) {
LOG.info("Configuring zookeeper to use {} as the server principal",
zkPrincipal);
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
index fd15a0c2b1b..4365e43e491 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
@@ -22,10 +22,15 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.security.auth.login.AppConfigurationEntry;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
import org.apache.curator.test.TestingServer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -193,6 +198,36 @@ public class TestZKCuratorManager {
}
}
+ @Test
+ public void testCuratorFrameworkFactory() throws Exception{
+// By not explicitly calling the NewZooKeeper method validate that the
Curator override works.
+ZKClientConfig zkClientConfig = new ZKClientConfig();
+Configuration conf = new Configuration();
+conf.set(CommonConfigurationKeys.ZK_ADDRESS,
this.server.getConnectString());
+int numRetries = conf.getInt(CommonConfigurationKeys.ZK_NUM_RETRIES,
+CommonConfigurationKeys.ZK_NUM_RETRIES_DEFAULT);
+int zkSessionTimeout = conf.getInt(CommonConfigurationKeys.ZK_TIMEOUT_MS,
+CommonConfigurationKeys.ZK_TIMEOUT_MS_DEFAULT);
+int zkRetryInterval = conf.getInt(
+CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS,
+CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS_DEFAULT);
+RetryNTimes retryPolicy = new RetryNTimes(numRetries, zkRetryInterval);
+
+CuratorFramework client = CuratorFrameworkFactory.builder()
+.connectString(conf.get(CommonConfigurationKeys.ZK_ADDRESS))
+.zkClientConfig(zkClientConfig)
+.sessionTimeoutMs(zkSessionTimeout).retryPolicy(retryPolicy)
+.authorization(new ArrayList<>())
+.zookeeperFactory(new ZKCuratorManager.HadoopZookeeperFactory(
+"foo1", "bar1", "bar1.keytab", false,
+new ZKCuratorManager.TruststoreKeystore(conf))
+
+).build();
+client.start();
+
validateJaasConfiguration(ZKCuratorManager.HadoopZookeeperFactory.JAAS_CLIENT_ENTRY,
+"bar1", "bar1.keytab", client.getZookeeperClient().getZooKeeper());
+ }
+
private void validateJaasConfiguration(String clientConfig, String
principal,
[hadoop] branch trunk updated (e1dde3bc231 -> 07c8b69f7cc)
This is an automated email from the ASF dual-hosted git repository. inigoiri pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git from e1dde3bc231 YARN-11537. [Federation] Router CLI Supports List SubClusterPolicyConfiguration Of Queues. (#5944) Contributed by Shilun Fan. add 07c8b69f7cc HDFS-17167. Add config to startup NameNode as Observer (#6013) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../hadoop/hdfs/server/namenode/BackupNode.java| 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 14 +++-- .../src/main/resources/hdfs-default.xml| 9 .../hdfs/server/namenode/ha/TestObserverNode.java | 60 ++ 5 files changed, 83 insertions(+), 4 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11537. [Federation] Router CLI Supports List SubClusterPolicyConfiguration Of Queues. (#5944) Contributed by Shilun Fan.
This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e1dde3bc231 YARN-11537. [Federation] Router CLI Supports List
SubClusterPolicyConfiguration Of Queues. (#5944) Contributed by Shilun Fan.
e1dde3bc231 is described below
commit e1dde3bc23120160c407b4c65d5b61c8f3a593a5
Author: slfan1989 <55643692+slfan1...@users.noreply.github.com>
AuthorDate: Wed Sep 6 22:08:59 2023 +0800
YARN-11537. [Federation] Router CLI Supports List
SubClusterPolicyConfiguration Of Queues. (#5944) Contributed by Shilun Fan.
Reviewed-by: Inigo Goiri
Signed-off-by: Shilun Fan
---
.../api/ResourceManagerAdministrationProtocol.java | 17 +-
.../QueryFederationQueuePoliciesRequest.java | 132 +
.../QueryFederationQueuePoliciesResponse.java | 149 +++
.../resourcemanager_administration_protocol.proto | 1 +
...arn_server_resourcemanager_service_protos.proto | 15 ++
.../apache/hadoop/yarn/client/cli/RouterCLI.java | 103 +-
.../hadoop/yarn/client/cli/TestRouterCLI.java | 26 +++
...eManagerAdministrationProtocolPBClientImpl.java | 19 ++
...ManagerAdministrationProtocolPBServiceImpl.java | 23 +++
.../QueryFederationQueuePoliciesRequestPBImpl.java | 196 +++
...QueryFederationQueuePoliciesResponsePBImpl.java | 212 +
.../yarn/server/MockResourceManagerFacade.java | 8 +
.../yarn/server/resourcemanager/AdminService.java | 10 +
.../hadoop/yarn/server/router/RouterMetrics.java | 32
.../rmadmin/DefaultRMAdminRequestInterceptor.java | 8 +
.../rmadmin/FederationRMAdminInterceptor.java | 211
.../router/rmadmin/RouterRMAdminService.java | 9 +
.../yarn/server/router/TestRouterMetrics.java | 34
.../PassThroughRMAdminRequestInterceptor.java | 8 +
.../rmadmin/TestFederationRMAdminInterceptor.java | 126 +++-
20 files changed, 1333 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
index 1ad77e0b30e..916dcd50aa0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
@@ -62,6 +62,8 @@ import
org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePoli
import
org.apache.hadoop.yarn.server.api.protocolrecords.SaveFederationQueuePolicyResponse;
import
org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesRequest;
import
org.apache.hadoop.yarn.server.api.protocolrecords.BatchSaveFederationQueuePoliciesResponse;
+import
org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesRequest;
+import
org.apache.hadoop.yarn.server.api.protocolrecords.QueryFederationQueuePoliciesResponse;
@Private
public interface ResourceManagerAdministrationProtocol extends
GetUserMappingsProtocol {
@@ -194,7 +196,7 @@ public interface ResourceManagerAdministrationProtocol
extends GetUserMappingsPr
/**
* In YARN-Federation mode, this method provides a way to save queue
policies in batches.
*
- * @param request BatchSaveFederationQueuePolicies Request
+ * @param request BatchSaveFederationQueuePolicies Request.
* @return Response from batchSaveFederationQueuePolicies.
* @throws YarnException exceptions from yarn servers.
* @throws IOException if an IO error occurred.
@@ -203,4 +205,17 @@ public interface ResourceManagerAdministrationProtocol
extends GetUserMappingsPr
@Idempotent
BatchSaveFederationQueuePoliciesResponse batchSaveFederationQueuePolicies(
BatchSaveFederationQueuePoliciesRequest request) throws YarnException,
IOException;
+
+ /**
+ * In YARN-Federation mode, this method provides a way to list policies.
+ *
+ * @param request QueryFederationQueuePoliciesRequest Request.
+ * @return Response from listFederationQueuePolicies.
+ * @throws YarnException exceptions from yarn servers.
+ * @throws IOException if an IO error occurred.
+ */
+ @Private
+ @Idempotent
+ QueryFederationQueuePoliciesResponse listFederationQueuePolicies(
+ QueryFederationQueuePoliciesRequest request) throws YarnException,
IOException;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/QueryFederationQueuePoliciesRequest.java
[hadoop] branch trunk updated (7c941e00b45 -> c2c6972f251)
This is an automated email from the ASF dual-hosted git repository. hexiaoqiao pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git from 7c941e00b45 HDFS-16933. A race in SerialNumberMap will cause wrong owner, group and XATTR. (#5430). Contributed by ZanderXu. add c2c6972f251 HDFS-17140. Revisit the BPOfferService.reportBadBlocks() method. (#5924). Contributed by Liangjun He. No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java| 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
