[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5196: YARN-11391 Add yarn RM DNS support
slfan1989 commented on code in PR #5196:
URL: https://github.com/apache/hadoop/pull/5196#discussion_r1042961649
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java:
##
@@ -262,6 +301,180 @@ public static String getRMHAId(Configuration conf) {
return currentRMId;
}
+ /**
+ * This function resolves all RMIds with their address. For multi-A DNS
records,
+ * it will resolve all of them, and generate a new Id for each of them.
+ *
+ * @param conf Configuration
+ * @return Map key as RMId, value as its address
+ */
+ public static Map getResolvedRMIdPairs(
+ Configuration conf) {
+boolean resolveNeeded = conf.getBoolean(
+YarnConfiguration.RESOLVE_RM_ADDRESS_NEEDED_KEY,
+YarnConfiguration.RESOLVE_RM_ADDRESS_NEEDED_DEFAULT);
+boolean requireFQDN = conf.getBoolean(
+YarnConfiguration.RESOLVE_RM_ADDRESS_TO_FQDN,
+YarnConfiguration.RESOLVE_RM_ADDRESS_TO_FQDN_DEFAULT);
+// In case client using DIFFERENT addresses for each service address
+// need to categorize them first
+Map, List> addressesConfigKeysMap = new HashMap<>();
+Collection rmIds = getOriginalRMHAIds(conf);
+for (String configKey : RM_ADDRESS_CONFIG_KEYS) {
+ List addresses = new ArrayList<>();
+ for (String rmId : rmIds) {
+String keyToRead = addSuffix(configKey, rmId);
+InetSocketAddress address = getInetSocketAddressFromString(
+conf.get(keyToRead));
+if (address != null) {
+ addresses.add(address.getHostName());
+}
+ }
+ Collections.sort(addresses);
+ List configKeysOfTheseAddresses =
addressesConfigKeysMap.get(addresses);
+ if (configKeysOfTheseAddresses == null) {
+configKeysOfTheseAddresses = new ArrayList<>();
+addressesConfigKeysMap.put(addresses, configKeysOfTheseAddresses);
+ }
+ configKeysOfTheseAddresses.add(configKey);
+}
+// We need to resolve and override by group (categorized by their input
host)
+// But since the function is called from "getRMHAId",
+// this function would only return value which is corresponded to
YarnConfiguration.RM_ADDRESS
+Map ret = null;
+for (List configKeys : addressesConfigKeysMap.values()) {
+ Map res = getResolvedIdPairs(
+ conf, resolveNeeded, requireFQDN, getOriginalRMHAIds(conf),
+ configKeys.get(0), YarnConfiguration.RM_HA_IDS, configKeys);
+ if (configKeys.contains(YarnConfiguration.RM_ADDRESS)) {
+ret = res;
+ }
+}
+return ret;
+ }
+
+ private static Map getResolvedIdPairs(
+ Configuration conf, boolean resolveNeeded, boolean requireFQDN,
Collection ids,
+ String configKey, String configKeyToReplace, List
listOfConfigKeysToReplace) {
+Map idAddressPairs = new HashMap<>();
+Map generatedIdToOriginalId = new HashMap<>();
+for (String id : ids) {
+ String key = addSuffix(configKey, id);
+ String addr = conf.get(key); // string with port
+ InetSocketAddress address = getInetSocketAddressFromString(addr);
+ if (address == null) {
+continue;
+ }
+ if (resolveNeeded) {
+if (dnr == null) {
+ setDnrByConfiguration(conf);
+}
+// If the address needs to be resolved, get all of the IP addresses
+// from this address and pass them into the map
+LOG.info("Multi-A domain name " + addr +
+" will be resolved by " + dnr.getClass().getName());
+int port = address.getPort();
+String[] resolvedHostNames;
+try {
+ resolvedHostNames = dnr.getAllResolvedHostnameByDomainName(
+ address.getHostName(), requireFQDN);
+} catch (UnknownHostException e) {
+ LOG.warn("Exception in resolving socket address "
+ + address.getHostName(), e);
+ continue;
+}
+LOG.info("Resolved addresses for " + addr +
+" is " + Arrays.toString(resolvedHostNames));
Review Comment:
record logs, generally using placeholders in this way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
-
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5196: YARN-11391 Add yarn RM DNS support
slfan1989 commented on code in PR #5196:
URL: https://github.com/apache/hadoop/pull/5196#discussion_r1042961032
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java:
##
@@ -262,6 +301,180 @@ public static String getRMHAId(Configuration conf) {
return currentRMId;
}
+ /**
+ * This function resolves all RMIds with their address. For multi-A DNS
records,
+ * it will resolve all of them, and generate a new Id for each of them.
+ *
+ * @param conf Configuration
+ * @return Map key as RMId, value as its address
+ */
+ public static Map getResolvedRMIdPairs(
+ Configuration conf) {
+boolean resolveNeeded = conf.getBoolean(
+YarnConfiguration.RESOLVE_RM_ADDRESS_NEEDED_KEY,
+YarnConfiguration.RESOLVE_RM_ADDRESS_NEEDED_DEFAULT);
+boolean requireFQDN = conf.getBoolean(
+YarnConfiguration.RESOLVE_RM_ADDRESS_TO_FQDN,
+YarnConfiguration.RESOLVE_RM_ADDRESS_TO_FQDN_DEFAULT);
+// In case client using DIFFERENT addresses for each service address
+// need to categorize them first
+Map, List> addressesConfigKeysMap = new HashMap<>();
+Collection rmIds = getOriginalRMHAIds(conf);
+for (String configKey : RM_ADDRESS_CONFIG_KEYS) {
+ List addresses = new ArrayList<>();
+ for (String rmId : rmIds) {
+String keyToRead = addSuffix(configKey, rmId);
+InetSocketAddress address = getInetSocketAddressFromString(
+conf.get(keyToRead));
+if (address != null) {
+ addresses.add(address.getHostName());
+}
+ }
+ Collections.sort(addresses);
+ List configKeysOfTheseAddresses =
addressesConfigKeysMap.get(addresses);
+ if (configKeysOfTheseAddresses == null) {
+configKeysOfTheseAddresses = new ArrayList<>();
+addressesConfigKeysMap.put(addresses, configKeysOfTheseAddresses);
+ }
+ configKeysOfTheseAddresses.add(configKey);
+}
+// We need to resolve and override by group (categorized by their input
host)
+// But since the function is called from "getRMHAId",
+// this function would only return value which is corresponded to
YarnConfiguration.RM_ADDRESS
+Map ret = null;
+for (List configKeys : addressesConfigKeysMap.values()) {
+ Map res = getResolvedIdPairs(
+ conf, resolveNeeded, requireFQDN, getOriginalRMHAIds(conf),
+ configKeys.get(0), YarnConfiguration.RM_HA_IDS, configKeys);
+ if (configKeys.contains(YarnConfiguration.RM_ADDRESS)) {
+ret = res;
+ }
+}
+return ret;
+ }
+
+ private static Map getResolvedIdPairs(
+ Configuration conf, boolean resolveNeeded, boolean requireFQDN,
Collection ids,
+ String configKey, String configKeyToReplace, List
listOfConfigKeysToReplace) {
+Map idAddressPairs = new HashMap<>();
+Map generatedIdToOriginalId = new HashMap<>();
+for (String id : ids) {
+ String key = addSuffix(configKey, id);
+ String addr = conf.get(key); // string with port
+ InetSocketAddress address = getInetSocketAddressFromString(addr);
+ if (address == null) {
+continue;
+ }
+ if (resolveNeeded) {
+if (dnr == null) {
+ setDnrByConfiguration(conf);
+}
+// If the address needs to be resolved, get all of the IP addresses
+// from this address and pass them into the map
+LOG.info("Multi-A domain name " + addr +
+" will be resolved by " + dnr.getClass().getName());
+int port = address.getPort();
+String[] resolvedHostNames;
+try {
+ resolvedHostNames = dnr.getAllResolvedHostnameByDomainName(
+ address.getHostName(), requireFQDN);
+} catch (UnknownHostException e) {
+ LOG.warn("Exception in resolving socket address "
+ + address.getHostName(), e);
+ continue;
+}
+LOG.info("Resolved addresses for " + addr +
+" is " + Arrays.toString(resolvedHostNames));
Review Comment:
{}
##
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java:
##
@@ -262,6 +301,180 @@ public static String getRMHAId(Configuration conf) {
return currentRMId;
}
+ /**
+ * This function resolves all RMIds with their address. For multi-A DNS
records,
+ * it will resolve all of them, and generate a new Id for each of them.
+ *
+ * @param conf Configuration
+ * @return Map key as RMId, value as its address
+ */
+ public static Map getResolvedRMIdPairs(
+ Configuration conf) {
+boolean resolveNeeded = conf.getBoolean(
+YarnConfiguration.RESOLVE_RM_ADDRESS_NEEDED_KEY,
+YarnConfiguration.RESOLVE_RM_ADDRESS_NEEDED_DEFAULT);
+boolean requireFQDN = conf.getBoolean(
+YarnConfiguration.RESOLVE_RM_ADDRESS_TO_FQDN,
+YarnConfiguration.RESOLVE_RM_ADDRESS_TO_FQDN_DEFAULT);
+// In
