[
https://issues.apache.org/jira/browse/YARN-11823?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17956417#comment-17956417
]
ASF GitHub Bot commented on YARN-11823:
---------------------------------------
K0K0V0K commented on code in PR #7726:
URL: https://github.com/apache/hadoop/pull/7726#discussion_r2129503829
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java:
##########
@@ -0,0 +1,134 @@
+/** * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class DiagnosticJStackService {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(DiagnosticJStackService.class);
+ private static final String PYTHON_COMMAND = "python3";
+ private static String scriptLocation = null;
+
+ static {
+ try {
+ // Extract script from JAR to a temp file
+ InputStream in = DiagnosticJStackService.class.getClassLoader()
+ .getResourceAsStream("diagnostics/jstack_collector.py");
+ File tempScript = File.createTempFile("jstack_collector", ".py");
+ Files.copy(in, tempScript.toPath(),
StandardCopyOption.REPLACE_EXISTING);
+ tempScript.setExecutable(true); // Set execute permission
+ scriptLocation = tempScript.getAbsolutePath();
+ } catch (IOException e) {
+ LOG.error("Failed to extract Python script from JAR", e);
+ }
+ }
+
+ public static String collectNodeJStack()
Review Comment:
First i read NodeJS, can we use other name here?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java:
##########
@@ -0,0 +1,134 @@
+/** * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class DiagnosticJStackService {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(DiagnosticJStackService.class);
+ private static final String PYTHON_COMMAND = "python3";
+ private static String scriptLocation = null;
+
+ static {
Review Comment:
This static block will block the NM to start up, till it is not done
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/diagnostics/jstack_collector.py:
##########
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import sys
+
+NUMBER_OF_JSTACK = 3
+
+def get_nodemanager_pid():
Review Comment:
I beleive from security perspective, these should not be available in REST
API in case of not secure cluster, and we should do authorisation in secured
clusters.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java:
##########
@@ -271,6 +273,35 @@ public ContainerInfo
getNodeContainer(@javax.ws.rs.core.Context
}
+ @GET
+ @Path("/jstack")
Review Comment:
I think this can be a bit misleading name cause we already have a /stacks
API for jstack
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DiagnosticJStackService.java:
##########
@@ -0,0 +1,134 @@
+/** * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class DiagnosticJStackService {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(DiagnosticJStackService.class);
+ private static final String PYTHON_COMMAND = "python3";
+ private static String scriptLocation = null;
+
+ static {
+ try {
+ // Extract script from JAR to a temp file
+ InputStream in = DiagnosticJStackService.class.getClassLoader()
+ .getResourceAsStream("diagnostics/jstack_collector.py");
+ File tempScript = File.createTempFile("jstack_collector", ".py");
+ Files.copy(in, tempScript.toPath(),
StandardCopyOption.REPLACE_EXISTING);
+ tempScript.setExecutable(true); // Set execute permission
+ scriptLocation = tempScript.getAbsolutePath();
+ } catch (IOException e) {
+ LOG.error("Failed to extract Python script from JAR", e);
+ }
+ }
+
+ public static String collectNodeJStack()
+ throws Exception {
+ if (Shell.WINDOWS) {
+ throw new UnsupportedOperationException("Not implemented for
Windows");
+ }
+
+ ProcessBuilder pb = createProcessBuilder();
+
+ return executeCommand(pb);
+
+ }
+
+
+
+ public static String collectAppJStack(String appId)
+ throws Exception {
+ if (Shell.WINDOWS) {
+ throw new UnsupportedOperationException("Not implemented for
Windows.");
+ }
+ ProcessBuilder pb = createProcessBuilder(appId);
+
+ LOG.info("Diagnostic process environment: {}", pb.environment());
+
+ return executeCommand(pb);
+ }
+
+ protected static ProcessBuilder createProcessBuilder() {
+ List<String> commandList =
+ new ArrayList<>(Arrays.asList(PYTHON_COMMAND, scriptLocation));
Review Comment:
Why we need ArrayList?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java:
##########
@@ -271,6 +273,35 @@ public ContainerInfo
getNodeContainer(@javax.ws.rs.core.Context
}
+ @GET
+ @Path("/jstack")
+ @Produces({MediaType.TEXT_PLAIN})
+ public Response getNodeJStack() {
+ try {
+ return Response.status(Status.OK)
+ .entity(DiagnosticJStackService.collectNodeJStack()) // Make
sure the NodeManager have python3 install
Review Comment:
What will happen if py3 is not present?
> New JStack endpoint for running containers and nodes
> ----------------------------------------------------
>
> Key: YARN-11823
> URL: https://issues.apache.org/jira/browse/YARN-11823
> Project: Hadoop YARN
> Issue Type: New Feature
> Components: yarn
> Reporter: chhinlinghean
> Assignee: chhinlinghean
> Priority: Major
> Labels: pull-request-available
>
> Getting jstack for container is not trivial to most people. This new endpoint
> aims to streamline the process of getting jstack of running containers and
> nodes. It mainly use in the DiagnosticCollector script.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
