This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 13c5c1f [SPARK-27180][BUILD][YARN] Fix testing issues with yarn
module in Hadoop-3
13c5c1f is described below
commit 13c5c1fb4b88e15c6cd09e6e275a9fc03f3085bd
Author: Yuming Wang <yumw...@ebay.com>
AuthorDate: Tue Apr 2 15:38:26 2019 -0500
[SPARK-27180][BUILD][YARN] Fix testing issues with yarn module in Hadoop-3
## What changes were proposed in this pull request?
Fix testing issues with `yarn` module in Hadoop-3:
1. Upgrade jersey-1 to `1.19` to fix ```Cause:
java.lang.NoClassDefFoundError:
com/sun/jersey/spi/container/servlet/ServletContainer```.
2. Copy `ServerSocketUtil` from
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
to fix ```java.lang.NoClassDefFoundError:
org/apache/hadoop/net/ServerSocketUtil```.
3. Adapte `SessionHandler` from
jetty-9.3.25.v20180904/jetty-server/src/main/java/org/eclipse/jetty/server/session/SessionHandler.java
to fix ```java.lang.NoSuchMethodError:
org.eclipse.jetty.server.session.SessionHandler.getSessionManager()Lorg/eclipse/jetty/server/SessionManager```.
## How was this patch tested?
manual tests:
```shell
build/sbt yarn/test -Pyarn
build/sbt yarn/test -Phadoop-3.2 -Pyarn
build/mvn -Dtest=none
-DwildcardSuites=org.apache.spark.deploy.yarn.YarnClusterSuite -pl
resource-managers/yarn test -Pyarn
build/mvn -Dtest=none
-DwildcardSuites=org.apache.spark.deploy.yarn.YarnClusterSuite -pl
resource-managers/yarn test -Pyarn -Phadoop-3.2
```
Closes #24115 from wangyum/hadoop3-yarn.
Authored-by: Yuming Wang <yumw...@ebay.com>
Signed-off-by: Sean Owen <sean.o...@databricks.com>
---
dev/.rat-excludes | 2 +
resource-managers/yarn/pom.xml | 8 +-
.../org/apache/hadoop/net/ServerSocketUtil.java | 132 ++++++++++
.../org/eclipse/jetty/server/SessionManager.java | 290 +++++++++++++++++++++
.../jetty/server/session/SessionHandler.java | 90 +++++++
5 files changed, 521 insertions(+), 1 deletion(-)
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index 59e619b..ccf266c 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -115,3 +115,5 @@ structured-streaming/*
kafka-source-initial-offset-version-2.1.0.bin
kafka-source-initial-offset-future-version.bin
vote.tmpl
+SessionManager.java
+SessionHandler.java
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index df910c1..0e5df14 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -29,7 +29,7 @@
<name>Spark Project YARN</name>
<properties>
<sbt.project.name>yarn</sbt.project.name>
- <jersey-1.version>1.9</jersey-1.version>
+ <jersey-1.version>1.19</jersey-1.version>
</properties>
<dependencies>
@@ -166,6 +166,12 @@
<scope>test</scope>
<version>${jersey-1.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ <scope>test</scope>
+ <version>${jersey-1.version}</version>
+ </dependency>
<!-- These dependencies are duplicated from core, because dependencies in
the "provided"
scope are not transitive.-->
diff --git
a/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
b/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
new file mode 100644
index 0000000..df0ebcc
--- /dev/null
+++
b/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.net;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.util.Random;
+
+/**
+ * Copied from
+ *
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
+ * for Hadoop-3.x testing
+ */
+public class ServerSocketUtil {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(ServerSocketUtil.class);
+ private static Random rand = new Random();
+
+ /**
+ * Port scan & allocate is how most other apps find ports
+ *
+ * @param port given port
+ * @param retries number of retries
+ * @return
+ * @throws IOException
+ */
+ public static int getPort(int port, int retries) throws IOException {
+ int tryPort = port;
+ int tries = 0;
+ while (true) {
+ if (tries > 0 || tryPort == 0) {
+ tryPort = port + rand.nextInt(65535 - port);
+ }
+ if (tryPort == 0) {
+ continue;
+ }
+ try (ServerSocket s = new ServerSocket(tryPort)) {
+ LOG.info("Using port " + tryPort);
+ return tryPort;
+ } catch (IOException e) {
+ tries++;
+ if (tries >= retries) {
+ LOG.info("Port is already in use; giving up");
+ throw e;
+ } else {
+ LOG.info("Port is already in use; trying again");
+ }
+ }
+ }
+ }
+
+ /**
+ * Check whether port is available or not.
+ *
+ * @param port given port
+ * @return
+ */
+ private static boolean isPortAvailable(int port) {
+ try (ServerSocket s = new ServerSocket(port)) {
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Wait till the port available.
+ *
+ * @param port given port
+ * @param retries number of retries for given port
+ * @return
+ * @throws InterruptedException
+ * @throws IOException
+ */
+ public static int waitForPort(int port, int retries)
+ throws InterruptedException, IOException {
+ int tries = 0;
+ while (true) {
+ if (isPortAvailable(port)) {
+ return port;
+ } else {
+ tries++;
+ if (tries >= retries) {
+ throw new IOException(
+ "Port is already in use; giving up after " + tries + "
times.");
+ }
+ Thread.sleep(1000);
+ }
+ }
+ }
+
+ /**
+ * Find the specified number of unique ports available.
+ * The ports are all closed afterwards,
+ * so other network services started may grab those same ports.
+ *
+ * @param numPorts number of required port nubmers
+ * @return array of available port numbers
+ * @throws IOException
+ */
+ public static int[] getPorts(int numPorts) throws IOException {
+ ServerSocket[] sockets = new ServerSocket[numPorts];
+ int[] ports = new int[numPorts];
+ for (int i = 0; i < numPorts; i++) {
+ ServerSocket sock = new ServerSocket(0);
+ sockets[i] = sock;
+ ports[i] = sock.getLocalPort();
+ }
+ for (ServerSocket sock : sockets) {
+ sock.close();
+ }
+ return ports;
+ }
+}
diff --git
a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java
b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java
new file mode 100644
index 0000000..bf89f8d
--- /dev/null
+++
b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java
@@ -0,0 +1,290 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd.
+// ------------------------------------------------------------------------
+// All rights reserved. This program and the accompanying materials
+// are made available under the terms of the Eclipse Public License v1.0
+// and Apache License v2.0 which accompanies this distribution.
+//
+// The Eclipse Public License is available at
+// http://www.eclipse.org/legal/epl-v10.html
+//
+// The Apache License v2.0 is available at
+// http://www.opensource.org/licenses/apache2.0.php
+//
+// You may elect to redistribute this code under either of these licenses.
+// ========================================================================
+//
+
+package org.eclipse.jetty.server;
+
+import javax.servlet.SessionCookieConfig;
+import javax.servlet.SessionTrackingMode;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpSession;
+import java.util.EventListener;
+import java.util.Set;
+
+import org.eclipse.jetty.http.HttpCookie;
+import org.eclipse.jetty.server.session.SessionHandler;
+import org.eclipse.jetty.util.component.LifeCycle;
+
+/**
+ * Adapted from
https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/
+ * jetty-server/src/main/java/org/eclipse/jetty/server/SessionManager.java
+ */
+public interface SessionManager extends LifeCycle {
+ /**
+ * Session cookie name.
+ * Defaults to <code>JSESSIONID</code>, but can be set with the
+ * <code>org.eclipse.jetty.servlet.SessionCookie</code> context init
parameter.
+ */
+ String __SessionCookieProperty = "org.eclipse.jetty.servlet.SessionCookie";
+ String __DefaultSessionCookie = "JSESSIONID";
+
+ /**
+ * Session id path parameter name.
+ * Defaults to <code>jsessionid</code>, but can be set with the
+ * <code>org.eclipse.jetty.servlet.SessionIdPathParameterName</code> context
init parameter.
+ * If set to null or "none" no URL rewriting will be done.
+ */
+ String __SessionIdPathParameterNameProperty =
+ "org.eclipse.jetty.servlet.SessionIdPathParameterName";
+ String __DefaultSessionIdPathParameterName = "jsessionid";
+ String __CheckRemoteSessionEncoding =
"org.eclipse.jetty.servlet.CheckingRemoteSessionIdEncoding";
+
+ /**
+ * Session Domain.
+ * If this property is set as a ServletContext InitParam, then it is
+ * used as the domain for session cookies. If it is not set, then
+ * no domain is specified for the session cookie.
+ */
+ String __SessionDomainProperty = "org.eclipse.jetty.servlet.SessionDomain";
+ String __DefaultSessionDomain = null;
+
+ /**
+ * Session Path.
+ * If this property is set as a ServletContext InitParam, then it is
+ * used as the path for the session cookie. If it is not set, then
+ * the context path is used as the path for the cookie.
+ */
+ String __SessionPathProperty = "org.eclipse.jetty.servlet.SessionPath";
+
+ /**
+ * Session Max Age.
+ * If this property is set as a ServletContext InitParam, then it is
+ * used as the max age for the session cookie. If it is not set, then
+ * a max age of -1 is used.
+ */
+ String __MaxAgeProperty = "org.eclipse.jetty.servlet.MaxAge";
+
+ /**
+ * Returns the <code>HttpSession</code> with the given session id
+ *
+ * @param id the session id
+ * @return the <code>HttpSession</code> with the corresponding id
+ * or null if no session with the given id exists
+ */
+ HttpSession getHttpSession(String id);
+
+ /**
+ * Creates a new <code>HttpSession</code>.
+ *
+ * @param request the HttpServletRequest containing the requested session id
+ * @return the new <code>HttpSession</code>
+ */
+ HttpSession newHttpSession(HttpServletRequest request);
+
+ /**
+ * @return true if session cookies should be HTTP-only (Microsoft extension)
+ * @see HttpCookie#isHttpOnly()
+ */
+ boolean getHttpOnly();
+
+ /**
+ * @return the max period of inactivity, after which the session is
invalidated, in seconds.
+ * @see #setMaxInactiveInterval(int)
+ */
+ int getMaxInactiveInterval();
+
+ /**
+ * Sets the max period of inactivity, after which the session is
invalidated, in seconds.
+ *
+ * @param seconds the max inactivity period, in seconds.
+ * @see #getMaxInactiveInterval()
+ */
+ void setMaxInactiveInterval(int seconds);
+
+ /**
+ * Sets the {@link SessionHandler}.
+ *
+ * @param handler the <code>SessionHandler</code> object
+ */
+ void setSessionHandler(SessionHandler handler);
+
+ /**
+ * Adds an event listener for session-related events.
+ *
+ * @param listener the session event listener to add
+ * Individual SessionManagers implementations may accept
arbitrary listener types,
+ * but they are expected to at least handle
HttpSessionActivationListener,
+ * HttpSessionAttributeListener,
+ * HttpSessionBindingListener and HttpSessionListener.
+ * @see #removeEventListener(EventListener)
+ */
+ void addEventListener(EventListener listener);
+
+ /**
+ * Removes an event listener for for session-related events.
+ *
+ * @param listener the session event listener to remove
+ * @see #addEventListener(EventListener)
+ */
+ void removeEventListener(EventListener listener);
+
+ /**
+ * Removes all event listeners for session-related events.
+ *
+ * @see #removeEventListener(EventListener)
+ */
+ void clearEventListeners();
+
+ /**
+ * Gets a Cookie for a session.
+ *
+ * @param session the session to which the cookie should refer.
+ * @param contextPath the context to which the cookie should be linked.
+ * The client will only send the cookie value when
+ * requesting resources under this path.
+ * @param requestIsSecure whether the client is accessing the server over
+ * a secure protocol (i.e. HTTPS).
+ * @return if this <code>SessionManager</code> uses cookies, then this
method will return a new
+ * {@link Cookie cookie object} that should be set on the client
+ * in order to link future HTTP requests
+ * with the <code>session</code>. If cookies are not in use,
+ * this method returns <code>null</code>.
+ */
+ HttpCookie getSessionCookie(HttpSession session, String contextPath, boolean
requestIsSecure);
+
+ /**
+ * @return the cross context session id manager.
+ * @see #setSessionIdManager(SessionIdManager)
+ */
+ SessionIdManager getSessionIdManager();
+
+ /**
+ * @return the cross context session id manager.
+ * @deprecated use {@link #getSessionIdManager()}
+ */
+ @Deprecated
+ SessionIdManager getMetaManager();
+
+ /**
+ * Sets the cross context session id manager
+ *
+ * @param idManager the cross context session id manager.
+ * @see #getSessionIdManager()
+ */
+ void setSessionIdManager(SessionIdManager idManager);
+
+ /**
+ * @param session the session to test for validity
+ * @return whether the given session is valid, that is, it has not been
invalidated.
+ */
+ boolean isValid(HttpSession session);
+
+ /**
+ * @param session the session object
+ * @return the unique id of the session within the cluster, extended with an
optional node id.
+ * @see #getClusterId(HttpSession)
+ */
+ String getNodeId(HttpSession session);
+
+ /**
+ * @param session the session object
+ * @return the unique id of the session within the cluster (without a node
id extension)
+ * @see #getNodeId(HttpSession)
+ */
+ String getClusterId(HttpSession session);
+
+ /**
+ * Called by the {@link SessionHandler} when a session is first accessed by
a request.
+ *
+ * @param session the session object
+ * @param secure whether the request is secure or not
+ * @return the session cookie. If not null,
+ * this cookie should be set on the response to either migrate
+ * the session or to refresh a session cookie that may expire.
+ * @see #complete(HttpSession)
+ */
+ HttpCookie access(HttpSession session, boolean secure);
+
+ /**
+ * Called by the {@link SessionHandler} when a session is last accessed by a
request.
+ *
+ * @param session the session object
+ * @see #access(HttpSession, boolean)
+ */
+ void complete(HttpSession session);
+
+ /**
+ * Sets the session id URL path parameter name.
+ *
+ * @param parameterName the URL path parameter name
+ * for session id URL rewriting (null or "none" for no
rewriting).
+ * @see #getSessionIdPathParameterName()
+ * @see #getSessionIdPathParameterNamePrefix()
+ */
+ void setSessionIdPathParameterName(String parameterName);
+
+ /**
+ * @return the URL path parameter name for session id URL rewriting, by
default "jsessionid".
+ * @see #setSessionIdPathParameterName(String)
+ */
+ String getSessionIdPathParameterName();
+
+ /**
+ * @return a formatted version of {@link #getSessionIdPathParameterName()},
by default
+ * ";" + sessionIdParameterName + "=", for easier lookup in URL
strings.
+ * @see #getSessionIdPathParameterName()
+ */
+ String getSessionIdPathParameterNamePrefix();
+
+ /**
+ * @return whether the session management is handled via cookies.
+ */
+ boolean isUsingCookies();
+
+ /**
+ * @return whether the session management is handled via URLs.
+ */
+ boolean isUsingURLs();
+
+ Set<SessionTrackingMode> getDefaultSessionTrackingModes();
+
+ Set<SessionTrackingMode> getEffectiveSessionTrackingModes();
+
+ void setSessionTrackingModes(Set<SessionTrackingMode> sessionTrackingModes);
+
+ SessionCookieConfig getSessionCookieConfig();
+
+ /**
+ * @return True if absolute URLs are check for remoteness before being
session encoded.
+ */
+ boolean isCheckingRemoteSessionIdEncoding();
+
+ /**
+ * @param remote True if absolute URLs are check for remoteness before being
session encoded.
+ */
+ void setCheckingRemoteSessionIdEncoding(boolean remote);
+
+ /** Change the existing session id.
+ *
+ * @param oldClusterId the old cluster id
+ * @param oldNodeId the old node id
+ * @param newClusterId the new cluster id
+ * @param newNodeId the new node id
+ */
+ void renewSessionId(String oldClusterId, String oldNodeId, String
newClusterId, String newNodeId);
+}
diff --git
a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java
b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java
new file mode 100644
index 0000000..f8bcf8d
--- /dev/null
+++
b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java
@@ -0,0 +1,90 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd.
+// ------------------------------------------------------------------------
+// All rights reserved. This program and the accompanying materials
+// are made available under the terms of the Eclipse Public License v1.0
+// and Apache License v2.0 which accompanies this distribution.
+//
+// The Eclipse Public License is available at
+// http://www.eclipse.org/legal/epl-v10.html
+//
+// The Apache License v2.0 is available at
+// http://www.opensource.org/licenses/apache2.0.php
+//
+// You may elect to redistribute this code under either of these licenses.
+// ========================================================================
+//
+
+package org.eclipse.jetty.server.session;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.SessionManager;
+import org.eclipse.jetty.server.handler.ScopedHandler;
+
+/**
+ * Adapted from
https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/
+ *
jetty-server/src/main/java/org/eclipse/jetty/server/session/SessionHandler.java
+ */
+public class SessionHandler extends ScopedHandler {
+ private SessionManager _sessionManager;
+
+ public SessionHandler() {
+ }
+
+ /**
+ * @param manager
+ * The session manager
+ */
+ public SessionHandler(SessionManager manager) {
+ setSessionManager(manager);
+ }
+
+ /**
+ * @return Returns the sessionManager.
+ */
+ public SessionManager getSessionManager() {
+ return _sessionManager;
+ }
+
+ /**
+ * @param sessionManager
+ * The sessionManager to set.
+ */
+ public void setSessionManager(SessionManager sessionManager) {
+ if (isStarted()) {
+ throw new IllegalStateException();
+ }
+ if (sessionManager != null) {
+ updateBean(_sessionManager,sessionManager);
+ _sessionManager=sessionManager;
+ }
+ }
+
+ /*
+ * @see
org.eclipse.jetty.server.Handler#handle(javax.servlet.http.HttpServletRequest,
+ * javax.servlet.http.HttpServletResponse, int)
+ */
+ @Override
+ public void doHandle(String target, Request baseRequest, HttpServletRequest
request,
+ HttpServletResponse response) throws IOException, ServletException {
+ // start manual inline of nextHandle(target,baseRequest,request,response);
+ if (_nextScope != null && _nextScope == _handler) {
+ _nextScope.doHandle(target,baseRequest,request,response);
+ } else if (_handler != null) {
+ _handler.handle(target,baseRequest,request,response);
+ // end manual inline
+ }
+ }
+
+ public void clearEventListeners() {
+ if (_sessionManager != null) {
+ _sessionManager.clearEventListeners();
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
