This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 64e5928e863 [SPARK-42463][SPARK-27180][YARN][TESTS] Clean up the third-party Java files copy introduced by 64e5928e863 is described below commit 64e5928e86377a216e48bb7be12150ad130322f3 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Thu Feb 16 09:26:33 2023 -0600 [SPARK-42463][SPARK-27180][YARN][TESTS] Clean up the third-party Java files copy introduced by ### What changes were proposed in this pull request? SPARK-27180 introduced some third-party Java source code to solve Yarn module test failure, but maven and sbt can also test pass without them, so this pr remove these files. ### Why are the changes needed? Clean up the third-party Java source code copy in Spark. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? - Pass GitHub Actions - manual check: **Maven** ``` build/mvn clean build/mvn clean install -DskipTestes -pl resource-managers/yarn -am -Pyarn build/mvn -Dtest=none -DwildcardSuites=org.apache.spark.deploy.yarn.YarnClusterSuite -pl resource-managers/yarn test -Pyarn build/mvn test -pl resource-managers/yarn -Pyarn -Dtest.exclude.tags=org.apache.spark.tags.ExtendedLevelDBTest ``` Both `YarnClusterSuite` and full module test passed. **SBT** ``` build/sbt clean yarn/test -Pyarn -Dtest.exclude.tags=org.apache.spark.tags.ExtendedLevelDBTest ``` All tests passed. Closes #40052 from LuciferYang/SPARK-42463. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- dev/.rat-excludes | 2 - .../org/apache/hadoop/net/ServerSocketUtil.java | 132 ---------- .../org/eclipse/jetty/server/SessionManager.java | 290 --------------------- .../jetty/server/session/SessionHandler.java | 90 ------- 4 files changed, 514 deletions(-) diff --git a/dev/.rat-excludes b/dev/.rat-excludes index 458fe7594b2..9ac820fc216 100644 --- a/dev/.rat-excludes +++ b/dev/.rat-excludes @@ -118,8 +118,6 @@ kafka-source-initial-offset-version-2.1.0.bin kafka-source-initial-offset-future-version.bin announce.tmpl vote.tmpl -SessionManager.java -SessionHandler.java GangliaReporter.java application_1578436911597_0052 config.properties diff --git a/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java b/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java deleted file mode 100644 index 89e012ecd42..00000000000 --- a/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.net; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.ServerSocket; -import java.util.Random; - -/** - * Copied from - * hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java - * for Hadoop-3.x testing - */ -public class ServerSocketUtil { - - private static final Logger LOG = LoggerFactory.getLogger(ServerSocketUtil.class); - private static Random rand = new Random(); - - /** - * Port scan & allocate is how most other apps find ports - * - * @param port given port - * @param retries number of retries - * @return - * @throws IOException - */ - public static int getPort(int port, int retries) throws IOException { - int tryPort = port; - int tries = 0; - while (true) { - if (tries > 0 || tryPort == 0) { - tryPort = port + rand.nextInt(65535 - port); - } - if (tryPort == 0) { - continue; - } - try (ServerSocket s = new ServerSocket(tryPort)) { - LOG.info("Using port " + tryPort); - return tryPort; - } catch (IOException e) { - tries++; - if (tries >= retries) { - LOG.info("Port is already in use; giving up"); - throw e; - } else { - LOG.info("Port is already in use; trying again"); - } - } - } - } - - /** - * Check whether port is available or not. - * - * @param port given port - * @return - */ - private static boolean isPortAvailable(int port) { - try (ServerSocket s = new ServerSocket(port)) { - return true; - } catch (IOException e) { - return false; - } - } - - /** - * Wait till the port available. - * - * @param port given port - * @param retries number of retries for given port - * @return - * @throws InterruptedException - * @throws IOException - */ - public static int waitForPort(int port, int retries) - throws InterruptedException, IOException { - int tries = 0; - while (true) { - if (isPortAvailable(port)) { - return port; - } else { - tries++; - if (tries >= retries) { - throw new IOException( - "Port is already in use; giving up after " + tries + " times."); - } - Thread.sleep(1000); - } - } - } - - /** - * Find the specified number of unique ports available. - * The ports are all closed afterwards, - * so other network services started may grab those same ports. - * - * @param numPorts number of required port numbers - * @return array of available port numbers - * @throws IOException - */ - public static int[] getPorts(int numPorts) throws IOException { - ServerSocket[] sockets = new ServerSocket[numPorts]; - int[] ports = new int[numPorts]; - for (int i = 0; i < numPorts; i++) { - ServerSocket sock = new ServerSocket(0); - sockets[i] = sock; - ports[i] = sock.getLocalPort(); - } - for (ServerSocket sock : sockets) { - sock.close(); - } - return ports; - } -} diff --git a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java deleted file mode 100644 index bf89f8d7c80..00000000000 --- a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java +++ /dev/null @@ -1,290 +0,0 @@ -// -// ======================================================================== -// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd. -// ------------------------------------------------------------------------ -// All rights reserved. This program and the accompanying materials -// are made available under the terms of the Eclipse Public License v1.0 -// and Apache License v2.0 which accompanies this distribution. -// -// The Eclipse Public License is available at -// http://www.eclipse.org/legal/epl-v10.html -// -// The Apache License v2.0 is available at -// http://www.opensource.org/licenses/apache2.0.php -// -// You may elect to redistribute this code under either of these licenses. -// ======================================================================== -// - -package org.eclipse.jetty.server; - -import javax.servlet.SessionCookieConfig; -import javax.servlet.SessionTrackingMode; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpSession; -import java.util.EventListener; -import java.util.Set; - -import org.eclipse.jetty.http.HttpCookie; -import org.eclipse.jetty.server.session.SessionHandler; -import org.eclipse.jetty.util.component.LifeCycle; - -/** - * Adapted from https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/ - * jetty-server/src/main/java/org/eclipse/jetty/server/SessionManager.java - */ -public interface SessionManager extends LifeCycle { - /** - * Session cookie name. - * Defaults to <code>JSESSIONID</code>, but can be set with the - * <code>org.eclipse.jetty.servlet.SessionCookie</code> context init parameter. - */ - String __SessionCookieProperty = "org.eclipse.jetty.servlet.SessionCookie"; - String __DefaultSessionCookie = "JSESSIONID"; - - /** - * Session id path parameter name. - * Defaults to <code>jsessionid</code>, but can be set with the - * <code>org.eclipse.jetty.servlet.SessionIdPathParameterName</code> context init parameter. - * If set to null or "none" no URL rewriting will be done. - */ - String __SessionIdPathParameterNameProperty = - "org.eclipse.jetty.servlet.SessionIdPathParameterName"; - String __DefaultSessionIdPathParameterName = "jsessionid"; - String __CheckRemoteSessionEncoding = "org.eclipse.jetty.servlet.CheckingRemoteSessionIdEncoding"; - - /** - * Session Domain. - * If this property is set as a ServletContext InitParam, then it is - * used as the domain for session cookies. If it is not set, then - * no domain is specified for the session cookie. - */ - String __SessionDomainProperty = "org.eclipse.jetty.servlet.SessionDomain"; - String __DefaultSessionDomain = null; - - /** - * Session Path. - * If this property is set as a ServletContext InitParam, then it is - * used as the path for the session cookie. If it is not set, then - * the context path is used as the path for the cookie. - */ - String __SessionPathProperty = "org.eclipse.jetty.servlet.SessionPath"; - - /** - * Session Max Age. - * If this property is set as a ServletContext InitParam, then it is - * used as the max age for the session cookie. If it is not set, then - * a max age of -1 is used. - */ - String __MaxAgeProperty = "org.eclipse.jetty.servlet.MaxAge"; - - /** - * Returns the <code>HttpSession</code> with the given session id - * - * @param id the session id - * @return the <code>HttpSession</code> with the corresponding id - * or null if no session with the given id exists - */ - HttpSession getHttpSession(String id); - - /** - * Creates a new <code>HttpSession</code>. - * - * @param request the HttpServletRequest containing the requested session id - * @return the new <code>HttpSession</code> - */ - HttpSession newHttpSession(HttpServletRequest request); - - /** - * @return true if session cookies should be HTTP-only (Microsoft extension) - * @see HttpCookie#isHttpOnly() - */ - boolean getHttpOnly(); - - /** - * @return the max period of inactivity, after which the session is invalidated, in seconds. - * @see #setMaxInactiveInterval(int) - */ - int getMaxInactiveInterval(); - - /** - * Sets the max period of inactivity, after which the session is invalidated, in seconds. - * - * @param seconds the max inactivity period, in seconds. - * @see #getMaxInactiveInterval() - */ - void setMaxInactiveInterval(int seconds); - - /** - * Sets the {@link SessionHandler}. - * - * @param handler the <code>SessionHandler</code> object - */ - void setSessionHandler(SessionHandler handler); - - /** - * Adds an event listener for session-related events. - * - * @param listener the session event listener to add - * Individual SessionManagers implementations may accept arbitrary listener types, - * but they are expected to at least handle HttpSessionActivationListener, - * HttpSessionAttributeListener, - * HttpSessionBindingListener and HttpSessionListener. - * @see #removeEventListener(EventListener) - */ - void addEventListener(EventListener listener); - - /** - * Removes an event listener for for session-related events. - * - * @param listener the session event listener to remove - * @see #addEventListener(EventListener) - */ - void removeEventListener(EventListener listener); - - /** - * Removes all event listeners for session-related events. - * - * @see #removeEventListener(EventListener) - */ - void clearEventListeners(); - - /** - * Gets a Cookie for a session. - * - * @param session the session to which the cookie should refer. - * @param contextPath the context to which the cookie should be linked. - * The client will only send the cookie value when - * requesting resources under this path. - * @param requestIsSecure whether the client is accessing the server over - * a secure protocol (i.e. HTTPS). - * @return if this <code>SessionManager</code> uses cookies, then this method will return a new - * {@link Cookie cookie object} that should be set on the client - * in order to link future HTTP requests - * with the <code>session</code>. If cookies are not in use, - * this method returns <code>null</code>. - */ - HttpCookie getSessionCookie(HttpSession session, String contextPath, boolean requestIsSecure); - - /** - * @return the cross context session id manager. - * @see #setSessionIdManager(SessionIdManager) - */ - SessionIdManager getSessionIdManager(); - - /** - * @return the cross context session id manager. - * @deprecated use {@link #getSessionIdManager()} - */ - @Deprecated - SessionIdManager getMetaManager(); - - /** - * Sets the cross context session id manager - * - * @param idManager the cross context session id manager. - * @see #getSessionIdManager() - */ - void setSessionIdManager(SessionIdManager idManager); - - /** - * @param session the session to test for validity - * @return whether the given session is valid, that is, it has not been invalidated. - */ - boolean isValid(HttpSession session); - - /** - * @param session the session object - * @return the unique id of the session within the cluster, extended with an optional node id. - * @see #getClusterId(HttpSession) - */ - String getNodeId(HttpSession session); - - /** - * @param session the session object - * @return the unique id of the session within the cluster (without a node id extension) - * @see #getNodeId(HttpSession) - */ - String getClusterId(HttpSession session); - - /** - * Called by the {@link SessionHandler} when a session is first accessed by a request. - * - * @param session the session object - * @param secure whether the request is secure or not - * @return the session cookie. If not null, - * this cookie should be set on the response to either migrate - * the session or to refresh a session cookie that may expire. - * @see #complete(HttpSession) - */ - HttpCookie access(HttpSession session, boolean secure); - - /** - * Called by the {@link SessionHandler} when a session is last accessed by a request. - * - * @param session the session object - * @see #access(HttpSession, boolean) - */ - void complete(HttpSession session); - - /** - * Sets the session id URL path parameter name. - * - * @param parameterName the URL path parameter name - * for session id URL rewriting (null or "none" for no rewriting). - * @see #getSessionIdPathParameterName() - * @see #getSessionIdPathParameterNamePrefix() - */ - void setSessionIdPathParameterName(String parameterName); - - /** - * @return the URL path parameter name for session id URL rewriting, by default "jsessionid". - * @see #setSessionIdPathParameterName(String) - */ - String getSessionIdPathParameterName(); - - /** - * @return a formatted version of {@link #getSessionIdPathParameterName()}, by default - * ";" + sessionIdParameterName + "=", for easier lookup in URL strings. - * @see #getSessionIdPathParameterName() - */ - String getSessionIdPathParameterNamePrefix(); - - /** - * @return whether the session management is handled via cookies. - */ - boolean isUsingCookies(); - - /** - * @return whether the session management is handled via URLs. - */ - boolean isUsingURLs(); - - Set<SessionTrackingMode> getDefaultSessionTrackingModes(); - - Set<SessionTrackingMode> getEffectiveSessionTrackingModes(); - - void setSessionTrackingModes(Set<SessionTrackingMode> sessionTrackingModes); - - SessionCookieConfig getSessionCookieConfig(); - - /** - * @return True if absolute URLs are check for remoteness before being session encoded. - */ - boolean isCheckingRemoteSessionIdEncoding(); - - /** - * @param remote True if absolute URLs are check for remoteness before being session encoded. - */ - void setCheckingRemoteSessionIdEncoding(boolean remote); - - /** Change the existing session id. - * - * @param oldClusterId the old cluster id - * @param oldNodeId the old node id - * @param newClusterId the new cluster id - * @param newNodeId the new node id - */ - void renewSessionId(String oldClusterId, String oldNodeId, String newClusterId, String newNodeId); -} diff --git a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java deleted file mode 100644 index f8bcf8de82f..00000000000 --- a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java +++ /dev/null @@ -1,90 +0,0 @@ -// -// ======================================================================== -// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd. -// ------------------------------------------------------------------------ -// All rights reserved. This program and the accompanying materials -// are made available under the terms of the Eclipse Public License v1.0 -// and Apache License v2.0 which accompanies this distribution. -// -// The Eclipse Public License is available at -// http://www.eclipse.org/legal/epl-v10.html -// -// The Apache License v2.0 is available at -// http://www.opensource.org/licenses/apache2.0.php -// -// You may elect to redistribute this code under either of these licenses. -// ======================================================================== -// - -package org.eclipse.jetty.server.session; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; - -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.SessionManager; -import org.eclipse.jetty.server.handler.ScopedHandler; - -/** - * Adapted from https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/ - * jetty-server/src/main/java/org/eclipse/jetty/server/session/SessionHandler.java - */ -public class SessionHandler extends ScopedHandler { - private SessionManager _sessionManager; - - public SessionHandler() { - } - - /** - * @param manager - * The session manager - */ - public SessionHandler(SessionManager manager) { - setSessionManager(manager); - } - - /** - * @return Returns the sessionManager. - */ - public SessionManager getSessionManager() { - return _sessionManager; - } - - /** - * @param sessionManager - * The sessionManager to set. - */ - public void setSessionManager(SessionManager sessionManager) { - if (isStarted()) { - throw new IllegalStateException(); - } - if (sessionManager != null) { - updateBean(_sessionManager,sessionManager); - _sessionManager=sessionManager; - } - } - - /* - * @see org.eclipse.jetty.server.Handler#handle(javax.servlet.http.HttpServletRequest, - * javax.servlet.http.HttpServletResponse, int) - */ - @Override - public void doHandle(String target, Request baseRequest, HttpServletRequest request, - HttpServletResponse response) throws IOException, ServletException { - // start manual inline of nextHandle(target,baseRequest,request,response); - if (_nextScope != null && _nextScope == _handler) { - _nextScope.doHandle(target,baseRequest,request,response); - } else if (_handler != null) { - _handler.handle(target,baseRequest,request,response); - // end manual inline - } - } - - public void clearEventListeners() { - if (_sessionManager != null) { - _sessionManager.clearEventListeners(); - } - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org