This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 64e5928e863 [SPARK-42463][SPARK-27180][YARN][TESTS] Clean up the
third-party Java files copy introduced by
64e5928e863 is described below
commit 64e5928e86377a216e48bb7be12150ad130322f3
Author: yangjie01 <yangji...@baidu.com>
AuthorDate: Thu Feb 16 09:26:33 2023 -0600
[SPARK-42463][SPARK-27180][YARN][TESTS] Clean up the third-party Java files
copy introduced by
### What changes were proposed in this pull request?
SPARK-27180 introduced some third-party Java source code to solve Yarn
module test failure, but maven and sbt can also test pass without them, so
this pr remove these files.
### Why are the changes needed?
Clean up the third-party Java source code copy in Spark.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- Pass GitHub Actions
- manual check:
**Maven**
```
build/mvn clean
build/mvn clean install -DskipTestes -pl resource-managers/yarn -am -Pyarn
build/mvn -Dtest=none
-DwildcardSuites=org.apache.spark.deploy.yarn.YarnClusterSuite -pl
resource-managers/yarn test -Pyarn
build/mvn test -pl resource-managers/yarn -Pyarn
-Dtest.exclude.tags=org.apache.spark.tags.ExtendedLevelDBTest
```
Both `YarnClusterSuite` and full module test passed.
**SBT**
```
build/sbt clean yarn/test -Pyarn
-Dtest.exclude.tags=org.apache.spark.tags.ExtendedLevelDBTest
```
All tests passed.
Closes #40052 from LuciferYang/SPARK-42463.
Authored-by: yangjie01 <yangji...@baidu.com>
Signed-off-by: Sean Owen <sro...@gmail.com>
---
dev/.rat-excludes | 2 -
.../org/apache/hadoop/net/ServerSocketUtil.java | 132 ----------
.../org/eclipse/jetty/server/SessionManager.java | 290 ---------------------
.../jetty/server/session/SessionHandler.java | 90 -------
4 files changed, 514 deletions(-)
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index 458fe7594b2..9ac820fc216 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -118,8 +118,6 @@ kafka-source-initial-offset-version-2.1.0.bin
kafka-source-initial-offset-future-version.bin
announce.tmpl
vote.tmpl
-SessionManager.java
-SessionHandler.java
GangliaReporter.java
application_1578436911597_0052
config.properties
diff --git
a/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
b/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
deleted file mode 100644
index 89e012ecd42..00000000000
---
a/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.net;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.util.Random;
-
-/**
- * Copied from
- *
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
- * for Hadoop-3.x testing
- */
-public class ServerSocketUtil {
-
- private static final Logger LOG =
LoggerFactory.getLogger(ServerSocketUtil.class);
- private static Random rand = new Random();
-
- /**
- * Port scan & allocate is how most other apps find ports
- *
- * @param port given port
- * @param retries number of retries
- * @return
- * @throws IOException
- */
- public static int getPort(int port, int retries) throws IOException {
- int tryPort = port;
- int tries = 0;
- while (true) {
- if (tries > 0 || tryPort == 0) {
- tryPort = port + rand.nextInt(65535 - port);
- }
- if (tryPort == 0) {
- continue;
- }
- try (ServerSocket s = new ServerSocket(tryPort)) {
- LOG.info("Using port " + tryPort);
- return tryPort;
- } catch (IOException e) {
- tries++;
- if (tries >= retries) {
- LOG.info("Port is already in use; giving up");
- throw e;
- } else {
- LOG.info("Port is already in use; trying again");
- }
- }
- }
- }
-
- /**
- * Check whether port is available or not.
- *
- * @param port given port
- * @return
- */
- private static boolean isPortAvailable(int port) {
- try (ServerSocket s = new ServerSocket(port)) {
- return true;
- } catch (IOException e) {
- return false;
- }
- }
-
- /**
- * Wait till the port available.
- *
- * @param port given port
- * @param retries number of retries for given port
- * @return
- * @throws InterruptedException
- * @throws IOException
- */
- public static int waitForPort(int port, int retries)
- throws InterruptedException, IOException {
- int tries = 0;
- while (true) {
- if (isPortAvailable(port)) {
- return port;
- } else {
- tries++;
- if (tries >= retries) {
- throw new IOException(
- "Port is already in use; giving up after " + tries + "
times.");
- }
- Thread.sleep(1000);
- }
- }
- }
-
- /**
- * Find the specified number of unique ports available.
- * The ports are all closed afterwards,
- * so other network services started may grab those same ports.
- *
- * @param numPorts number of required port numbers
- * @return array of available port numbers
- * @throws IOException
- */
- public static int[] getPorts(int numPorts) throws IOException {
- ServerSocket[] sockets = new ServerSocket[numPorts];
- int[] ports = new int[numPorts];
- for (int i = 0; i < numPorts; i++) {
- ServerSocket sock = new ServerSocket(0);
- sockets[i] = sock;
- ports[i] = sock.getLocalPort();
- }
- for (ServerSocket sock : sockets) {
- sock.close();
- }
- return ports;
- }
-}
diff --git
a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java
b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java
deleted file mode 100644
index bf89f8d7c80..00000000000
---
a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java
+++ /dev/null
@@ -1,290 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd.
-// ------------------------------------------------------------------------
-// All rights reserved. This program and the accompanying materials
-// are made available under the terms of the Eclipse Public License v1.0
-// and Apache License v2.0 which accompanies this distribution.
-//
-// The Eclipse Public License is available at
-// http://www.eclipse.org/legal/epl-v10.html
-//
-// The Apache License v2.0 is available at
-// http://www.opensource.org/licenses/apache2.0.php
-//
-// You may elect to redistribute this code under either of these licenses.
-// ========================================================================
-//
-
-package org.eclipse.jetty.server;
-
-import javax.servlet.SessionCookieConfig;
-import javax.servlet.SessionTrackingMode;
-import javax.servlet.http.Cookie;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpSession;
-import java.util.EventListener;
-import java.util.Set;
-
-import org.eclipse.jetty.http.HttpCookie;
-import org.eclipse.jetty.server.session.SessionHandler;
-import org.eclipse.jetty.util.component.LifeCycle;
-
-/**
- * Adapted from
https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/
- * jetty-server/src/main/java/org/eclipse/jetty/server/SessionManager.java
- */
-public interface SessionManager extends LifeCycle {
- /**
- * Session cookie name.
- * Defaults to <code>JSESSIONID</code>, but can be set with the
- * <code>org.eclipse.jetty.servlet.SessionCookie</code> context init
parameter.
- */
- String __SessionCookieProperty = "org.eclipse.jetty.servlet.SessionCookie";
- String __DefaultSessionCookie = "JSESSIONID";
-
- /**
- * Session id path parameter name.
- * Defaults to <code>jsessionid</code>, but can be set with the
- * <code>org.eclipse.jetty.servlet.SessionIdPathParameterName</code> context
init parameter.
- * If set to null or "none" no URL rewriting will be done.
- */
- String __SessionIdPathParameterNameProperty =
- "org.eclipse.jetty.servlet.SessionIdPathParameterName";
- String __DefaultSessionIdPathParameterName = "jsessionid";
- String __CheckRemoteSessionEncoding =
"org.eclipse.jetty.servlet.CheckingRemoteSessionIdEncoding";
-
- /**
- * Session Domain.
- * If this property is set as a ServletContext InitParam, then it is
- * used as the domain for session cookies. If it is not set, then
- * no domain is specified for the session cookie.
- */
- String __SessionDomainProperty = "org.eclipse.jetty.servlet.SessionDomain";
- String __DefaultSessionDomain = null;
-
- /**
- * Session Path.
- * If this property is set as a ServletContext InitParam, then it is
- * used as the path for the session cookie. If it is not set, then
- * the context path is used as the path for the cookie.
- */
- String __SessionPathProperty = "org.eclipse.jetty.servlet.SessionPath";
-
- /**
- * Session Max Age.
- * If this property is set as a ServletContext InitParam, then it is
- * used as the max age for the session cookie. If it is not set, then
- * a max age of -1 is used.
- */
- String __MaxAgeProperty = "org.eclipse.jetty.servlet.MaxAge";
-
- /**
- * Returns the <code>HttpSession</code> with the given session id
- *
- * @param id the session id
- * @return the <code>HttpSession</code> with the corresponding id
- * or null if no session with the given id exists
- */
- HttpSession getHttpSession(String id);
-
- /**
- * Creates a new <code>HttpSession</code>.
- *
- * @param request the HttpServletRequest containing the requested session id
- * @return the new <code>HttpSession</code>
- */
- HttpSession newHttpSession(HttpServletRequest request);
-
- /**
- * @return true if session cookies should be HTTP-only (Microsoft extension)
- * @see HttpCookie#isHttpOnly()
- */
- boolean getHttpOnly();
-
- /**
- * @return the max period of inactivity, after which the session is
invalidated, in seconds.
- * @see #setMaxInactiveInterval(int)
- */
- int getMaxInactiveInterval();
-
- /**
- * Sets the max period of inactivity, after which the session is
invalidated, in seconds.
- *
- * @param seconds the max inactivity period, in seconds.
- * @see #getMaxInactiveInterval()
- */
- void setMaxInactiveInterval(int seconds);
-
- /**
- * Sets the {@link SessionHandler}.
- *
- * @param handler the <code>SessionHandler</code> object
- */
- void setSessionHandler(SessionHandler handler);
-
- /**
- * Adds an event listener for session-related events.
- *
- * @param listener the session event listener to add
- * Individual SessionManagers implementations may accept
arbitrary listener types,
- * but they are expected to at least handle
HttpSessionActivationListener,
- * HttpSessionAttributeListener,
- * HttpSessionBindingListener and HttpSessionListener.
- * @see #removeEventListener(EventListener)
- */
- void addEventListener(EventListener listener);
-
- /**
- * Removes an event listener for for session-related events.
- *
- * @param listener the session event listener to remove
- * @see #addEventListener(EventListener)
- */
- void removeEventListener(EventListener listener);
-
- /**
- * Removes all event listeners for session-related events.
- *
- * @see #removeEventListener(EventListener)
- */
- void clearEventListeners();
-
- /**
- * Gets a Cookie for a session.
- *
- * @param session the session to which the cookie should refer.
- * @param contextPath the context to which the cookie should be linked.
- * The client will only send the cookie value when
- * requesting resources under this path.
- * @param requestIsSecure whether the client is accessing the server over
- * a secure protocol (i.e. HTTPS).
- * @return if this <code>SessionManager</code> uses cookies, then this
method will return a new
- * {@link Cookie cookie object} that should be set on the client
- * in order to link future HTTP requests
- * with the <code>session</code>. If cookies are not in use,
- * this method returns <code>null</code>.
- */
- HttpCookie getSessionCookie(HttpSession session, String contextPath, boolean
requestIsSecure);
-
- /**
- * @return the cross context session id manager.
- * @see #setSessionIdManager(SessionIdManager)
- */
- SessionIdManager getSessionIdManager();
-
- /**
- * @return the cross context session id manager.
- * @deprecated use {@link #getSessionIdManager()}
- */
- @Deprecated
- SessionIdManager getMetaManager();
-
- /**
- * Sets the cross context session id manager
- *
- * @param idManager the cross context session id manager.
- * @see #getSessionIdManager()
- */
- void setSessionIdManager(SessionIdManager idManager);
-
- /**
- * @param session the session to test for validity
- * @return whether the given session is valid, that is, it has not been
invalidated.
- */
- boolean isValid(HttpSession session);
-
- /**
- * @param session the session object
- * @return the unique id of the session within the cluster, extended with an
optional node id.
- * @see #getClusterId(HttpSession)
- */
- String getNodeId(HttpSession session);
-
- /**
- * @param session the session object
- * @return the unique id of the session within the cluster (without a node
id extension)
- * @see #getNodeId(HttpSession)
- */
- String getClusterId(HttpSession session);
-
- /**
- * Called by the {@link SessionHandler} when a session is first accessed by
a request.
- *
- * @param session the session object
- * @param secure whether the request is secure or not
- * @return the session cookie. If not null,
- * this cookie should be set on the response to either migrate
- * the session or to refresh a session cookie that may expire.
- * @see #complete(HttpSession)
- */
- HttpCookie access(HttpSession session, boolean secure);
-
- /**
- * Called by the {@link SessionHandler} when a session is last accessed by a
request.
- *
- * @param session the session object
- * @see #access(HttpSession, boolean)
- */
- void complete(HttpSession session);
-
- /**
- * Sets the session id URL path parameter name.
- *
- * @param parameterName the URL path parameter name
- * for session id URL rewriting (null or "none" for no
rewriting).
- * @see #getSessionIdPathParameterName()
- * @see #getSessionIdPathParameterNamePrefix()
- */
- void setSessionIdPathParameterName(String parameterName);
-
- /**
- * @return the URL path parameter name for session id URL rewriting, by
default "jsessionid".
- * @see #setSessionIdPathParameterName(String)
- */
- String getSessionIdPathParameterName();
-
- /**
- * @return a formatted version of {@link #getSessionIdPathParameterName()},
by default
- * ";" + sessionIdParameterName + "=", for easier lookup in URL
strings.
- * @see #getSessionIdPathParameterName()
- */
- String getSessionIdPathParameterNamePrefix();
-
- /**
- * @return whether the session management is handled via cookies.
- */
- boolean isUsingCookies();
-
- /**
- * @return whether the session management is handled via URLs.
- */
- boolean isUsingURLs();
-
- Set<SessionTrackingMode> getDefaultSessionTrackingModes();
-
- Set<SessionTrackingMode> getEffectiveSessionTrackingModes();
-
- void setSessionTrackingModes(Set<SessionTrackingMode> sessionTrackingModes);
-
- SessionCookieConfig getSessionCookieConfig();
-
- /**
- * @return True if absolute URLs are check for remoteness before being
session encoded.
- */
- boolean isCheckingRemoteSessionIdEncoding();
-
- /**
- * @param remote True if absolute URLs are check for remoteness before being
session encoded.
- */
- void setCheckingRemoteSessionIdEncoding(boolean remote);
-
- /** Change the existing session id.
- *
- * @param oldClusterId the old cluster id
- * @param oldNodeId the old node id
- * @param newClusterId the new cluster id
- * @param newNodeId the new node id
- */
- void renewSessionId(String oldClusterId, String oldNodeId, String
newClusterId, String newNodeId);
-}
diff --git
a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java
b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java
deleted file mode 100644
index f8bcf8de82f..00000000000
---
a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java
+++ /dev/null
@@ -1,90 +0,0 @@
-//
-// ========================================================================
-// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd.
-// ------------------------------------------------------------------------
-// All rights reserved. This program and the accompanying materials
-// are made available under the terms of the Eclipse Public License v1.0
-// and Apache License v2.0 which accompanies this distribution.
-//
-// The Eclipse Public License is available at
-// http://www.eclipse.org/legal/epl-v10.html
-//
-// The Apache License v2.0 is available at
-// http://www.opensource.org/licenses/apache2.0.php
-//
-// You may elect to redistribute this code under either of these licenses.
-// ========================================================================
-//
-
-package org.eclipse.jetty.server.session;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-
-import org.eclipse.jetty.server.Request;
-import org.eclipse.jetty.server.SessionManager;
-import org.eclipse.jetty.server.handler.ScopedHandler;
-
-/**
- * Adapted from
https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/
- *
jetty-server/src/main/java/org/eclipse/jetty/server/session/SessionHandler.java
- */
-public class SessionHandler extends ScopedHandler {
- private SessionManager _sessionManager;
-
- public SessionHandler() {
- }
-
- /**
- * @param manager
- * The session manager
- */
- public SessionHandler(SessionManager manager) {
- setSessionManager(manager);
- }
-
- /**
- * @return Returns the sessionManager.
- */
- public SessionManager getSessionManager() {
- return _sessionManager;
- }
-
- /**
- * @param sessionManager
- * The sessionManager to set.
- */
- public void setSessionManager(SessionManager sessionManager) {
- if (isStarted()) {
- throw new IllegalStateException();
- }
- if (sessionManager != null) {
- updateBean(_sessionManager,sessionManager);
- _sessionManager=sessionManager;
- }
- }
-
- /*
- * @see
org.eclipse.jetty.server.Handler#handle(javax.servlet.http.HttpServletRequest,
- * javax.servlet.http.HttpServletResponse, int)
- */
- @Override
- public void doHandle(String target, Request baseRequest, HttpServletRequest
request,
- HttpServletResponse response) throws IOException, ServletException {
- // start manual inline of nextHandle(target,baseRequest,request,response);
- if (_nextScope != null && _nextScope == _handler) {
- _nextScope.doHandle(target,baseRequest,request,response);
- } else if (_handler != null) {
- _handler.handle(target,baseRequest,request,response);
- // end manual inline
- }
- }
-
- public void clearEventListeners() {
- if (_sessionManager != null) {
- _sessionManager.clearEventListeners();
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
