spark git commit: fix security issue of zinc(simplier version)

2018-10-21 Thread wenchen 
Repository: spark
Updated Branches:
  refs/heads/branch-2.2 ca950c17f -> 8906696ac


fix security issue of zinc(simplier version)


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8906696a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8906696a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8906696a

Branch: refs/heads/branch-2.2
Commit: 8906696ac2089f3d6500b0496af7d9995c7de99b
Parents: ca950c1
Author: Wenchen Fan 
Authored: Fri Oct 19 23:54:15 2018 +0800
Committer: Wenchen Fan 
Committed: Mon Oct 22 12:24:31 2018 +0800

--
 build/mvn   | 33 ++--
 dev/create-release/release-build.sh |  6 --
 dev/run-tests.py| 10 --
 3 files changed, 10 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/8906696a/build/mvn
--
diff --git a/build/mvn b/build/mvn
index 926027e..6a7d4db 100755
--- a/build/mvn
+++ b/build/mvn
@@ -130,17 +130,8 @@ if [ "$1" == "--force" ]; then
   shift
 fi
 
-if [ "$1" == "--zinc" ]; then
-  echo "Using zinc for incremental compilation. Be sure you are aware of the 
implications of "
-  echo "running this server process on your machine"
-  USE_ZINC=1
-  shift
-fi
-
 # Install the proper version of Scala, Zinc and Maven for the build
-if [ -n "${USE_ZINC}" ]; then
-  install_zinc
-fi
+install_zinc
 install_scala
 install_mvn
 
@@ -149,15 +140,13 @@ cd "${_CALLING_DIR}"
 
 # Now that zinc is ensured to be installed, check its status and, if its
 # not running or just installed, start it
-if [ -n "${USE_ZINC}" ]; then
-  if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port 
${ZINC_PORT}`" ]; then
-export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
-"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-"${ZINC_BIN}" -start -port ${ZINC_PORT} -server 127.0.0.1 \
-  -idle-timeout 30m \
-  -scala-compiler "${SCALA_COMPILER}" \
-  -scala-library "${SCALA_LIBRARY}" &>/dev/null
-  fi
+if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port 
${ZINC_PORT}`" ]; then
+  export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
+  "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
+  "${ZINC_BIN}" -start -port ${ZINC_PORT} \
+-server 127.0.0.1 -idle-timeout 30m \
+-scala-compiler "${SCALA_COMPILER}" \
+-scala-library "${SCALA_LIBRARY}" &>/dev/null
 fi
 
 # Set any `mvn` options if not already present
@@ -168,7 +157,5 @@ echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
 # Last, call the `mvn` command as usual
 ${MVN_BIN} -DzincPort=${ZINC_PORT} "$@"
 
-if [ -n "${USE_ZINC}" ]; then
-  # Try to shut down zinc explicitly
-  "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-fi
+# Try to shut down zinc explicitly
+"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}

http://git-wip-us.apache.org/repos/asf/spark/blob/8906696a/dev/create-release/release-build.sh
--
diff --git a/dev/create-release/release-build.sh 
b/dev/create-release/release-build.sh
index 1272b6d..b4d27ed 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -343,9 +343,6 @@ if [[ "$1" == "publish-snapshot" ]]; then
   $MVN -DzincPort=$ZINC_PORT -Dscala-2.10 --settings $tmp_settings \
 -DskipTests $PUBLISH_PROFILES clean deploy
 
-  # Clean-up Zinc nailgun process
-  $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
   rm $tmp_settings
   cd ..
   exit 0
@@ -381,9 +378,6 @@ if [[ "$1" == "publish-release" ]]; then
   $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.10 \
 -DskipTests $PUBLISH_PROFILES clean install
 
-  # Clean-up Zinc nailgun process
-  $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
   ./dev/change-version-to-2.10.sh
 
   pushd $tmp_repo/org/apache/spark

http://git-wip-us.apache.org/repos/asf/spark/blob/8906696a/dev/run-tests.py
--
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 72d148d..0a399ab 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -249,15 +249,6 @@ def get_zinc_port():
 return random.randrange(3030, 4030)
 
 
-def kill_zinc_on_port(zinc_port):
-"""
-Kill the Zinc process running on the given port, if one exists.
-"""
-cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN "
-   "| awk '{ print $2; }' | xargs kill") % zinc_port
-subprocess.check_call(cmd, shell=True)
-
-
 def exec_maven(mvn_args=()):
 """Will call Maven in the current directory with the list of mvn_args 
passed
 in and returns the subprocess for any further processing"""
@@ -267,7 +258,6 @@ def exec_maven(mvn_args=()):

spark git commit: fix security issue of zinc(simplier version)

2018-10-21 Thread wenchen 
Repository: spark
Updated Branches:
  refs/heads/branch-2.3 719ff7af6 -> d7a35877b


fix security issue of zinc(simplier version)


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d7a35877
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d7a35877
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d7a35877

Branch: refs/heads/branch-2.3
Commit: d7a35877b96dce8b742acf77e79bda189e402ae2
Parents: 719ff7a
Author: Wenchen Fan 
Authored: Fri Oct 19 23:54:15 2018 +0800
Committer: Wenchen Fan 
Committed: Mon Oct 22 12:22:10 2018 +0800

--
 build/mvn   | 33 ++--
 dev/create-release/release-build.sh |  6 --
 dev/run-tests.py| 10 --
 3 files changed, 10 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/d7a35877/build/mvn
--
diff --git a/build/mvn b/build/mvn
index 9c0d1a7..7951e10 100755
--- a/build/mvn
+++ b/build/mvn
@@ -130,17 +130,8 @@ if [ "$1" == "--force" ]; then
   shift
 fi
 
-if [ "$1" == "--zinc" ]; then
-  echo "Using zinc for incremental compilation. Be sure you are aware of the 
implications of "
-  echo "running this server process on your machine"
-  USE_ZINC=1
-  shift
-fi
-
 # Install the proper version of Scala, Zinc and Maven for the build
-if [ -n "${USE_ZINC}" ]; then
-  install_zinc
-fi
+install_zinc
 install_scala
 install_mvn
 
@@ -149,15 +140,13 @@ cd "${_CALLING_DIR}"
 
 # Now that zinc is ensured to be installed, check its status and, if its
 # not running or just installed, start it
-if [ -n "${USE_ZINC}" ]; then
-  if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port 
${ZINC_PORT}`" ]; then
-export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
-"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-"${ZINC_BIN}" -start -port ${ZINC_PORT} -server 127.0.0.1 \
-  -idle-timeout 30m \
-  -scala-compiler "${SCALA_COMPILER}" \
-  -scala-library "${SCALA_LIBRARY}" &>/dev/null
-  fi
+if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port 
${ZINC_PORT}`" ]; then
+  export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
+  "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
+  "${ZINC_BIN}" -start -port ${ZINC_PORT} \
+-server 127.0.0.1 -idle-timeout 30m \
+-scala-compiler "${SCALA_COMPILER}" \
+-scala-library "${SCALA_LIBRARY}" &>/dev/null
 fi
 
 # Set any `mvn` options if not already present
@@ -168,7 +157,5 @@ echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
 # Last, call the `mvn` command as usual
 ${MVN_BIN} -DzincPort=${ZINC_PORT} "$@"
 
-if [ -n "${USE_ZINC}" ]; then
-  # Try to shut down zinc explicitly
-  "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-fi
+# Try to shut down zinc explicitly
+"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}

http://git-wip-us.apache.org/repos/asf/spark/blob/d7a35877/dev/create-release/release-build.sh
--
diff --git a/dev/create-release/release-build.sh 
b/dev/create-release/release-build.sh
index 5faa3d3..3e60641 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -323,9 +323,6 @@ if [[ "$1" == "publish-snapshot" ]]; then
   #$MVN -DzincPort=$ZINC_PORT --settings $tmp_settings \
   #  -DskipTests $SCALA_2_12_PROFILES $PUBLISH_PROFILES clean deploy
 
-  # Clean-up Zinc nailgun process
-  $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
   rm $tmp_settings
   cd ..
   exit 0
@@ -360,9 +357,6 @@ if [[ "$1" == "publish-release" ]]; then
   #$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo \
   #  -DskipTests $SCALA_2_12_PROFILES §$PUBLISH_PROFILES clean install
 
-  # Clean-up Zinc nailgun process
-  $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
   #./dev/change-scala-version.sh 2.11
 
   pushd $tmp_repo/org/apache/spark

http://git-wip-us.apache.org/repos/asf/spark/blob/d7a35877/dev/run-tests.py
--
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 7e6f7ff..f86b122 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -249,15 +249,6 @@ def get_zinc_port():
 return random.randrange(3030, 4030)
 
 
-def kill_zinc_on_port(zinc_port):
-"""
-Kill the Zinc process running on the given port, if one exists.
-"""
-cmd = "%s -P |grep %s | grep LISTEN | awk '{ print $2; }' | xargs kill"
-lsof_exe = which("lsof")
-subprocess.check_call(cmd % (lsof_exe if lsof_exe else "/usr/sbin/lsof", 
zinc_port), shell=True)
-
-
 def exec_maven(mvn_args=()):
 """Will call Maven in the current directory with the list of mvn_args 
passed
 in and returns the subprocess for any further

spark git commit: fix security issue of zinc(simplier version)

2018-10-21 Thread wenchen 
Repository: spark
Updated Branches:
  refs/heads/branch-2.4 0239277dd -> c21d7e1bb


fix security issue of zinc(simplier version)


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c21d7e1b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c21d7e1b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c21d7e1b

Branch: refs/heads/branch-2.4
Commit: c21d7e1bb958a0cfa4cba34a688d594466088c9e
Parents: 0239277
Author: Wenchen Fan 
Authored: Fri Oct 19 23:54:15 2018 +0800
Committer: Wenchen Fan 
Committed: Mon Oct 22 12:19:24 2018 +0800

--
 build/mvn   | 33 ++--
 dev/create-release/release-build.sh |  6 --
 dev/run-tests.py| 10 --
 3 files changed, 10 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/c21d7e1b/build/mvn
--
diff --git a/build/mvn b/build/mvn
index 0289ef3..b60ea64 100755
--- a/build/mvn
+++ b/build/mvn
@@ -139,17 +139,8 @@ if [ "$1" == "--force" ]; then
   shift
 fi
 
-if [ "$1" == "--zinc" ]; then
-  echo "Using zinc for incremental compilation. Be sure you are aware of the 
implications of "
-  echo "running this server process on your machine"
-  USE_ZINC=1
-  shift
-fi
-
 # Install the proper version of Scala, Zinc and Maven for the build
-if [ -n "${USE_ZINC}" ]; then
-  install_zinc
-fi
+install_zinc
 install_scala
 install_mvn
 
@@ -158,15 +149,13 @@ cd "${_CALLING_DIR}"
 
 # Now that zinc is ensured to be installed, check its status and, if its
 # not running or just installed, start it
-if [ -n "${USE_ZINC}" ]; then
-  if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port 
${ZINC_PORT}`" ]; then
-export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
-"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-"${ZINC_BIN}" -start -port ${ZINC_PORT} -server 127.0.0.1 \
-  -idle-timeout 30m \
-  -scala-compiler "${SCALA_COMPILER}" \
-  -scala-library "${SCALA_LIBRARY}" &>/dev/null
-  fi
+if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port 
${ZINC_PORT}`" ]; then
+  export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
+  "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
+  "${ZINC_BIN}" -start -port ${ZINC_PORT} \
+-server 127.0.0.1 -idle-timeout 30m \
+-scala-compiler "${SCALA_COMPILER}" \
+-scala-library "${SCALA_LIBRARY}" &>/dev/null
 fi
 
 # Set any `mvn` options if not already present
@@ -177,7 +166,5 @@ echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
 # Last, call the `mvn` command as usual
 "${MVN_BIN}" -DzincPort=${ZINC_PORT} "$@"
 
-if [ -n "${USE_ZINC}" ]; then
-  # Try to shut down zinc explicitly
-  "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-fi
+# Try to shut down zinc explicitly
+"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}

http://git-wip-us.apache.org/repos/asf/spark/blob/c21d7e1b/dev/create-release/release-build.sh
--
diff --git a/dev/create-release/release-build.sh 
b/dev/create-release/release-build.sh
index cce5f8b..02c4193 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -386,9 +386,6 @@ if [[ "$1" == "publish-snapshot" ]]; then
   #$MVN -DzincPort=$ZINC_PORT --settings $tmp_settings \
   #  -DskipTests $SCALA_2_12_PROFILES $PUBLISH_PROFILES clean deploy
 
-  # Clean-up Zinc nailgun process
-  $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
   rm $tmp_settings
   cd ..
   exit 0
@@ -433,9 +430,6 @@ if [[ "$1" == "publish-release" ]]; then
   -DskipTests $PUBLISH_PROFILES $SCALA_2_12_PROFILES clean install
   fi
 
-  # Clean-up Zinc nailgun process
-  $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
   ./dev/change-scala-version.sh 2.11
 
   pushd $tmp_repo/org/apache/spark

http://git-wip-us.apache.org/repos/asf/spark/blob/c21d7e1b/dev/run-tests.py
--
diff --git a/dev/run-tests.py b/dev/run-tests.py
index f534637..65e6f29 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -249,15 +249,6 @@ def get_zinc_port():
 return random.randrange(3030, 4030)
 
 
-def kill_zinc_on_port(zinc_port):
-"""
-Kill the Zinc process running on the given port, if one exists.
-"""
-cmd = "%s -P |grep %s | grep LISTEN | awk '{ print $2; }' | xargs kill"
-lsof_exe = which("lsof")
-subprocess.check_call(cmd % (lsof_exe if lsof_exe else "/usr/sbin/lsof", 
zinc_port), shell=True)
-
-
 def exec_maven(mvn_args=()):
 """Will call Maven in the current directory with the list of mvn_args 
passed
 in and returns the subprocess for any further processing"""
@@ -267,7 +258,6 @@ def