svn commit: r30200 - in /dev/spark/2.4.1-SNAPSHOT-2018_10_21_22_10-c21d7e1-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Mon Oct 22 05:27:30 2018 New Revision: 30200 Log: Apache Spark 2.4.1-SNAPSHOT-2018_10_21_22_10-c21d7e1 docs [This commit notification would consist of 1477 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: fix security issue of zinc(simplier version)
Repository: spark
Updated Branches:
refs/heads/branch-2.2 ca950c17f -> 8906696ac
fix security issue of zinc(simplier version)
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8906696a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8906696a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8906696a
Branch: refs/heads/branch-2.2
Commit: 8906696ac2089f3d6500b0496af7d9995c7de99b
Parents: ca950c1
Author: Wenchen Fan
Authored: Fri Oct 19 23:54:15 2018 +0800
Committer: Wenchen Fan
Committed: Mon Oct 22 12:24:31 2018 +0800
--
build/mvn | 33 ++--
dev/create-release/release-build.sh | 6 --
dev/run-tests.py| 10 --
3 files changed, 10 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/8906696a/build/mvn
--
diff --git a/build/mvn b/build/mvn
index 926027e..6a7d4db 100755
--- a/build/mvn
+++ b/build/mvn
@@ -130,17 +130,8 @@ if [ "$1" == "--force" ]; then
shift
fi
-if [ "$1" == "--zinc" ]; then
- echo "Using zinc for incremental compilation. Be sure you are aware of the
implications of "
- echo "running this server process on your machine"
- USE_ZINC=1
- shift
-fi
-
# Install the proper version of Scala, Zinc and Maven for the build
-if [ -n "${USE_ZINC}" ]; then
- install_zinc
-fi
+install_zinc
install_scala
install_mvn
@@ -149,15 +140,13 @@ cd "${_CALLING_DIR}"
# Now that zinc is ensured to be installed, check its status and, if its
# not running or just installed, start it
-if [ -n "${USE_ZINC}" ]; then
- if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port
${ZINC_PORT}`" ]; then
-export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
-"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-"${ZINC_BIN}" -start -port ${ZINC_PORT} -server 127.0.0.1 \
- -idle-timeout 30m \
- -scala-compiler "${SCALA_COMPILER}" \
- -scala-library "${SCALA_LIBRARY}" &>/dev/null
- fi
+if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port
${ZINC_PORT}`" ]; then
+ export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
+ "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
+ "${ZINC_BIN}" -start -port ${ZINC_PORT} \
+-server 127.0.0.1 -idle-timeout 30m \
+-scala-compiler "${SCALA_COMPILER}" \
+-scala-library "${SCALA_LIBRARY}" &>/dev/null
fi
# Set any `mvn` options if not already present
@@ -168,7 +157,5 @@ echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
# Last, call the `mvn` command as usual
${MVN_BIN} -DzincPort=${ZINC_PORT} "$@"
-if [ -n "${USE_ZINC}" ]; then
- # Try to shut down zinc explicitly
- "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-fi
+# Try to shut down zinc explicitly
+"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
http://git-wip-us.apache.org/repos/asf/spark/blob/8906696a/dev/create-release/release-build.sh
--
diff --git a/dev/create-release/release-build.sh
b/dev/create-release/release-build.sh
index 1272b6d..b4d27ed 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -343,9 +343,6 @@ if [[ "$1" == "publish-snapshot" ]]; then
$MVN -DzincPort=$ZINC_PORT -Dscala-2.10 --settings $tmp_settings \
-DskipTests $PUBLISH_PROFILES clean deploy
- # Clean-up Zinc nailgun process
- $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
rm $tmp_settings
cd ..
exit 0
@@ -381,9 +378,6 @@ if [[ "$1" == "publish-release" ]]; then
$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.10 \
-DskipTests $PUBLISH_PROFILES clean install
- # Clean-up Zinc nailgun process
- $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
./dev/change-version-to-2.10.sh
pushd $tmp_repo/org/apache/spark
http://git-wip-us.apache.org/repos/asf/spark/blob/8906696a/dev/run-tests.py
--
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 72d148d..0a399ab 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -249,15 +249,6 @@ def get_zinc_port():
return random.randrange(3030, 4030)
-def kill_zinc_on_port(zinc_port):
-"""
-Kill the Zinc process running on the given port, if one exists.
-"""
-cmd = ("/usr/sbin/lsof -P |grep %s | grep LISTEN "
- "| awk '{ print $2; }' | xargs kill") % zinc_port
-subprocess.check_call(cmd, shell=True)
-
-
def exec_maven(mvn_args=()):
"""Will call Maven in the current directory with the list of mvn_args
passed
in and returns the subprocess for any further processing"""
@@ -267,7 +258,6 @@ def exec_maven(mvn_args=()):
spark git commit: fix security issue of zinc(simplier version)
Repository: spark
Updated Branches:
refs/heads/branch-2.3 719ff7af6 -> d7a35877b
fix security issue of zinc(simplier version)
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d7a35877
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d7a35877
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d7a35877
Branch: refs/heads/branch-2.3
Commit: d7a35877b96dce8b742acf77e79bda189e402ae2
Parents: 719ff7a
Author: Wenchen Fan
Authored: Fri Oct 19 23:54:15 2018 +0800
Committer: Wenchen Fan
Committed: Mon Oct 22 12:22:10 2018 +0800
--
build/mvn | 33 ++--
dev/create-release/release-build.sh | 6 --
dev/run-tests.py| 10 --
3 files changed, 10 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d7a35877/build/mvn
--
diff --git a/build/mvn b/build/mvn
index 9c0d1a7..7951e10 100755
--- a/build/mvn
+++ b/build/mvn
@@ -130,17 +130,8 @@ if [ "$1" == "--force" ]; then
shift
fi
-if [ "$1" == "--zinc" ]; then
- echo "Using zinc for incremental compilation. Be sure you are aware of the
implications of "
- echo "running this server process on your machine"
- USE_ZINC=1
- shift
-fi
-
# Install the proper version of Scala, Zinc and Maven for the build
-if [ -n "${USE_ZINC}" ]; then
- install_zinc
-fi
+install_zinc
install_scala
install_mvn
@@ -149,15 +140,13 @@ cd "${_CALLING_DIR}"
# Now that zinc is ensured to be installed, check its status and, if its
# not running or just installed, start it
-if [ -n "${USE_ZINC}" ]; then
- if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port
${ZINC_PORT}`" ]; then
-export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
-"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-"${ZINC_BIN}" -start -port ${ZINC_PORT} -server 127.0.0.1 \
- -idle-timeout 30m \
- -scala-compiler "${SCALA_COMPILER}" \
- -scala-library "${SCALA_LIBRARY}" &>/dev/null
- fi
+if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port
${ZINC_PORT}`" ]; then
+ export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
+ "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
+ "${ZINC_BIN}" -start -port ${ZINC_PORT} \
+-server 127.0.0.1 -idle-timeout 30m \
+-scala-compiler "${SCALA_COMPILER}" \
+-scala-library "${SCALA_LIBRARY}" &>/dev/null
fi
# Set any `mvn` options if not already present
@@ -168,7 +157,5 @@ echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
# Last, call the `mvn` command as usual
${MVN_BIN} -DzincPort=${ZINC_PORT} "$@"
-if [ -n "${USE_ZINC}" ]; then
- # Try to shut down zinc explicitly
- "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-fi
+# Try to shut down zinc explicitly
+"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
http://git-wip-us.apache.org/repos/asf/spark/blob/d7a35877/dev/create-release/release-build.sh
--
diff --git a/dev/create-release/release-build.sh
b/dev/create-release/release-build.sh
index 5faa3d3..3e60641 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -323,9 +323,6 @@ if [[ "$1" == "publish-snapshot" ]]; then
#$MVN -DzincPort=$ZINC_PORT --settings $tmp_settings \
# -DskipTests $SCALA_2_12_PROFILES $PUBLISH_PROFILES clean deploy
- # Clean-up Zinc nailgun process
- $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
rm $tmp_settings
cd ..
exit 0
@@ -360,9 +357,6 @@ if [[ "$1" == "publish-release" ]]; then
#$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo \
# -DskipTests $SCALA_2_12_PROFILES §$PUBLISH_PROFILES clean install
- # Clean-up Zinc nailgun process
- $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
#./dev/change-scala-version.sh 2.11
pushd $tmp_repo/org/apache/spark
http://git-wip-us.apache.org/repos/asf/spark/blob/d7a35877/dev/run-tests.py
--
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 7e6f7ff..f86b122 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -249,15 +249,6 @@ def get_zinc_port():
return random.randrange(3030, 4030)
-def kill_zinc_on_port(zinc_port):
-"""
-Kill the Zinc process running on the given port, if one exists.
-"""
-cmd = "%s -P |grep %s | grep LISTEN | awk '{ print $2; }' | xargs kill"
-lsof_exe = which("lsof")
-subprocess.check_call(cmd % (lsof_exe if lsof_exe else "/usr/sbin/lsof",
zinc_port), shell=True)
-
-
def exec_maven(mvn_args=()):
"""Will call Maven in the current directory with the list of mvn_args
passed
in and returns the subprocess for any further
spark git commit: fix security issue of zinc(simplier version)
Repository: spark
Updated Branches:
refs/heads/branch-2.4 0239277dd -> c21d7e1bb
fix security issue of zinc(simplier version)
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c21d7e1b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c21d7e1b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c21d7e1b
Branch: refs/heads/branch-2.4
Commit: c21d7e1bb958a0cfa4cba34a688d594466088c9e
Parents: 0239277
Author: Wenchen Fan
Authored: Fri Oct 19 23:54:15 2018 +0800
Committer: Wenchen Fan
Committed: Mon Oct 22 12:19:24 2018 +0800
--
build/mvn | 33 ++--
dev/create-release/release-build.sh | 6 --
dev/run-tests.py| 10 --
3 files changed, 10 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/c21d7e1b/build/mvn
--
diff --git a/build/mvn b/build/mvn
index 0289ef3..b60ea64 100755
--- a/build/mvn
+++ b/build/mvn
@@ -139,17 +139,8 @@ if [ "$1" == "--force" ]; then
shift
fi
-if [ "$1" == "--zinc" ]; then
- echo "Using zinc for incremental compilation. Be sure you are aware of the
implications of "
- echo "running this server process on your machine"
- USE_ZINC=1
- shift
-fi
-
# Install the proper version of Scala, Zinc and Maven for the build
-if [ -n "${USE_ZINC}" ]; then
- install_zinc
-fi
+install_zinc
install_scala
install_mvn
@@ -158,15 +149,13 @@ cd "${_CALLING_DIR}"
# Now that zinc is ensured to be installed, check its status and, if its
# not running or just installed, start it
-if [ -n "${USE_ZINC}" ]; then
- if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port
${ZINC_PORT}`" ]; then
-export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
-"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-"${ZINC_BIN}" -start -port ${ZINC_PORT} -server 127.0.0.1 \
- -idle-timeout 30m \
- -scala-compiler "${SCALA_COMPILER}" \
- -scala-library "${SCALA_LIBRARY}" &>/dev/null
- fi
+if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`"${ZINC_BIN}" -status -port
${ZINC_PORT}`" ]; then
+ export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
+ "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
+ "${ZINC_BIN}" -start -port ${ZINC_PORT} \
+-server 127.0.0.1 -idle-timeout 30m \
+-scala-compiler "${SCALA_COMPILER}" \
+-scala-library "${SCALA_LIBRARY}" &>/dev/null
fi
# Set any `mvn` options if not already present
@@ -177,7 +166,5 @@ echo "Using \`mvn\` from path: $MVN_BIN" 1>&2
# Last, call the `mvn` command as usual
"${MVN_BIN}" -DzincPort=${ZINC_PORT} "$@"
-if [ -n "${USE_ZINC}" ]; then
- # Try to shut down zinc explicitly
- "${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
-fi
+# Try to shut down zinc explicitly
+"${ZINC_BIN}" -shutdown -port ${ZINC_PORT}
http://git-wip-us.apache.org/repos/asf/spark/blob/c21d7e1b/dev/create-release/release-build.sh
--
diff --git a/dev/create-release/release-build.sh
b/dev/create-release/release-build.sh
index cce5f8b..02c4193 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -386,9 +386,6 @@ if [[ "$1" == "publish-snapshot" ]]; then
#$MVN -DzincPort=$ZINC_PORT --settings $tmp_settings \
# -DskipTests $SCALA_2_12_PROFILES $PUBLISH_PROFILES clean deploy
- # Clean-up Zinc nailgun process
- $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
rm $tmp_settings
cd ..
exit 0
@@ -433,9 +430,6 @@ if [[ "$1" == "publish-release" ]]; then
-DskipTests $PUBLISH_PROFILES $SCALA_2_12_PROFILES clean install
fi
- # Clean-up Zinc nailgun process
- $LSOF -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
-
./dev/change-scala-version.sh 2.11
pushd $tmp_repo/org/apache/spark
http://git-wip-us.apache.org/repos/asf/spark/blob/c21d7e1b/dev/run-tests.py
--
diff --git a/dev/run-tests.py b/dev/run-tests.py
index f534637..65e6f29 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -249,15 +249,6 @@ def get_zinc_port():
return random.randrange(3030, 4030)
-def kill_zinc_on_port(zinc_port):
-"""
-Kill the Zinc process running on the given port, if one exists.
-"""
-cmd = "%s -P |grep %s | grep LISTEN | awk '{ print $2; }' | xargs kill"
-lsof_exe = which("lsof")
-subprocess.check_call(cmd % (lsof_exe if lsof_exe else "/usr/sbin/lsof",
zinc_port), shell=True)
-
-
def exec_maven(mvn_args=()):
"""Will call Maven in the current directory with the list of mvn_args
passed
in and returns the subprocess for any further processing"""
@@ -267,7 +258,6 @@ def
svn commit: r30197 - in /dev/spark/3.0.0-SNAPSHOT-2018_10_21_16_02-fc64e83-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Sun Oct 21 23:17:05 2018 New Revision: 30197 Log: Apache Spark 3.0.0-SNAPSHOT-2018_10_21_16_02-fc64e83 docs [This commit notification would consist of 1483 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-24207][R] add R API for PrefixSpan
Repository: spark
Updated Branches:
refs/heads/master 4c6c6711d -> fc64e83f9
[SPARK-24207][R] add R API for PrefixSpan
## What changes were proposed in this pull request?
add R API for PrefixSpan
## How was this patch tested?
add test in test_mllib_fpm.R
Author: Huaxin Gao
Closes #21710 from huaxingao/spark-24207.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fc64e83f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fc64e83f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fc64e83f
Branch: refs/heads/master
Commit: fc64e83f9538d6b7e13359a4933a454ba7ed89ec
Parents: 4c6c671
Author: Huaxin Gao
Authored: Sun Oct 21 12:32:43 2018 -0700
Committer: Felix Cheung
Committed: Sun Oct 21 12:32:43 2018 -0700
--
R/pkg/NAMESPACE | 3 +-
R/pkg/R/generics.R | 4 ++
R/pkg/R/mllib_fpm.R | 64 +++
R/pkg/tests/fulltests/test_mllib_fpm.R | 16 +
R/pkg/vignettes/sparkr-vignettes.Rmd| 13
docs/ml-frequent-pattern-mining.md | 53
docs/sparkr.md | 1 +
.../examples/ml/JavaPrefixSpanExample.java | 67
.../src/main/python/ml/prefixspan_example.py| 48 ++
examples/src/main/r/ml/prefixSpan.R | 42
.../spark/examples/ml/PrefixSpanExample.scala | 62 ++
.../apache/spark/ml/r/PrefixSpanWrapper.scala | 34 ++
12 files changed, 406 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/fc64e83f/R/pkg/NAMESPACE
--
diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE
index c512284..36d7a9b 100644
--- a/R/pkg/NAMESPACE
+++ b/R/pkg/NAMESPACE
@@ -70,7 +70,8 @@ exportMethods("glm",
"spark.svmLinear",
"spark.fpGrowth",
"spark.freqItemsets",
- "spark.associationRules")
+ "spark.associationRules",
+ "spark.findFrequentSequentialPatterns")
# Job group lifecycle management methods
export("setJobGroup",
http://git-wip-us.apache.org/repos/asf/spark/blob/fc64e83f/R/pkg/R/generics.R
--
diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R
index d501f73..045e075 100644
--- a/R/pkg/R/generics.R
+++ b/R/pkg/R/generics.R
@@ -1457,6 +1457,10 @@ setGeneric("spark.freqItemsets", function(object) {
standardGeneric("spark.freqI
#' @rdname spark.fpGrowth
setGeneric("spark.associationRules", function(object) {
standardGeneric("spark.associationRules") })
+#' @rdname spark.prefixSpan
+setGeneric("spark.findFrequentSequentialPatterns",
+function(data, ...) {
standardGeneric("spark.findFrequentSequentialPatterns") })
+
#' @param object a fitted ML model object.
#' @param path the directory where the model is saved.
#' @param ... additional argument(s) passed to the method.
http://git-wip-us.apache.org/repos/asf/spark/blob/fc64e83f/R/pkg/R/mllib_fpm.R
--
diff --git a/R/pkg/R/mllib_fpm.R b/R/pkg/R/mllib_fpm.R
index 4ad34fe..ac37580 100644
--- a/R/pkg/R/mllib_fpm.R
+++ b/R/pkg/R/mllib_fpm.R
@@ -23,6 +23,12 @@
#' @note FPGrowthModel since 2.2.0
setClass("FPGrowthModel", slots = list(jobj = "jobj"))
+#' S4 class that represents a PrefixSpan
+#'
+#' @param jobj a Java object reference to the backing Scala PrefixSpan
+#' @note PrefixSpan since 3.0.0
+setClass("PrefixSpan", slots = list(jobj = "jobj"))
+
#' FP-growth
#'
#' A parallel FP-growth algorithm to mine frequent itemsets.
@@ -155,3 +161,61 @@ setMethod("write.ml", signature(object = "FPGrowthModel",
path = "character"),
function(object, path, overwrite = FALSE) {
write_internal(object, path, overwrite)
})
+
+#' PrefixSpan
+#'
+#' A parallel PrefixSpan algorithm to mine frequent sequential patterns.
+#' \code{spark.findFrequentSequentialPatterns} returns a complete set of
frequent sequential
+#' patterns.
+#' For more details, see
+#'
\href{https://spark.apache.org/docs/latest/mllib-frequent-pattern-mining.html#prefixspan}{
+#' PrefixSpan}.
+#'
+# Find frequent sequential patterns.
+#' @param data A SparkDataFrame.
+#' @param minSupport Minimal support level.
+#' @param maxPatternLength Maximal pattern length.
+#' @param maxLocalProjDBSize Maximum number of items (including delimiters
used in the internal
+#' storage format) allowed in a projected database
before local
+#' processing.
+#' @param sequenceCol name of the sequence column in dataset.
+#' @param ...
svn commit: r30196 - in /dev/spark/3.0.0-SNAPSHOT-2018_10_21_12_02-4c6c671-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Sun Oct 21 19:16:51 2018 New Revision: 30196 Log: Apache Spark 3.0.0-SNAPSHOT-2018_10_21_12_02-4c6c671 docs [This commit notification would consist of 1483 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-25675][SPARK JOB HISTORY] Job UI page does not show pagination with one page
Repository: spark
Updated Branches:
refs/heads/master ffe256ce1 -> 4c6c6711d
[SPARK-25675][SPARK JOB HISTORY] Job UI page does not show pagination with one
page
## What changes were proposed in this pull request?
Currently in PagedTable.scala pageNavigation() method, if it is having only one
page, they were not using the pagination.
Now it is made to use the pagination, even if it is having one page.
## How was this patch tested?
This tested with Spark webUI and History page in spark local setup.
Author: shivusondur
Closes #22668 from shivusondur/pagination.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4c6c6711
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4c6c6711
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4c6c6711
Branch: refs/heads/master
Commit: 4c6c6711d5e94378c7bb5f672314f4db751372ea
Parents: ffe256c
Author: shivusondur
Authored: Sun Oct 21 11:44:48 2018 -0700
Committer: Felix Cheung
Committed: Sun Oct 21 11:44:48 2018 -0700
--
.../scala/org/apache/spark/ui/PagedTable.scala | 210 +--
1 file changed, 101 insertions(+), 109 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/4c6c6711/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
--
diff --git a/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
b/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
index 2fc0259..0bbb10a 100644
--- a/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
@@ -122,13 +122,9 @@ private[spark] trait PagedTable[T] {
/**
* Return a page navigation.
- *
- * If the totalPages is 1, the page navigation will be empty
- *
- * If the totalPages is more than 1, it will create a page navigation
including a group of
- * page numbers and a form to submit the page number.
- *
- *
+ *
+ * It will create a page navigation including a group of page numbers and a
form
+ * to submit the page number.
*
* Here are some examples of the page navigation:
* {{{
@@ -154,120 +150,116 @@ private[spark] trait PagedTable[T] {
* }}}
*/
private[ui] def pageNavigation(page: Int, pageSize: Int, totalPages: Int):
Seq[Node] = {
-if (totalPages == 1) {
- Nil
-} else {
- // A group includes all page numbers will be shown in the page
navigation.
- // The size of group is 10 means there are 10 page numbers will be shown.
- // The first group is 1 to 10, the second is 2 to 20, and so on
- val groupSize = 10
- val firstGroup = 0
- val lastGroup = (totalPages - 1) / groupSize
- val currentGroup = (page - 1) / groupSize
- val startPage = currentGroup * groupSize + 1
- val endPage = totalPages.min(startPage + groupSize - 1)
- val pageTags = (startPage to endPage).map { p =>
-if (p == page) {
- // The current page should be disabled so that it cannot be clicked.
- {p}
-} else {
- {p}
-}
+// A group includes all page numbers will be shown in the page navigation.
+// The size of group is 10 means there are 10 page numbers will be shown.
+// The first group is 1 to 10, the second is 2 to 20, and so on
+val groupSize = 10
+val firstGroup = 0
+val lastGroup = (totalPages - 1) / groupSize
+val currentGroup = (page - 1) / groupSize
+val startPage = currentGroup * groupSize + 1
+val endPage = totalPages.min(startPage + groupSize - 1)
+val pageTags = (startPage to endPage).map { p =>
+ if (p == page) {
+// The current page should be disabled so that it cannot be clicked.
+{p}
+ } else {
+{p}
}
+}
- val hiddenFormFields = {
-if (goButtonFormPath.contains('?')) {
- val queryString = goButtonFormPath.split("\\?", 2)(1)
- val search = queryString.split("#")(0)
- Splitter
-.on('&')
-.trimResults()
-.omitEmptyStrings()
-.withKeyValueSeparator("=")
-.split(search)
-.asScala
-.filterKeys(_ != pageSizeFormField)
-.filterKeys(_ != prevPageSizeFormField)
-.filterKeys(_ != pageNumberFormField)
-.mapValues(URLDecoder.decode(_, "UTF-8"))
-.map { case (k, v) =>
-
-}
-} else {
- Seq.empty
-}
+val hiddenFormFields = {
+ if (goButtonFormPath.contains('?')) {
+val queryString = goButtonFormPath.split("\\?", 2)(1)
+val search =
spark git commit: [SPARK-25730][K8S] Delete executor pods from kubernetes after figuring out why they died
Repository: spark
Updated Branches:
refs/heads/master c77aa42f5 -> ffe256ce1
[SPARK-25730][K8S] Delete executor pods from kubernetes after figuring out why
they died
## What changes were proposed in this pull request?
`removeExecutorFromSpark` tries to fetch the reason the executor exited from
Kubernetes, which may be useful if the pod was OOMKilled. However, the code
previously deleted the pod from Kubernetes first which made retrieving this
status impossible. This fixes the ordering.
On a separate but related note, it would be nice to wait some time before
removing the pod - to let the operator examine logs and such.
## How was this patch tested?
Running on my local cluster.
Author: Mike Kaplinskiy
Closes #22720 from mikekap/patch-1.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ffe256ce
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ffe256ce
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ffe256ce
Branch: refs/heads/master
Commit: ffe256ce161884f0a1304b4925d51d39a9bfa5df
Parents: c77aa42
Author: Mike Kaplinskiy
Authored: Sun Oct 21 11:32:33 2018 -0700
Committer: Felix Cheung
Committed: Sun Oct 21 11:32:33 2018 -0700
--
.../spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ffe256ce/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
--
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
index cc254b8..1a75ae0 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
@@ -112,8 +112,8 @@ private[spark] class ExecutorPodsLifecycleManager(
execId: Long,
schedulerBackend: KubernetesClusterSchedulerBackend,
execIdsRemovedInRound: mutable.Set[Long]): Unit = {
-removeExecutorFromK8s(podState.pod)
removeExecutorFromSpark(schedulerBackend, podState, execId)
+removeExecutorFromK8s(podState.pod)
execIdsRemovedInRound += execId
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
svn commit: r30193 - in /dev/spark/3.0.0-SNAPSHOT-2018_10_21_00_02-c77aa42-docs: ./ _site/ _site/api/ _site/api/R/ _site/api/java/ _site/api/java/lib/ _site/api/java/org/ _site/api/java/org/apache/ _s
Author: pwendell Date: Sun Oct 21 07:16:41 2018 New Revision: 30193 Log: Apache Spark 3.0.0-SNAPSHOT-2018_10_21_00_02-c77aa42 docs [This commit notification would consist of 1483 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.] - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
