spark git commit: BUILD: Minor tweaks to internal build scripts
Repository: spark
Updated Branches:
refs/heads/branch-1.3 1aa846165 - ae60eb998
BUILD: Minor tweaks to internal build scripts
This adds two features:
1. The ability to publish with a different maven version than
that specified in the release source.
2. Forking of different Zinc instances during the parallel dist
creation (to help with some stability issues).
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ae60eb99
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ae60eb99
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ae60eb99
Branch: refs/heads/branch-1.3
Commit: ae60eb9984de56c36c7f63220f9281fbaac10931
Parents: 1aa8461
Author: Patrick Wendell patr...@databricks.com
Authored: Tue Mar 3 00:38:12 2015 -0800
Committer: Patrick Wendell patr...@databricks.com
Committed: Tue Mar 3 01:54:06 2015 -0800
--
dev/create-release/create-release.sh | 24 +++-
1 file changed, 19 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ae60eb99/dev/create-release/create-release.sh
--
diff --git a/dev/create-release/create-release.sh
b/dev/create-release/create-release.sh
index da15ce3..6f87fcd 100755
--- a/dev/create-release/create-release.sh
+++ b/dev/create-release/create-release.sh
@@ -34,6 +34,9 @@ ASF_PASSWORD=${ASF_PASSWORD:-XXX}
GPG_PASSPHRASE=${GPG_PASSPHRASE:-XXX}
GIT_BRANCH=${GIT_BRANCH:-branch-1.0}
RELEASE_VERSION=${RELEASE_VERSION:-1.2.0}
+# Allows publishing under a different version identifier than
+# was present in the actual release sources (e.g. rc-X)
+PUBLISH_VERSION=${PUBLISH_VERSION:-$RELEASE_VERSION}
NEXT_VERSION=${NEXT_VERSION:-1.2.1}
RC_NAME=${RC_NAME:-rc2}
@@ -97,30 +100,35 @@ if [[ ! $@ =~ --skip-publish ]]; then
pushd spark
git checkout --force $GIT_TAG
+ # Substitute in case published version is different than released
+ old=^\( \{2,4\}\)version${RELEASE_VERSION}\/version$
+ new=\1version${PUBLISH_VERSION}\/version
+ find . -name pom.xml | grep -v dev | xargs -I {} sed -i \
+-e s/${old}/${new}/ {}
+
# Using Nexus API documented here:
#
https://support.sonatype.com/entries/39720203-Uploading-to-a-Staging-Repository-via-REST-API
echo Creating Nexus staging repository
- repo_request=promoteRequestdatadescriptionApache Spark
$GIT_TAG/description/data/promoteRequest
+ repo_request=promoteRequestdatadescriptionApache Spark $GIT_TAG
(published as $PUBLISH_VERSION)/description/data/promoteRequest
out=$(curl -X POST -d $repo_request -u $ASF_USERNAME:$ASF_PASSWORD \
-H Content-Type:application/xml -v \
$NEXUS_ROOT/profiles/$NEXUS_PROFILE/start)
staged_repo_id=$(echo $out | sed -e
s/.*\(orgapachespark-[0-9]\{4\}\).*/\1/)
echo Created Nexus staging repository: $staged_repo_id
- rm -rf $SPARK_REPO
-
- mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
+ build/mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
-Pyarn -Phive -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \
clean install
./dev/change-version-to-2.11.sh
- mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
+ build/mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
-Dscala-2.11 -Pyarn -Phive -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl
\
clean install
./dev/change-version-to-2.10.sh
+ rm -rf $SPARK_REPO
pushd $SPARK_REPO
# Remove any extra files generated during install
@@ -197,6 +205,12 @@ if [[ ! $@ =~ --skip-package ]]; then
./dev/change-version-to-2.11.sh
fi
+# Create new Zinc instances for each binary release to avoid interference
+# that causes OOM's and random compiler crashes.
+zinc_port=${zinc_port:-3030}
+zinc_port=$[$zinc_port + 1]
+export ZINC_PORT=$zinc_port
+
./make-distribution.sh --name $NAME --tgz $FLAGS 21 | tee
../binary-release-$NAME.log
cd ..
cp spark-$RELEASE_VERSION-bin-$NAME/spark-$RELEASE_VERSION-bin-$NAME.tgz .
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[1/3] spark git commit: Adding CHANGES.txt for Spark 1.3
Repository: spark Updated Branches: refs/heads/branch-1.3 ae60eb998 - ce7158cf7 http://git-wip-us.apache.org/repos/asf/spark/blob/ce7158cf/dev/create-release/generate-changelist.py -- diff --git a/dev/create-release/generate-changelist.py b/dev/create-release/generate-changelist.py index 2e1a35a..2a2e4b7 100755 --- a/dev/create-release/generate-changelist.py +++ b/dev/create-release/generate-changelist.py @@ -31,8 +31,8 @@ import time import traceback SPARK_HOME = os.environ[SPARK_HOME] -NEW_RELEASE_VERSION = 1.0.0 -PREV_RELEASE_GIT_TAG = v0.9.1 +NEW_RELEASE_VERSION = 1.3.0 +PREV_RELEASE_GIT_TAG = v1.2.1 CHANGELIST = CHANGES.txt OLD_CHANGELIST = %s.old % (CHANGELIST) - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[2/3] spark git commit: Adding CHANGES.txt for Spark 1.3
http://git-wip-us.apache.org/repos/asf/spark/blob/ce7158cf/CHANGES.txt -- diff --git a/CHANGES.txt b/CHANGES.txt new file mode 100644 index 000..46d612a --- /dev/null +++ b/CHANGES.txt @@ -0,0 +1,6520 @@ +Spark Change Log + + +Release 1.3.0 + + BUILD: Minor tweaks to internal build scripts + Patrick Wendell patr...@databricks.com + 2015-03-03 00:38:12 -0800 + Commit: ae60eb9 + + HOTFIX: Bump HBase version in MapR profiles. + Patrick Wendell patr...@databricks.com + 2015-03-03 01:38:07 -0800 + Commit: 1aa8461 + + [SPARK-5537][MLlib][Docs] Add user guide for multinomial logistic regression + DB Tsai dbt...@alpinenow.com + 2015-03-02 22:37:12 -0800 + Commit: 841d2a2, github.com/apache/spark/pull/4866 + + [SPARK-6120] [mllib] Warnings about memory in tree, ensemble model save + Joseph K. Bradley jos...@databricks.com + 2015-03-02 22:33:51 -0800 + Commit: 81648a7, github.com/apache/spark/pull/4864 + + [SPARK-6097][MLLIB] Support tree model save/load in PySpark/MLlib + Xiangrui Meng m...@databricks.com + 2015-03-02 22:27:01 -0800 + Commit: 62c53be, github.com/apache/spark/pull/4854 + + [SPARK-5310][SQL] Fixes to Docs and Datasources API + Reynold Xin r...@databricks.com, Michael Armbrust mich...@databricks.com + 2015-03-02 22:14:08 -0800 + Commit: 4e6e008, github.com/apache/spark/pull/4868 + + [SPARK-5950][SQL]Insert array into a metastore table saved as parquet should work when using datasource api + Yin Huai yh...@databricks.com + 2015-03-02 19:31:55 -0800 + Commit: 1b490e9, github.com/apache/spark/pull/4826 + + [SPARK-6127][Streaming][Docs] Add Kafka to Python api docs + Tathagata Das tathagata.das1...@gmail.com + 2015-03-02 18:40:46 -0800 + Commit: ffd0591, github.com/apache/spark/pull/4860 + + [SPARK-5537] Add user guide for multinomial logistic regression + Xiangrui Meng m...@databricks.com, DB Tsai dbt...@alpinenow.com + 2015-03-02 18:10:50 -0800 + Commit: 11389f0, github.com/apache/spark/pull/4801 + + [SPARK-6121][SQL][MLLIB] simpleString for UDT + Xiangrui Meng m...@databricks.com + 2015-03-02 17:14:34 -0800 + Commit: 1b8ab57, github.com/apache/spark/pull/4858 + + [SPARK-6048] SparkConf should not translate deprecated configs on set + Andrew Or and...@databricks.com + 2015-03-02 16:36:42 -0800 + Commit: ea69cf2, github.com/apache/spark/pull/4799 + + [SPARK-6066] Make event log format easier to parse + Andrew Or and...@databricks.com + 2015-03-02 16:34:32 -0800 + Commit: 8100b79, github.com/apache/spark/pull/4821 + + [SPARK-6082] [SQL] Provides better error message for malformed rows when caching tables + Cheng Lian l...@databricks.com + 2015-03-02 16:18:00 -0800 + Commit: 866f281, github.com/apache/spark/pull/4842 + + [SPARK-6114][SQL] Avoid metastore conversions before plan is resolved + Michael Armbrust mich...@databricks.com + 2015-03-02 16:10:54 -0800 + Commit: 3899c7c, github.com/apache/spark/pull/4855 + + [SPARK-6050] [yarn] Relax matching of vcore count in received containers. + Marcelo Vanzin van...@cloudera.com + 2015-03-02 16:41:43 -0600 + Commit: 650d1e7, github.com/apache/spark/pull/4818 + + [SPARK-6040][SQL] Fix the percent bug in tablesample + q00251598 qiyad...@huawei.com + 2015-03-02 13:16:29 -0800 + Commit: a83b9bb, github.com/apache/spark/pull/4789 + + [Minor] Fix doc typo for describing primitiveTerm effectiveness condition + Liang-Chi Hsieh vii...@gmail.com + 2015-03-02 13:11:17 -0800 + Commit: f92876a, github.com/apache/spark/pull/4762 + + SPARK-5390 [DOCS] Encourage users to post on Stack Overflow in Community Docs + Sean Owen so...@cloudera.com + 2015-03-02 21:10:08 + + Commit: 58e7198, github.com/apache/spark/pull/4843 + + [DOCS] Refactored Dataframe join comment to use correct parameter ordering + Paul Power paul.po...@peerside.com + 2015-03-02 13:08:47 -0800 + Commit: 54ac243, github.com/apache/spark/pull/4847 + + [SPARK-6080] [PySpark] correct LogisticRegressionWithLBFGS regType parameter for pyspark + Yanbo Liang yblia...@gmail.com + 2015-03-02 10:17:24 -0800 + Commit: 4ffaf85, github.com/apache/spark/pull/4831 + + [SPARK-5741][SQL] Support the path contains comma in HiveContext + q00251598 qiyad...@huawei.com + 2015-03-02 10:13:11 -0800 + Commit: f476108, github.com/apache/spark/pull/4532 + + [SPARK-6111] Fixed usage string in documentation. + Kenneth Myers myer...@us.ibm.com + 2015-03-02 17:25:24 + + Commit: b2b7f01, github.com/apache/spark/pull/4852 + + [SPARK-6052][SQL]In JSON schema inference, we should always set containsNull of an ArrayType to true + Yin Huai yh...@databricks.com + 2015-03-02 23:18:07 +0800 + Commit: a3fef2c, github.com/apache/spark/pull/4806 + + [SPARK-6073][SQL] Need to refresh metastore cache after append data in CreateMetastoreDataSourceAsSelect + Yin Huai yh...@databricks.com + 2015-03-02 22:42:18 +0800 + Commit: c59871c,
[3/3] spark git commit: Adding CHANGES.txt for Spark 1.3
Adding CHANGES.txt for Spark 1.3 Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ce7158cf Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ce7158cf Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ce7158cf Branch: refs/heads/branch-1.3 Commit: ce7158cf70c1003c1011d9a755813b31feae91e4 Parents: ae60eb9 Author: Patrick Wendell patr...@databricks.com Authored: Tue Mar 3 02:19:19 2015 -0800 Committer: Patrick Wendell patr...@databricks.com Committed: Tue Mar 3 02:19:19 2015 -0800 -- CHANGES.txt | 6520 dev/create-release/generate-changelist.py |4 +- 2 files changed, 6522 insertions(+), 2 deletions(-) -- - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[1/2] spark git commit: Preparing development version 1.3.1-SNAPSHOT
Repository: spark Updated Branches: refs/heads/branch-1.3 b012ed189 - 05d5a29eb Preparing development version 1.3.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/05d5a29e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/05d5a29e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/05d5a29e Branch: refs/heads/branch-1.3 Commit: 05d5a29eb3193aeb57d177bafe39eb75edce72a1 Parents: 3af2687 Author: Patrick Wendell patr...@databricks.com Authored: Tue Mar 3 10:23:07 2015 + Committer: Patrick Wendell patr...@databricks.com Committed: Tue Mar 3 10:23:07 2015 + -- assembly/pom.xml | 2 +- bagel/pom.xml | 2 +- core/pom.xml | 2 +- examples/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml| 2 +- external/kafka-assembly/pom.xml | 2 +- external/kafka/pom.xml| 2 +- external/mqtt/pom.xml | 2 +- external/twitter/pom.xml | 2 +- external/zeromq/pom.xml | 2 +- extras/java8-tests/pom.xml| 2 +- extras/kinesis-asl/pom.xml| 2 +- extras/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml| 2 +- mllib/pom.xml | 2 +- network/common/pom.xml| 2 +- network/shuffle/pom.xml | 2 +- network/yarn/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- yarn/pom.xml | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/05d5a29e/assembly/pom.xml -- diff --git a/assembly/pom.xml b/assembly/pom.xml index 0764d2d..87b3e6f 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.1-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/05d5a29e/bagel/pom.xml -- diff --git a/bagel/pom.xml b/bagel/pom.xml index 4f73cf7..4a13c58 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.1-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/05d5a29e/core/pom.xml -- diff --git a/core/pom.xml b/core/pom.xml index 5612149..aca0f58 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.1-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/05d5a29e/examples/pom.xml -- diff --git a/examples/pom.xml b/examples/pom.xml index f7d6030..c424592 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.1-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/05d5a29e/external/flume-sink/pom.xml -- diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 45aa775..ef960a8 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.1-SNAPSHOT/version relativePath../../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/05d5a29e/external/flume/pom.xml -- diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 455304f..f01d6e8 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.1-SNAPSHOT/version
svn commit: r1663693 - /spark/site/downloads.html
Author: srowen
Date: Tue Mar 3 13:04:20 2015
New Revision: 1663693
URL: http://svn.apache.org/r1663693
Log:
SPARK-5055: fix Chose - Choose typo
Modified:
spark/site/downloads.html
Modified: spark/site/downloads.html
URL:
http://svn.apache.org/viewvc/spark/site/downloads.html?rev=1663693r1=1663692r2=1663693view=diff
==
--- spark/site/downloads.html (original)
+++ spark/site/downloads.html Tue Mar 3 13:04:20 2015
@@ -182,15 +182,15 @@ $(document).ready(function() {
ol
li
-pChose a Spark release:
+pChoose a Spark release:
select id=sparkVersionSelect
onchange=javascript:onVersionSelect();/selectbr //p
/li
li
-pChose a package type:
+pChoose a package type:
select id=sparkPackageSelect
onchange=javascript:onPackageSelect();/selectbr //p
/li
li
-pChose a download type:
+pChoose a download type:
select id=sparkDownloadSelect
onchange=javascript:onDownloadSelect()/selectbr //p
/li
li
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: Revert [SPARK-5423][Core] Cleanup resources in DiskMapIterator.finalize to ensure deleting the temp file
Repository: spark
Updated Branches:
refs/heads/master e359794ce - 9af001749
Revert [SPARK-5423][Core] Cleanup resources in DiskMapIterator.finalize to
ensure deleting the temp file
This reverts commit 90095bf3ce9304d09a32ceffaa99069079071b59.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9af00174
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9af00174
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9af00174
Branch: refs/heads/master
Commit: 9af001749a37a86ccbf78063ec514a21801645fa
Parents: e359794
Author: Andrew Or and...@databricks.com
Authored: Tue Mar 3 13:03:52 2015 -0800
Committer: Andrew Or and...@databricks.com
Committed: Tue Mar 3 13:03:52 2015 -0800
--
.../util/collection/ExternalAppendOnlyMap.scala | 52
1 file changed, 9 insertions(+), 43 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/9af00174/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
index fc7e86e..8a0f5a6 100644
---
a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
+++
b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -387,15 +387,6 @@ class ExternalAppendOnlyMap[K, V, C](
private var batchIndex = 0 // Which batch we're in
private var fileStream: FileInputStream = null
-@volatile private var closed = false
-
-// A volatile variable to remember which DeserializationStream is using.
Need to set it when we
-// open a DeserializationStream. But we should use `deserializeStream`
rather than
-// `deserializeStreamToBeClosed` to read the content because touching a
volatile variable will
-// reduce the performance. It must be volatile so that we can see its
correct value in the
-// `finalize` method, which could run in any thread.
-@volatile private var deserializeStreamToBeClosed: DeserializationStream =
null
-
// An intermediate stream that reads from exactly one batch
// This guards against pre-fetching and other arbitrary behavior of higher
level streams
private var deserializeStream = nextBatchStream()
@@ -410,7 +401,6 @@ class ExternalAppendOnlyMap[K, V, C](
// we're still in a valid batch.
if (batchIndex batchOffsets.length - 1) {
if (deserializeStream != null) {
- deserializeStreamToBeClosed = null
deserializeStream.close()
fileStream.close()
deserializeStream = null
@@ -429,11 +419,7 @@ class ExternalAppendOnlyMap[K, V, C](
val bufferedStream = new
BufferedInputStream(ByteStreams.limit(fileStream, end - start))
val compressedStream = blockManager.wrapForCompression(blockId,
bufferedStream)
-// Before returning the stream, assign it to
`deserializeStreamToBeClosed` so that we can
-// close it in `finalize` and also avoid to touch the volatile
`deserializeStreamToBeClosed`
-// during reading the (K, C) pairs.
-deserializeStreamToBeClosed = ser.deserializeStream(compressedStream)
-deserializeStreamToBeClosed
+ser.deserializeStream(compressedStream)
} else {
// No more batches left
cleanup()
@@ -482,34 +468,14 @@ class ExternalAppendOnlyMap[K, V, C](
item
}
-// TODO: Now only use `finalize` to ensure `close` gets called to clean up
the resources. In the
-// future, we need some mechanism to ensure this gets called once the
resources are not used.
-private def cleanup(): Unit = {
- if (!closed) {
-closed = true
-batchIndex = batchOffsets.length // Prevent reading any other batch
-fileStream = null
-try {
- val ds = deserializeStreamToBeClosed
- deserializeStreamToBeClosed = null
- deserializeStream = null
- if (ds != null) {
-ds.close()
- }
-} finally {
- if (file.exists()) {
-file.delete()
- }
-}
- }
-}
-
-override def finalize(): Unit = {
- try {
-cleanup()
- } finally {
-super.finalize()
- }
+// TODO: Ensure this gets called even if the iterator isn't drained.
+private def cleanup() {
+ batchIndex = batchOffsets.length // Prevent reading any other batch
+ val ds = deserializeStream
+ deserializeStream = null
+ fileStream = null
+ ds.close()
+ file.delete()
}
}
-
To
spark git commit: Revert [SPARK-5423][Core] Cleanup resources in DiskMapIterator.finalize to ensure deleting the temp file
Repository: spark
Updated Branches:
refs/heads/branch-1.1 91d0effb3 - d70754df0
Revert [SPARK-5423][Core] Cleanup resources in DiskMapIterator.finalize to
ensure deleting the temp file
This reverts commit 36f3c499fd1ad53a68a084d6a16a2c68099e7049.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d70754df
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d70754df
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d70754df
Branch: refs/heads/branch-1.1
Commit: d70754df04031a8bd99e2aa104303e98e358128e
Parents: 91d0eff
Author: Andrew Or and...@databricks.com
Authored: Tue Mar 3 13:05:13 2015 -0800
Committer: Andrew Or and...@databricks.com
Committed: Tue Mar 3 13:05:13 2015 -0800
--
.../util/collection/ExternalAppendOnlyMap.scala | 52
1 file changed, 9 insertions(+), 43 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d70754df/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
index abb1f11..5619b30 100644
---
a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
+++
b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -434,15 +434,6 @@ class ExternalAppendOnlyMap[K, V, C](
private var batchIndex = 0 // Which batch we're in
private var fileStream: FileInputStream = null
-@volatile private var closed = false
-
-// A volatile variable to remember which DeserializationStream is using.
Need to set it when we
-// open a DeserializationStream. But we should use `deserializeStream`
rather than
-// `deserializeStreamToBeClosed` to read the content because touching a
volatile variable will
-// reduce the performance. It must be volatile so that we can see its
correct value in the
-// `finalize` method, which could run in any thread.
-@volatile private var deserializeStreamToBeClosed: DeserializationStream =
null
-
// An intermediate stream that reads from exactly one batch
// This guards against pre-fetching and other arbitrary behavior of higher
level streams
private var deserializeStream = nextBatchStream()
@@ -457,7 +448,6 @@ class ExternalAppendOnlyMap[K, V, C](
// we're still in a valid batch.
if (batchIndex batchOffsets.length - 1) {
if (deserializeStream != null) {
- deserializeStreamToBeClosed = null
deserializeStream.close()
fileStream.close()
deserializeStream = null
@@ -476,11 +466,7 @@ class ExternalAppendOnlyMap[K, V, C](
val bufferedStream = new
BufferedInputStream(ByteStreams.limit(fileStream, end - start))
val compressedStream = blockManager.wrapForCompression(blockId,
bufferedStream)
-// Before returning the stream, assign it to
`deserializeStreamToBeClosed` so that we can
-// close it in `finalize` and also avoid to touch the volatile
`deserializeStreamToBeClosed`
-// during reading the (K, C) pairs.
-deserializeStreamToBeClosed = ser.deserializeStream(compressedStream)
-deserializeStreamToBeClosed
+ser.deserializeStream(compressedStream)
} else {
// No more batches left
cleanup()
@@ -529,34 +515,14 @@ class ExternalAppendOnlyMap[K, V, C](
item
}
-// TODO: Now only use `finalize` to ensure `close` gets called to clean up
the resources. In the
-// future, we need some mechanism to ensure this gets called once the
resources are not used.
-private def cleanup(): Unit = {
- if (!closed) {
-closed = true
-batchIndex = batchOffsets.length // Prevent reading any other batch
-fileStream = null
-try {
- val ds = deserializeStreamToBeClosed
- deserializeStreamToBeClosed = null
- deserializeStream = null
- if (ds != null) {
-ds.close()
- }
-} finally {
- if (file.exists()) {
-file.delete()
- }
-}
- }
-}
-
-override def finalize(): Unit = {
- try {
-cleanup()
- } finally {
-super.finalize()
- }
+// TODO: Ensure this gets called even if the iterator isn't drained.
+private def cleanup() {
+ batchIndex = batchOffsets.length // Prevent reading any other batch
+ val ds = deserializeStream
+ deserializeStream = null
+ fileStream = null
+ ds.close()
+ file.delete()
}
}
-
spark git commit: SPARK-1911 [DOCS] Warn users if their assembly jars are not built with Java 6
Repository: spark Updated Branches: refs/heads/master 9af001749 - e750a6bfd SPARK-1911 [DOCS] Warn users if their assembly jars are not built with Java 6 Add warning about building with Java 7+ and running the JAR on early Java 6. CC andrewor14 Author: Sean Owen so...@cloudera.com Closes #4874 from srowen/SPARK-1911 and squashes the following commits: 79fa2f6 [Sean Owen] Add warning about building with Java 7+ and running the JAR on early Java 6. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e750a6bf Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e750a6bf Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e750a6bf Branch: refs/heads/master Commit: e750a6bfddf1d7bf7d3e99a424ec2b83a18b40d9 Parents: 9af0017 Author: Sean Owen so...@cloudera.com Authored: Tue Mar 3 13:40:11 2015 -0800 Committer: Andrew Or and...@databricks.com Committed: Tue Mar 3 13:40:11 2015 -0800 -- docs/building-spark.md | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/e750a6bf/docs/building-spark.md -- diff --git a/docs/building-spark.md b/docs/building-spark.md index 4c3988e..57d0ca8 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -9,6 +9,10 @@ redirect_from: building-with-maven.html Building Spark using Maven requires Maven 3.0.4 or newer and Java 6+. +**Note:** Building Spark with Java 7 or later can create JAR files that may not be +readable with early versions of Java 6, due to the large number of files in the JAR +archive. Build with Java 6 if this is an issue for your deployment. + # Building with `build/mvn` Spark now comes packaged with a self-contained Maven installation to ease building and deployment of Spark from source located under the `build/` directory. This script will automatically download and setup all necessary build requirements ([Maven](https://maven.apache.org/), [Scala](http://www.scala-lang.org/), and [Zinc](https://github.com/typesafehub/zinc)) locally within the `build/` directory itself. It honors any `mvn` binary if present already, however, will pull down its own copy of Scala and Zinc regardless to ensure proper version requirements are met. `build/mvn` execution acts as a pass through to the `mvn` call allowing easy transition from previous build methods. As an example, one can build a version of Spark as follows: - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-6132] ContextCleaner race condition across SparkContexts
Repository: spark
Updated Branches:
refs/heads/master e750a6bfd - fe63e8229
[SPARK-6132] ContextCleaner race condition across SparkContexts
The problem is that `ContextCleaner` may clean variables that belong to a
different `SparkContext`. This can happen if the `SparkContext` to which the
cleaner belongs stops, and a new one is started immediately afterwards in the
same JVM. In this case, if the cleaner is in the middle of cleaning a
broadcast, for instance, it will do so through `SparkEnv.get.blockManager`,
which could be one that belongs to a different `SparkContext`.
JoshRosen and I suspect that this is the cause of many flaky tests, most
notably the `JavaAPISuite`. We were able to reproduce the failure locally
(though it is not deterministic and very hard to reproduce).
Author: Andrew Or and...@databricks.com
Closes #4869 from andrewor14/cleaner-masquerade and squashes the following
commits:
29168c0 [Andrew Or] Synchronize ContextCleaner stop
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fe63e822
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fe63e822
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fe63e822
Branch: refs/heads/master
Commit: fe63e822918a01e1c1d741052b932e9944745fb6
Parents: e750a6b
Author: Andrew Or and...@databricks.com
Authored: Tue Mar 3 13:44:05 2015 -0800
Committer: Andrew Or and...@databricks.com
Committed: Tue Mar 3 13:44:05 2015 -0800
--
.../scala/org/apache/spark/ContextCleaner.scala | 39 +---
1 file changed, 26 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/fe63e822/core/src/main/scala/org/apache/spark/ContextCleaner.scala
--
diff --git a/core/src/main/scala/org/apache/spark/ContextCleaner.scala
b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
index 4a9d007..4dab886 100644
--- a/core/src/main/scala/org/apache/spark/ContextCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
@@ -105,9 +105,19 @@ private[spark] class ContextCleaner(sc: SparkContext)
extends Logging {
cleaningThread.start()
}
- /** Stop the cleaner. */
+ /**
+ * Stop the cleaning thread and wait until the thread has finished running
its current task.
+ */
def stop() {
stopped = true
+// Interrupt the cleaning thread, but wait until the current task has
finished before
+// doing so. This guards against the race condition where a cleaning
thread may
+// potentially clean similarly named variables created by a different
SparkContext,
+// resulting in otherwise inexplicable block-not-found exceptions
(SPARK-6132).
+synchronized {
+ cleaningThread.interrupt()
+}
+cleaningThread.join()
}
/** Register a RDD for cleanup when it is garbage collected. */
@@ -140,18 +150,21 @@ private[spark] class ContextCleaner(sc: SparkContext)
extends Logging {
try {
val reference =
Option(referenceQueue.remove(ContextCleaner.REF_QUEUE_POLL_TIMEOUT))
.map(_.asInstanceOf[CleanupTaskWeakReference])
-reference.map(_.task).foreach { task =
- logDebug(Got cleaning task + task)
- referenceBuffer -= reference.get
- task match {
-case CleanRDD(rddId) =
- doCleanupRDD(rddId, blocking = blockOnCleanupTasks)
-case CleanShuffle(shuffleId) =
- doCleanupShuffle(shuffleId, blocking =
blockOnShuffleCleanupTasks)
-case CleanBroadcast(broadcastId) =
- doCleanupBroadcast(broadcastId, blocking = blockOnCleanupTasks)
-case CleanAccum(accId) =
- doCleanupAccum(accId, blocking = blockOnCleanupTasks)
+// Synchronize here to avoid being interrupted on stop()
+synchronized {
+ reference.map(_.task).foreach { task =
+logDebug(Got cleaning task + task)
+referenceBuffer -= reference.get
+task match {
+ case CleanRDD(rddId) =
+doCleanupRDD(rddId, blocking = blockOnCleanupTasks)
+ case CleanShuffle(shuffleId) =
+doCleanupShuffle(shuffleId, blocking =
blockOnShuffleCleanupTasks)
+ case CleanBroadcast(broadcastId) =
+doCleanupBroadcast(broadcastId, blocking = blockOnCleanupTasks)
+ case CleanAccum(accId) =
+doCleanupAccum(accId, blocking = blockOnCleanupTasks)
+}
}
}
} catch {
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: Revert [SPARK-5423][Core] Cleanup resources in DiskMapIterator.finalize to ensure deleting the temp file
Repository: spark
Updated Branches:
refs/heads/branch-1.3 05d5a29eb - ee4929d1d
Revert [SPARK-5423][Core] Cleanup resources in DiskMapIterator.finalize to
ensure deleting the temp file
This reverts commit 25fae8e7e6c93b7817771342d370b73b40dcf92e.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ee4929d1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ee4929d1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ee4929d1
Branch: refs/heads/branch-1.3
Commit: ee4929d1d38d83382ccdc22bf99f61f24f989c8b
Parents: 05d5a29
Author: Andrew Or and...@databricks.com
Authored: Tue Mar 3 13:04:15 2015 -0800
Committer: Andrew Or and...@databricks.com
Committed: Tue Mar 3 13:04:15 2015 -0800
--
.../util/collection/ExternalAppendOnlyMap.scala | 52
1 file changed, 9 insertions(+), 43 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ee4929d1/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
index fc7e86e..8a0f5a6 100644
---
a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
+++
b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -387,15 +387,6 @@ class ExternalAppendOnlyMap[K, V, C](
private var batchIndex = 0 // Which batch we're in
private var fileStream: FileInputStream = null
-@volatile private var closed = false
-
-// A volatile variable to remember which DeserializationStream is using.
Need to set it when we
-// open a DeserializationStream. But we should use `deserializeStream`
rather than
-// `deserializeStreamToBeClosed` to read the content because touching a
volatile variable will
-// reduce the performance. It must be volatile so that we can see its
correct value in the
-// `finalize` method, which could run in any thread.
-@volatile private var deserializeStreamToBeClosed: DeserializationStream =
null
-
// An intermediate stream that reads from exactly one batch
// This guards against pre-fetching and other arbitrary behavior of higher
level streams
private var deserializeStream = nextBatchStream()
@@ -410,7 +401,6 @@ class ExternalAppendOnlyMap[K, V, C](
// we're still in a valid batch.
if (batchIndex batchOffsets.length - 1) {
if (deserializeStream != null) {
- deserializeStreamToBeClosed = null
deserializeStream.close()
fileStream.close()
deserializeStream = null
@@ -429,11 +419,7 @@ class ExternalAppendOnlyMap[K, V, C](
val bufferedStream = new
BufferedInputStream(ByteStreams.limit(fileStream, end - start))
val compressedStream = blockManager.wrapForCompression(blockId,
bufferedStream)
-// Before returning the stream, assign it to
`deserializeStreamToBeClosed` so that we can
-// close it in `finalize` and also avoid to touch the volatile
`deserializeStreamToBeClosed`
-// during reading the (K, C) pairs.
-deserializeStreamToBeClosed = ser.deserializeStream(compressedStream)
-deserializeStreamToBeClosed
+ser.deserializeStream(compressedStream)
} else {
// No more batches left
cleanup()
@@ -482,34 +468,14 @@ class ExternalAppendOnlyMap[K, V, C](
item
}
-// TODO: Now only use `finalize` to ensure `close` gets called to clean up
the resources. In the
-// future, we need some mechanism to ensure this gets called once the
resources are not used.
-private def cleanup(): Unit = {
- if (!closed) {
-closed = true
-batchIndex = batchOffsets.length // Prevent reading any other batch
-fileStream = null
-try {
- val ds = deserializeStreamToBeClosed
- deserializeStreamToBeClosed = null
- deserializeStream = null
- if (ds != null) {
-ds.close()
- }
-} finally {
- if (file.exists()) {
-file.delete()
- }
-}
- }
-}
-
-override def finalize(): Unit = {
- try {
-cleanup()
- } finally {
-super.finalize()
- }
+// TODO: Ensure this gets called even if the iterator isn't drained.
+private def cleanup() {
+ batchIndex = batchOffsets.length // Prevent reading any other batch
+ val ds = deserializeStream
+ deserializeStream = null
+ fileStream = null
+ ds.close()
+ file.delete()
}
}
-
spark git commit: Revert [SPARK-5423][Core] Cleanup resources in DiskMapIterator.finalize to ensure deleting the temp file
Repository: spark
Updated Branches:
refs/heads/branch-1.2 5226dc74d - eb30fc180
Revert [SPARK-5423][Core] Cleanup resources in DiskMapIterator.finalize to
ensure deleting the temp file
This reverts commit 61bde0049fac324b5004eadfa22b02cd76cf2187.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eb30fc18
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eb30fc18
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eb30fc18
Branch: refs/heads/branch-1.2
Commit: eb30fc180a8908ea294d6d0ad04e90e8a45813f4
Parents: 5226dc7
Author: Andrew Or and...@databricks.com
Authored: Tue Mar 3 13:04:50 2015 -0800
Committer: Andrew Or and...@databricks.com
Committed: Tue Mar 3 13:04:50 2015 -0800
--
.../util/collection/ExternalAppendOnlyMap.scala | 52
1 file changed, 9 insertions(+), 43 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/eb30fc18/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
index fc7e86e..8a0f5a6 100644
---
a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
+++
b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -387,15 +387,6 @@ class ExternalAppendOnlyMap[K, V, C](
private var batchIndex = 0 // Which batch we're in
private var fileStream: FileInputStream = null
-@volatile private var closed = false
-
-// A volatile variable to remember which DeserializationStream is using.
Need to set it when we
-// open a DeserializationStream. But we should use `deserializeStream`
rather than
-// `deserializeStreamToBeClosed` to read the content because touching a
volatile variable will
-// reduce the performance. It must be volatile so that we can see its
correct value in the
-// `finalize` method, which could run in any thread.
-@volatile private var deserializeStreamToBeClosed: DeserializationStream =
null
-
// An intermediate stream that reads from exactly one batch
// This guards against pre-fetching and other arbitrary behavior of higher
level streams
private var deserializeStream = nextBatchStream()
@@ -410,7 +401,6 @@ class ExternalAppendOnlyMap[K, V, C](
// we're still in a valid batch.
if (batchIndex batchOffsets.length - 1) {
if (deserializeStream != null) {
- deserializeStreamToBeClosed = null
deserializeStream.close()
fileStream.close()
deserializeStream = null
@@ -429,11 +419,7 @@ class ExternalAppendOnlyMap[K, V, C](
val bufferedStream = new
BufferedInputStream(ByteStreams.limit(fileStream, end - start))
val compressedStream = blockManager.wrapForCompression(blockId,
bufferedStream)
-// Before returning the stream, assign it to
`deserializeStreamToBeClosed` so that we can
-// close it in `finalize` and also avoid to touch the volatile
`deserializeStreamToBeClosed`
-// during reading the (K, C) pairs.
-deserializeStreamToBeClosed = ser.deserializeStream(compressedStream)
-deserializeStreamToBeClosed
+ser.deserializeStream(compressedStream)
} else {
// No more batches left
cleanup()
@@ -482,34 +468,14 @@ class ExternalAppendOnlyMap[K, V, C](
item
}
-// TODO: Now only use `finalize` to ensure `close` gets called to clean up
the resources. In the
-// future, we need some mechanism to ensure this gets called once the
resources are not used.
-private def cleanup(): Unit = {
- if (!closed) {
-closed = true
-batchIndex = batchOffsets.length // Prevent reading any other batch
-fileStream = null
-try {
- val ds = deserializeStreamToBeClosed
- deserializeStreamToBeClosed = null
- deserializeStream = null
- if (ds != null) {
-ds.close()
- }
-} finally {
- if (file.exists()) {
-file.delete()
- }
-}
- }
-}
-
-override def finalize(): Unit = {
- try {
-cleanup()
- } finally {
-super.finalize()
- }
+// TODO: Ensure this gets called even if the iterator isn't drained.
+private def cleanup() {
+ batchIndex = batchOffsets.length // Prevent reading any other batch
+ val ds = deserializeStream
+ deserializeStream = null
+ fileStream = null
+ ds.close()
+ file.delete()
}
}
-
spark git commit: SPARK-1911 [DOCS] Warn users if their assembly jars are not built with Java 6
Repository: spark Updated Branches: refs/heads/branch-1.3 ee4929d1d - 8446ad0eb SPARK-1911 [DOCS] Warn users if their assembly jars are not built with Java 6 Add warning about building with Java 7+ and running the JAR on early Java 6. CC andrewor14 Author: Sean Owen so...@cloudera.com Closes #4874 from srowen/SPARK-1911 and squashes the following commits: 79fa2f6 [Sean Owen] Add warning about building with Java 7+ and running the JAR on early Java 6. (cherry picked from commit e750a6bfddf1d7bf7d3e99a424ec2b83a18b40d9) Signed-off-by: Andrew Or and...@databricks.com Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8446ad0e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8446ad0e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8446ad0e Branch: refs/heads/branch-1.3 Commit: 8446ad0ebd2abb10ef405dc2ce53d2724604ce83 Parents: ee4929d Author: Sean Owen so...@cloudera.com Authored: Tue Mar 3 13:40:11 2015 -0800 Committer: Andrew Or and...@databricks.com Committed: Tue Mar 3 13:40:16 2015 -0800 -- docs/building-spark.md | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/spark/blob/8446ad0e/docs/building-spark.md -- diff --git a/docs/building-spark.md b/docs/building-spark.md index b666d14..d12ee8b 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -9,6 +9,10 @@ redirect_from: building-with-maven.html Building Spark using Maven requires Maven 3.0.4 or newer and Java 6+. +**Note:** Building Spark with Java 7 or later can create JAR files that may not be +readable with early versions of Java 6, due to the large number of files in the JAR +archive. Build with Java 6 if this is an issue for your deployment. + # Building with `build/mvn` Spark now comes packaged with a self-contained Maven installation to ease building and deployment of Spark from source located under the `build/` directory. This script will automatically download and setup all necessary build requirements ([Maven](https://maven.apache.org/), [Scala](http://www.scala-lang.org/), and [Zinc](https://github.com/typesafehub/zinc)) locally within the `build/` directory itself. It honors any `mvn` binary if present already, however, will pull down its own copy of Scala and Zinc regardless to ensure proper version requirements are met. `build/mvn` execution acts as a pass through to the `mvn` call allowing easy transition from previous build methods. As an example, one can build a version of Spark as follows: - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-6141][MLlib] Upgrade Breeze from 0.10 to 0.11 to fix convergence bug
Repository: spark
Updated Branches:
refs/heads/branch-1.3 9a0b75cdd - 9f249779f
[SPARK-6141][MLlib] Upgrade Breeze from 0.10 to 0.11 to fix convergence bug
LBFGS and OWLQN in Breeze 0.10 has convergence check bug.
This is fixed in 0.11, see the description in Breeze project for detail:
https://github.com/scalanlp/breeze/pull/373#issuecomment-76879760
Author: Xiangrui Meng m...@databricks.com
Author: DB Tsai dbt...@alpinenow.com
Author: DB Tsai dbt...@dbtsai.com
Closes #4879 from dbtsai/breeze and squashes the following commits:
d848f65 [DB Tsai] Merge pull request #1 from mengxr/AlpineNow-breeze
c2ca6ac [Xiangrui Meng] upgrade to breeze-0.11.1
35c2f26 [Xiangrui Meng] fix LRSuite
397a208 [DB Tsai] upgrade breeze
(cherry picked from commit 76e20a0a03cf2c02db35e00271924efb070eaaa5)
Signed-off-by: Xiangrui Meng m...@databricks.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9f249779
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9f249779
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9f249779
Branch: refs/heads/branch-1.3
Commit: 9f249779ffe65131d1b9b95154a8ccd3a89fe022
Parents: 9a0b75c
Author: Xiangrui Meng m...@databricks.com
Authored: Tue Mar 3 23:52:02 2015 -0800
Committer: Xiangrui Meng m...@databricks.com
Committed: Tue Mar 3 23:52:10 2015 -0800
--
mllib/pom.xml| 2 +-
.../spark/mllib/classification/LogisticRegressionSuite.scala | 4
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/9f249779/mllib/pom.xml
--
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 32e9847..48c9d06 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -63,7 +63,7 @@
dependency
groupIdorg.scalanlp/groupId
artifactIdbreeze_${scala.binary.version}/artifactId
- version0.10/version
+ version0.11.1/version
exclusions
!-- This is included as a compile-scoped dependency by jtransforms,
which is
a dependency of breeze. --
http://git-wip-us.apache.org/repos/asf/spark/blob/9f249779/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
--
diff --git
a/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
b/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
index d2b40f2..aaa81da 100644
---
a/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
@@ -372,8 +372,12 @@ class LogisticRegressionSuite extends FunSuite with
MLlibTestSparkContext with M
testRDD2.cache()
testRDD3.cache()
+val numIteration = 10
+
val lrA = new LogisticRegressionWithLBFGS().setIntercept(true)
+lrA.optimizer.setNumIterations(numIteration)
val lrB = new
LogisticRegressionWithLBFGS().setIntercept(true).setFeatureScaling(false)
+lrB.optimizer.setNumIterations(numIteration)
val modelA1 = lrA.run(testRDD1, initialWeights)
val modelA2 = lrA.run(testRDD2, initialWeights)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-5949] HighlyCompressedMapStatus needs more classes registered w/ kryo
Repository: spark
Updated Branches:
refs/heads/branch-1.3 8446ad0eb - 9a0b75cdd
[SPARK-5949] HighlyCompressedMapStatus needs more classes registered w/ kryo
https://issues.apache.org/jira/browse/SPARK-5949
Author: Imran Rashid iras...@cloudera.com
Closes #4877 from squito/SPARK-5949_register_roaring_bitmap and squashes the
following commits:
7e13316 [Imran Rashid] style style style
5f6bb6d [Imran Rashid] more style
709bfe0 [Imran Rashid] style
a5cb744 [Imran Rashid] update tests to cover both types of
RoaringBitmapContainers
09610c6 [Imran Rashid] formatting
f9a0b7c [Imran Rashid] put primitive array registrations together
97beaf8 [Imran Rashid] SPARK-5949 HighlyCompressedMapStatus needs more classes
registered w/ kryo
(cherry picked from commit 1f1fccc5ceb0c5b7656a0594be3a67bd3b432e85)
Signed-off-by: Reynold Xin r...@databricks.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9a0b75cd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9a0b75cd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9a0b75cd
Branch: refs/heads/branch-1.3
Commit: 9a0b75cdd85c1933c590480dd233b1803726ed71
Parents: 8446ad0
Author: Imran Rashid iras...@cloudera.com
Authored: Tue Mar 3 15:33:19 2015 -0800
Committer: Reynold Xin r...@databricks.com
Committed: Tue Mar 3 15:33:26 2015 -0800
--
.../spark/serializer/KryoSerializer.scala | 15 ++---
.../spark/serializer/KryoSerializerSuite.scala | 23 ++--
2 files changed, 33 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/9a0b75cd/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 02158aa..9ce64d4 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -20,22 +20,23 @@ package org.apache.spark.serializer
import java.io.{EOFException, InputStream, OutputStream}
import java.nio.ByteBuffer
+import scala.reflect.ClassTag
+
import com.esotericsoftware.kryo.{Kryo, KryoException}
import com.esotericsoftware.kryo.io.{Input = KryoInput, Output = KryoOutput}
import com.esotericsoftware.kryo.serializers.{JavaSerializer =
KryoJavaSerializer}
import com.twitter.chill.{AllScalaRegistrar, EmptyScalaKryoInstantiator}
+import org.roaringbitmap.{ArrayContainer, BitmapContainer, RoaringArray,
RoaringBitmap}
import org.apache.spark._
import org.apache.spark.api.python.PythonBroadcast
import org.apache.spark.broadcast.HttpBroadcast
-import org.apache.spark.network.nio.{PutBlock, GotBlock, GetBlock}
+import org.apache.spark.network.nio.{GetBlock, GotBlock, PutBlock}
import org.apache.spark.scheduler.{CompressedMapStatus,
HighlyCompressedMapStatus}
import org.apache.spark.storage._
import org.apache.spark.util.BoundedPriorityQueue
import org.apache.spark.util.collection.CompactBuffer
-import scala.reflect.ClassTag
-
/**
* A Spark serializer that uses the [[https://code.google.com/p/kryo/ Kryo
serialization library]].
*
@@ -202,9 +203,17 @@ private[serializer] object KryoSerializer {
classOf[GetBlock],
classOf[CompressedMapStatus],
classOf[HighlyCompressedMapStatus],
+classOf[RoaringBitmap],
+classOf[RoaringArray],
+classOf[RoaringArray.Element],
+classOf[Array[RoaringArray.Element]],
+classOf[ArrayContainer],
+classOf[BitmapContainer],
classOf[CompactBuffer[_]],
classOf[BlockManagerId],
classOf[Array[Byte]],
+classOf[Array[Short]],
+classOf[Array[Long]],
classOf[BoundedPriorityQueue[_]],
classOf[SparkConf]
)
http://git-wip-us.apache.org/repos/asf/spark/blob/9a0b75cd/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index a70f67a..523d898 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -23,9 +23,10 @@ import scala.reflect.ClassTag
import com.esotericsoftware.kryo.Kryo
import org.scalatest.FunSuite
-import org.apache.spark.{SparkConf, SharedSparkContext}
+import org.apache.spark.{SharedSparkContext, SparkConf}
+import org.apache.spark.scheduler.HighlyCompressedMapStatus
import org.apache.spark.serializer.KryoTest._
-
+import org.apache.spark.storage.BlockManagerId
class
spark git commit: [SPARK-5949] HighlyCompressedMapStatus needs more classes registered w/ kryo
Repository: spark
Updated Branches:
refs/heads/master 6c20f3529 - 1f1fccc5c
[SPARK-5949] HighlyCompressedMapStatus needs more classes registered w/ kryo
https://issues.apache.org/jira/browse/SPARK-5949
Author: Imran Rashid iras...@cloudera.com
Closes #4877 from squito/SPARK-5949_register_roaring_bitmap and squashes the
following commits:
7e13316 [Imran Rashid] style style style
5f6bb6d [Imran Rashid] more style
709bfe0 [Imran Rashid] style
a5cb744 [Imran Rashid] update tests to cover both types of
RoaringBitmapContainers
09610c6 [Imran Rashid] formatting
f9a0b7c [Imran Rashid] put primitive array registrations together
97beaf8 [Imran Rashid] SPARK-5949 HighlyCompressedMapStatus needs more classes
registered w/ kryo
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1f1fccc5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1f1fccc5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1f1fccc5
Branch: refs/heads/master
Commit: 1f1fccc5ceb0c5b7656a0594be3a67bd3b432e85
Parents: 6c20f35
Author: Imran Rashid iras...@cloudera.com
Authored: Tue Mar 3 15:33:19 2015 -0800
Committer: Reynold Xin r...@databricks.com
Committed: Tue Mar 3 15:33:19 2015 -0800
--
.../spark/serializer/KryoSerializer.scala | 15 ++---
.../spark/serializer/KryoSerializerSuite.scala | 23 ++--
2 files changed, 33 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/1f1fccc5/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 02158aa..9ce64d4 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -20,22 +20,23 @@ package org.apache.spark.serializer
import java.io.{EOFException, InputStream, OutputStream}
import java.nio.ByteBuffer
+import scala.reflect.ClassTag
+
import com.esotericsoftware.kryo.{Kryo, KryoException}
import com.esotericsoftware.kryo.io.{Input = KryoInput, Output = KryoOutput}
import com.esotericsoftware.kryo.serializers.{JavaSerializer =
KryoJavaSerializer}
import com.twitter.chill.{AllScalaRegistrar, EmptyScalaKryoInstantiator}
+import org.roaringbitmap.{ArrayContainer, BitmapContainer, RoaringArray,
RoaringBitmap}
import org.apache.spark._
import org.apache.spark.api.python.PythonBroadcast
import org.apache.spark.broadcast.HttpBroadcast
-import org.apache.spark.network.nio.{PutBlock, GotBlock, GetBlock}
+import org.apache.spark.network.nio.{GetBlock, GotBlock, PutBlock}
import org.apache.spark.scheduler.{CompressedMapStatus,
HighlyCompressedMapStatus}
import org.apache.spark.storage._
import org.apache.spark.util.BoundedPriorityQueue
import org.apache.spark.util.collection.CompactBuffer
-import scala.reflect.ClassTag
-
/**
* A Spark serializer that uses the [[https://code.google.com/p/kryo/ Kryo
serialization library]].
*
@@ -202,9 +203,17 @@ private[serializer] object KryoSerializer {
classOf[GetBlock],
classOf[CompressedMapStatus],
classOf[HighlyCompressedMapStatus],
+classOf[RoaringBitmap],
+classOf[RoaringArray],
+classOf[RoaringArray.Element],
+classOf[Array[RoaringArray.Element]],
+classOf[ArrayContainer],
+classOf[BitmapContainer],
classOf[CompactBuffer[_]],
classOf[BlockManagerId],
classOf[Array[Byte]],
+classOf[Array[Short]],
+classOf[Array[Long]],
classOf[BoundedPriorityQueue[_]],
classOf[SparkConf]
)
http://git-wip-us.apache.org/repos/asf/spark/blob/1f1fccc5/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index a70f67a..523d898 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -23,9 +23,10 @@ import scala.reflect.ClassTag
import com.esotericsoftware.kryo.Kryo
import org.scalatest.FunSuite
-import org.apache.spark.{SparkConf, SharedSparkContext}
+import org.apache.spark.{SharedSparkContext, SparkConf}
+import org.apache.spark.scheduler.HighlyCompressedMapStatus
import org.apache.spark.serializer.KryoTest._
-
+import org.apache.spark.storage.BlockManagerId
class KryoSerializerSuite extends FunSuite with SharedSparkContext {
conf.set(spark.serializer, org.apache.spark.serializer.KryoSerializer)
spark git commit: [SPARK-6141][MLlib] Upgrade Breeze from 0.10 to 0.11 to fix convergence bug
Repository: spark
Updated Branches:
refs/heads/master d334bfbcf - 76e20a0a0
[SPARK-6141][MLlib] Upgrade Breeze from 0.10 to 0.11 to fix convergence bug
LBFGS and OWLQN in Breeze 0.10 has convergence check bug.
This is fixed in 0.11, see the description in Breeze project for detail:
https://github.com/scalanlp/breeze/pull/373#issuecomment-76879760
Author: Xiangrui Meng m...@databricks.com
Author: DB Tsai dbt...@alpinenow.com
Author: DB Tsai dbt...@dbtsai.com
Closes #4879 from dbtsai/breeze and squashes the following commits:
d848f65 [DB Tsai] Merge pull request #1 from mengxr/AlpineNow-breeze
c2ca6ac [Xiangrui Meng] upgrade to breeze-0.11.1
35c2f26 [Xiangrui Meng] fix LRSuite
397a208 [DB Tsai] upgrade breeze
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/76e20a0a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/76e20a0a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/76e20a0a
Branch: refs/heads/master
Commit: 76e20a0a03cf2c02db35e00271924efb070eaaa5
Parents: d334bfb
Author: Xiangrui Meng m...@databricks.com
Authored: Tue Mar 3 23:52:02 2015 -0800
Committer: Xiangrui Meng m...@databricks.com
Committed: Tue Mar 3 23:52:02 2015 -0800
--
mllib/pom.xml| 2 +-
.../spark/mllib/classification/LogisticRegressionSuite.scala | 4
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/76e20a0a/mllib/pom.xml
--
diff --git a/mllib/pom.xml b/mllib/pom.xml
index a8cee3d..4c8f344 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -63,7 +63,7 @@
dependency
groupIdorg.scalanlp/groupId
artifactIdbreeze_${scala.binary.version}/artifactId
- version0.10/version
+ version0.11.1/version
exclusions
!-- This is included as a compile-scoped dependency by jtransforms,
which is
a dependency of breeze. --
http://git-wip-us.apache.org/repos/asf/spark/blob/76e20a0a/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
--
diff --git
a/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
b/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
index d2b40f2..aaa81da 100644
---
a/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
@@ -372,8 +372,12 @@ class LogisticRegressionSuite extends FunSuite with
MLlibTestSparkContext with M
testRDD2.cache()
testRDD3.cache()
+val numIteration = 10
+
val lrA = new LogisticRegressionWithLBFGS().setIntercept(true)
+lrA.optimizer.setNumIterations(numIteration)
val lrB = new
LogisticRegressionWithLBFGS().setIntercept(true).setFeatureScaling(false)
+lrB.optimizer.setNumIterations(numIteration)
val modelA1 = lrA.run(testRDD1, initialWeights)
val modelA2 = lrA.run(testRDD2, initialWeights)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-6133] Make sc.stop() idempotent
Repository: spark
Updated Branches:
refs/heads/master fe63e8229 - 6c20f3529
[SPARK-6133] Make sc.stop() idempotent
Before we would get the following (benign) error if we called `sc.stop()`
twice. This is because the listener bus would try to post the end event again
even after it has already stopped. This happens occasionally when flaky tests
fail, usually as a result of other sources of error. Either way we shouldn't be
logging this error when it is not the cause of the failure.
```
ERROR LiveListenerBus: SparkListenerBus has already stopped! Dropping event
SparkListenerApplicationEnd(1425348445682)
```
Author: Andrew Or and...@databricks.com
Closes #4871 from andrewor14/sc-stop and squashes the following commits:
a14afc5 [Andrew Or] Move code after code
915db16 [Andrew Or] Move code into code
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6c20f352
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6c20f352
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6c20f352
Branch: refs/heads/master
Commit: 6c20f35290e220e4a659a0222d62575ff959d703
Parents: fe63e82
Author: Andrew Or and...@databricks.com
Authored: Tue Mar 3 15:09:57 2015 -0800
Committer: Andrew Or and...@databricks.com
Committed: Tue Mar 3 15:09:57 2015 -0800
--
core/src/main/scala/org/apache/spark/SparkContext.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6c20f352/core/src/main/scala/org/apache/spark/SparkContext.scala
--
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala
b/core/src/main/scala/org/apache/spark/SparkContext.scala
index e231e83..1a0bee4 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -1392,10 +1392,10 @@ class SparkContext(config: SparkConf) extends Logging
with ExecutorAllocationCli
/** Shut down the SparkContext. */
def stop() {
SparkContext.SPARK_CONTEXT_CONSTRUCTOR_LOCK.synchronized {
- postApplicationEnd()
- ui.foreach(_.stop())
if (!stopped) {
stopped = true
+postApplicationEnd()
+ui.foreach(_.stop())
env.metricsSystem.report()
metadataCleaner.cancel()
cleaner.foreach(_.stop())
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-6133] Make sc.stop() idempotent
Repository: spark
Updated Branches:
refs/heads/branch-1.2 eb30fc180 - a91c1c594
[SPARK-6133] Make sc.stop() idempotent
Before we would get the following (benign) error if we called `sc.stop()`
twice. This is because the listener bus would try to post the end event again
even after it has already stopped. This happens occasionally when flaky tests
fail, usually as a result of other sources of error. Either way we shouldn't be
logging this error when it is not the cause of the failure.
```
ERROR LiveListenerBus: SparkListenerBus has already stopped! Dropping event
SparkListenerApplicationEnd(1425348445682)
```
Author: Andrew Or and...@databricks.com
Closes #4871 from andrewor14/sc-stop and squashes the following commits:
a14afc5 [Andrew Or] Move code after code
915db16 [Andrew Or] Move code into code
(cherry picked from commit 6c20f35290e220e4a659a0222d62575ff959d703)
Signed-off-by: Andrew Or and...@databricks.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a91c1c59
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a91c1c59
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a91c1c59
Branch: refs/heads/branch-1.2
Commit: a91c1c594c57d4bb0a3283de239ca1d1bd0d7741
Parents: eb30fc1
Author: Andrew Or and...@databricks.com
Authored: Tue Mar 3 15:09:57 2015 -0800
Committer: Andrew Or and...@databricks.com
Committed: Tue Mar 3 15:10:03 2015 -0800
--
core/src/main/scala/org/apache/spark/SparkContext.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/a91c1c59/core/src/main/scala/org/apache/spark/SparkContext.scala
--
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala
b/core/src/main/scala/org/apache/spark/SparkContext.scala
index b50a541..98e2fed 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -1229,10 +1229,10 @@ class SparkContext(config: SparkConf) extends Logging
with ExecutorAllocationCli
/** Shut down the SparkContext. */
def stop() {
SparkContext.SPARK_CONTEXT_CONSTRUCTOR_LOCK.synchronized {
- postApplicationEnd()
- ui.foreach(_.stop())
if (!stopped) {
stopped = true
+postApplicationEnd()
+ui.foreach(_.stop())
env.metricsSystem.report()
metadataCleaner.cancel()
env.actorSystem.stop(heartbeatReceiver)
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-6132][HOTFIX] ContextCleaner InterruptedException should be quiet
Repository: spark
Updated Branches:
refs/heads/master 1f1fccc5c - d334bfbcf
[SPARK-6132][HOTFIX] ContextCleaner InterruptedException should be quiet
If the cleaner is stopped, we shouldn't print a huge stack trace when the
cleaner thread is interrupted because we purposefully did this.
Author: Andrew Or and...@databricks.com
Closes #4882 from andrewor14/cleaner-interrupt and squashes the following
commits:
8652120 [Andrew Or] Just a hot fix
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d334bfbc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d334bfbc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d334bfbc
Branch: refs/heads/master
Commit: d334bfbcf38d0c1c6835e72bc85e42196e295744
Parents: 1f1fccc
Author: Andrew Or and...@databricks.com
Authored: Tue Mar 3 20:49:45 2015 -0800
Committer: Andrew Or and...@databricks.com
Committed: Tue Mar 3 20:49:45 2015 -0800
--
core/src/main/scala/org/apache/spark/ContextCleaner.scala | 1 +
1 file changed, 1 insertion(+)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/d334bfbc/core/src/main/scala/org/apache/spark/ContextCleaner.scala
--
diff --git a/core/src/main/scala/org/apache/spark/ContextCleaner.scala
b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
index 4dab886..0c59a61 100644
--- a/core/src/main/scala/org/apache/spark/ContextCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
@@ -168,6 +168,7 @@ private[spark] class ContextCleaner(sc: SparkContext)
extends Logging {
}
}
} catch {
+case ie: InterruptedException if stopped = // ignore
case e: Exception = logError(Error in cleaning thread, e)
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: BUILD: Minor tweaks to internal build scripts
Repository: spark
Updated Branches:
refs/heads/master 165ff3642 - 0c9a8eaed
BUILD: Minor tweaks to internal build scripts
This adds two features:
1. The ability to publish with a different maven version than
that specified in the release source.
2. Forking of different Zinc instances during the parallel dist
creation (to help with some stability issues).
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0c9a8eae
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0c9a8eae
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0c9a8eae
Branch: refs/heads/master
Commit: 0c9a8eaed74b2b381a1cc70e6e6a2783bac0912c
Parents: 165ff36
Author: Patrick Wendell patr...@databricks.com
Authored: Tue Mar 3 00:38:12 2015 -0800
Committer: Patrick Wendell patr...@databricks.com
Committed: Tue Mar 3 01:53:48 2015 -0800
--
dev/create-release/create-release.sh | 24 +++-
1 file changed, 19 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/0c9a8eae/dev/create-release/create-release.sh
--
diff --git a/dev/create-release/create-release.sh
b/dev/create-release/create-release.sh
index da15ce3..6f87fcd 100755
--- a/dev/create-release/create-release.sh
+++ b/dev/create-release/create-release.sh
@@ -34,6 +34,9 @@ ASF_PASSWORD=${ASF_PASSWORD:-XXX}
GPG_PASSPHRASE=${GPG_PASSPHRASE:-XXX}
GIT_BRANCH=${GIT_BRANCH:-branch-1.0}
RELEASE_VERSION=${RELEASE_VERSION:-1.2.0}
+# Allows publishing under a different version identifier than
+# was present in the actual release sources (e.g. rc-X)
+PUBLISH_VERSION=${PUBLISH_VERSION:-$RELEASE_VERSION}
NEXT_VERSION=${NEXT_VERSION:-1.2.1}
RC_NAME=${RC_NAME:-rc2}
@@ -97,30 +100,35 @@ if [[ ! $@ =~ --skip-publish ]]; then
pushd spark
git checkout --force $GIT_TAG
+ # Substitute in case published version is different than released
+ old=^\( \{2,4\}\)version${RELEASE_VERSION}\/version$
+ new=\1version${PUBLISH_VERSION}\/version
+ find . -name pom.xml | grep -v dev | xargs -I {} sed -i \
+-e s/${old}/${new}/ {}
+
# Using Nexus API documented here:
#
https://support.sonatype.com/entries/39720203-Uploading-to-a-Staging-Repository-via-REST-API
echo Creating Nexus staging repository
- repo_request=promoteRequestdatadescriptionApache Spark
$GIT_TAG/description/data/promoteRequest
+ repo_request=promoteRequestdatadescriptionApache Spark $GIT_TAG
(published as $PUBLISH_VERSION)/description/data/promoteRequest
out=$(curl -X POST -d $repo_request -u $ASF_USERNAME:$ASF_PASSWORD \
-H Content-Type:application/xml -v \
$NEXUS_ROOT/profiles/$NEXUS_PROFILE/start)
staged_repo_id=$(echo $out | sed -e
s/.*\(orgapachespark-[0-9]\{4\}\).*/\1/)
echo Created Nexus staging repository: $staged_repo_id
- rm -rf $SPARK_REPO
-
- mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
+ build/mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
-Pyarn -Phive -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl \
clean install
./dev/change-version-to-2.11.sh
- mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
+ build/mvn -DskipTests -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
-Dscala-2.11 -Pyarn -Phive -Phadoop-2.2 -Pspark-ganglia-lgpl -Pkinesis-asl
\
clean install
./dev/change-version-to-2.10.sh
+ rm -rf $SPARK_REPO
pushd $SPARK_REPO
# Remove any extra files generated during install
@@ -197,6 +205,12 @@ if [[ ! $@ =~ --skip-package ]]; then
./dev/change-version-to-2.11.sh
fi
+# Create new Zinc instances for each binary release to avoid interference
+# that causes OOM's and random compiler crashes.
+zinc_port=${zinc_port:-3030}
+zinc_port=$[$zinc_port + 1]
+export ZINC_PORT=$zinc_port
+
./make-distribution.sh --name $NAME --tgz $FLAGS 21 | tee
../binary-release-$NAME.log
cd ..
cp spark-$RELEASE_VERSION-bin-$NAME/spark-$RELEASE_VERSION-bin-$NAME.tgz .
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
[2/2] spark git commit: Preparing Spark release v1.3.0-rc2
Preparing Spark release v1.3.0-rc2 Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3af26870 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3af26870 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3af26870 Branch: refs/heads/branch-1.3 Commit: 3af26870e5163438868c4eb2df88380a533bb232 Parents: b012ed1 Author: Patrick Wendell patr...@databricks.com Authored: Tue Mar 3 10:23:07 2015 + Committer: Patrick Wendell patr...@databricks.com Committed: Tue Mar 3 10:23:07 2015 + -- assembly/pom.xml | 2 +- bagel/pom.xml | 2 +- core/pom.xml | 2 +- examples/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml| 2 +- external/kafka-assembly/pom.xml | 2 +- external/kafka/pom.xml| 2 +- external/mqtt/pom.xml | 2 +- external/twitter/pom.xml | 2 +- external/zeromq/pom.xml | 2 +- extras/java8-tests/pom.xml| 2 +- extras/kinesis-asl/pom.xml| 2 +- extras/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml| 2 +- mllib/pom.xml | 2 +- network/common/pom.xml| 2 +- network/shuffle/pom.xml | 2 +- network/yarn/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- yarn/pom.xml | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/3af26870/assembly/pom.xml -- diff --git a/assembly/pom.xml b/assembly/pom.xml index c1bcdbb..0764d2d 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0-SNAPSHOT/version +version1.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/3af26870/bagel/pom.xml -- diff --git a/bagel/pom.xml b/bagel/pom.xml index 510e926..4f73cf7 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0-SNAPSHOT/version +version1.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/3af26870/core/pom.xml -- diff --git a/core/pom.xml b/core/pom.xml index c993781..5612149 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0-SNAPSHOT/version +version1.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/3af26870/examples/pom.xml -- diff --git a/examples/pom.xml b/examples/pom.xml index 8caad2b..f7d6030 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0-SNAPSHOT/version +version1.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/3af26870/external/flume-sink/pom.xml -- diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 0706f1e..45aa775 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0-SNAPSHOT/version +version1.3.0/version relativePath../../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/3af26870/external/flume/pom.xml -- diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 1f26813..455304f 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0-SNAPSHOT/version +version1.3.0/version relativePath../../pom.xml/relativePath /parent
Git Push Summary
Repository: spark Updated Tags: refs/tags/v1.3.0-rc2 [created] 3af26870e - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-6118] making package name of deploy.worker.CommandUtils and deploy.CommandUtilsSuite consistent
Repository: spark
Updated Branches:
refs/heads/master 0c9a8eaed - 975643c25
[SPARK-6118] making package name of deploy.worker.CommandUtils and
deploy.CommandUtilsSuite consistent
https://issues.apache.org/jira/browse/SPARK-6118
I found that the object CommandUtils is placed under deploy.worker package,
while CommandUtilsSuite is under deploy
Conventionally, we put the implementation and unit test class under the same
package
here, to minimize the change, I move CommandUtilsSuite to worker package,
**However, CommandUtils seems to contain some general methods (though only used
by worker.* classes currently**, we may also consider to replace CommonUtils
Author: CodingCat zhunans...@gmail.com
Closes #4856 from CodingCat/SPARK-6118 and squashes the following commits:
cb93700 [CodingCat] making package name consistent
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/975643c2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/975643c2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/975643c2
Branch: refs/heads/master
Commit: 975643c256e548601bf9015c8840c947df5446bf
Parents: 0c9a8ea
Author: CodingCat zhunans...@gmail.com
Authored: Tue Mar 3 10:32:57 2015 +
Committer: Sean Owen so...@cloudera.com
Committed: Tue Mar 3 10:32:57 2015 +
--
.../apache/spark/deploy/CommandUtilsSuite.scala | 37
.../spark/deploy/worker/CommandUtilsSuite.scala | 36 +++
2 files changed, 36 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/975643c2/core/src/test/scala/org/apache/spark/deploy/CommandUtilsSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/deploy/CommandUtilsSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/CommandUtilsSuite.scala
deleted file mode 100644
index 7915ee7..000
--- a/core/src/test/scala/org/apache/spark/deploy/CommandUtilsSuite.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the License); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.deploy
-
-import org.apache.spark.deploy.worker.CommandUtils
-import org.apache.spark.util.Utils
-
-import org.scalatest.{FunSuite, Matchers}
-
-class CommandUtilsSuite extends FunSuite with Matchers {
-
- test(set libraryPath correctly) {
-val appId = 12345-worker321-9876
-val sparkHome = sys.props.getOrElse(spark.test.home,
fail(spark.test.home is not set!))
-val cmd = new Command(mainClass, Seq(), Map(), Seq(),
Seq(libraryPathToB), Seq())
-val builder = CommandUtils.buildProcessBuilder(cmd, 512, sparkHome, t = t)
-val libraryPath = Utils.libraryPathEnvName
-val env = builder.environment
-env.keySet should contain(libraryPath)
-assert(env.get(libraryPath).startsWith(libraryPathToB))
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/975643c2/core/src/test/scala/org/apache/spark/deploy/worker/CommandUtilsSuite.scala
--
diff --git
a/core/src/test/scala/org/apache/spark/deploy/worker/CommandUtilsSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/worker/CommandUtilsSuite.scala
new file mode 100644
index 000..1c27d83
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/CommandUtilsSuite.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the License); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
spark git commit: [SPARK-6138][CORE][minor] enhance the `toArray` method in `SizeTrackingVector`
Repository: spark
Updated Branches:
refs/heads/master 975643c25 - e359794ce
[SPARK-6138][CORE][minor] enhance the `toArray` method in `SizeTrackingVector`
Use array copy instead of `Iterator#toArray` to make it more efficient.
Author: Wenchen Fan cloud0...@outlook.com
Closes #4825 from cloud-fan/minor and squashes the following commits:
c933ee5 [Wenchen Fan] make toArray method just in parent
946a35b [Wenchen Fan] minor enhance
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e359794c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e359794c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e359794c
Branch: refs/heads/master
Commit: e359794cec7d30ece38752f62dc2a1d3d26b8feb
Parents: 975643c
Author: Wenchen Fan cloud0...@outlook.com
Authored: Tue Mar 3 12:12:23 2015 +
Committer: Sean Owen so...@cloudera.com
Committed: Tue Mar 3 12:12:23 2015 +
--
.../spark/util/collection/PrimitiveVector.scala | 15 ---
.../spark/util/collection/SizeTrackingVector.scala | 7 ---
2 files changed, 12 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/e359794c/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
b/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
index 7e76d06..b6c380a 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/PrimitiveVector.scala
@@ -71,12 +71,21 @@ class PrimitiveVector[@specialized(Long, Int, Double) V:
ClassTag](initialSize:
/** Resizes the array, dropping elements if the total length decreases. */
def resize(newLength: Int): PrimitiveVector[V] = {
-val newArray = new Array[V](newLength)
-_array.copyToArray(newArray)
-_array = newArray
+_array = copyArrayWithLength(newLength)
if (newLength _numElements) {
_numElements = newLength
}
this
}
+
+ /** Return a trimmed version of the underlying array. */
+ def toArray: Array[V] = {
+copyArrayWithLength(size)
+ }
+
+ private def copyArrayWithLength(length: Int): Array[V] = {
+val copy = new Array[V](length)
+_array.copyToArray(copy)
+copy
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/e359794c/core/src/main/scala/org/apache/spark/util/collection/SizeTrackingVector.scala
--
diff --git
a/core/src/main/scala/org/apache/spark/util/collection/SizeTrackingVector.scala
b/core/src/main/scala/org/apache/spark/util/collection/SizeTrackingVector.scala
index 65a7b4e..dfcfb66 100644
---
a/core/src/main/scala/org/apache/spark/util/collection/SizeTrackingVector.scala
+++
b/core/src/main/scala/org/apache/spark/util/collection/SizeTrackingVector.scala
@@ -36,11 +36,4 @@ private[spark] class SizeTrackingVector[T: ClassTag]
resetSamples()
this
}
-
- /**
- * Return a trimmed version of the underlying array.
- */
- def toArray: Array[T] = {
-super.iterator.toArray
- }
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: HOTFIX: Bump HBase version in MapR profiles.
Repository: spark Updated Branches: refs/heads/branch-1.3 841d2a27f - 1aa846165 HOTFIX: Bump HBase version in MapR profiles. After #2982 (SPARK-4048) we rely on the newer HBase packaging format. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1aa84616 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1aa84616 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1aa84616 Branch: refs/heads/branch-1.3 Commit: 1aa8461652ab02c3d5961dfb7b87d44f43d56093 Parents: 841d2a2 Author: Patrick Wendell patr...@databricks.com Authored: Tue Mar 3 01:38:07 2015 -0800 Committer: Patrick Wendell patr...@databricks.com Committed: Tue Mar 3 01:39:26 2015 -0800 -- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/1aa84616/pom.xml -- diff --git a/pom.xml b/pom.xml index 34d4d42..ce1e544 100644 --- a/pom.xml +++ b/pom.xml @@ -1605,7 +1605,7 @@ properties hadoop.version1.0.3-mapr-3.0.3/hadoop.version yarn.version2.4.1-mapr-1408/yarn.version -hbase.version0.94.17-mapr-1405/hbase.version +hbase.version0.98.4-mapr-1408/hbase.version zookeeper.version3.4.5-mapr-1406/zookeeper.version /properties /profile @@ -1615,7 +1615,7 @@ properties hadoop.version2.4.1-mapr-1408/hadoop.version yarn.version2.4.1-mapr-1408/yarn.version -hbase.version0.94.17-mapr-1405-4.0.0-FCS/hbase.version +hbase.version0.98.4-mapr-1408/hbase.version zookeeper.version3.4.5-mapr-1406/zookeeper.version /properties dependencies - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: HOTFIX: Bump HBase version in MapR profiles.
Repository: spark Updated Branches: refs/heads/master b19605619 - 165ff3642 HOTFIX: Bump HBase version in MapR profiles. After #2982 (SPARK-4048) we rely on the newer HBase packaging format. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/165ff364 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/165ff364 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/165ff364 Branch: refs/heads/master Commit: 165ff364265a5328f797e0901fe0dc02f30a7e7a Parents: b196056 Author: Patrick Wendell patr...@databricks.com Authored: Tue Mar 3 01:38:07 2015 -0800 Committer: Patrick Wendell patr...@databricks.com Committed: Tue Mar 3 01:38:50 2015 -0800 -- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/165ff364/pom.xml -- diff --git a/pom.xml b/pom.xml index bb355bf..77d08d6 100644 --- a/pom.xml +++ b/pom.xml @@ -1605,7 +1605,7 @@ properties hadoop.version1.0.3-mapr-3.0.3/hadoop.version yarn.version2.4.1-mapr-1408/yarn.version -hbase.version0.94.17-mapr-1405/hbase.version +hbase.version0.98.4-mapr-1408/hbase.version zookeeper.version3.4.5-mapr-1406/zookeeper.version /properties /profile @@ -1615,7 +1615,7 @@ properties hadoop.version2.4.1-mapr-1408/hadoop.version yarn.version2.4.1-mapr-1408/yarn.version -hbase.version0.94.17-mapr-1405-4.0.0-FCS/hbase.version +hbase.version0.98.4-mapr-1408/hbase.version zookeeper.version3.4.5-mapr-1406/zookeeper.version /properties dependencies - To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
[1/2] spark git commit: Revert Preparing development version 1.3.1-SNAPSHOT
Repository: spark Updated Branches: refs/heads/branch-1.3 ce7158cf7 - b012ed189 Revert Preparing development version 1.3.1-SNAPSHOT This reverts commit 2ab0ba04f66683be25cbe0e83cecf2bdcb0f13ba. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4fee08ef Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4fee08ef Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4fee08ef Branch: refs/heads/branch-1.3 Commit: 4fee08ef0141b1be5684d78b6fe9cb93c98b0bc4 Parents: ce7158c Author: Patrick Wendell patr...@databricks.com Authored: Tue Mar 3 02:20:03 2015 -0800 Committer: Patrick Wendell patr...@databricks.com Committed: Tue Mar 3 02:20:03 2015 -0800 -- assembly/pom.xml | 2 +- bagel/pom.xml | 2 +- core/pom.xml | 2 +- examples/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml| 2 +- external/kafka-assembly/pom.xml | 2 +- external/kafka/pom.xml| 2 +- external/mqtt/pom.xml | 2 +- external/twitter/pom.xml | 2 +- external/zeromq/pom.xml | 2 +- extras/java8-tests/pom.xml| 2 +- extras/kinesis-asl/pom.xml| 2 +- extras/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml| 2 +- mllib/pom.xml | 2 +- network/common/pom.xml| 2 +- network/shuffle/pom.xml | 2 +- network/yarn/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- yarn/pom.xml | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/4fee08ef/assembly/pom.xml -- diff --git a/assembly/pom.xml b/assembly/pom.xml index 87b3e6f..0764d2d 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.1-SNAPSHOT/version +version1.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/4fee08ef/bagel/pom.xml -- diff --git a/bagel/pom.xml b/bagel/pom.xml index 4a13c58..4f73cf7 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.1-SNAPSHOT/version +version1.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/4fee08ef/core/pom.xml -- diff --git a/core/pom.xml b/core/pom.xml index aca0f58..5612149 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.1-SNAPSHOT/version +version1.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/4fee08ef/examples/pom.xml -- diff --git a/examples/pom.xml b/examples/pom.xml index c424592..f7d6030 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.1-SNAPSHOT/version +version1.3.0/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/4fee08ef/external/flume-sink/pom.xml -- diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index ef960a8..45aa775 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.1-SNAPSHOT/version +version1.3.0/version relativePath../../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/4fee08ef/external/flume/pom.xml -- diff --git a/external/flume/pom.xml b/external/flume/pom.xml index f01d6e8..455304f 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -
[2/2] spark git commit: Revert Preparing Spark release v1.3.0-rc1
Revert Preparing Spark release v1.3.0-rc1 This reverts commit f97b0d4a6b26504916816d7aefcf3132cd1da6c2. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b012ed18 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b012ed18 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b012ed18 Branch: refs/heads/branch-1.3 Commit: b012ed189844d2a515a882144364921caf89b4c0 Parents: 4fee08e Author: Patrick Wendell patr...@databricks.com Authored: Tue Mar 3 02:20:05 2015 -0800 Committer: Patrick Wendell patr...@databricks.com Committed: Tue Mar 3 02:20:05 2015 -0800 -- assembly/pom.xml | 2 +- bagel/pom.xml | 2 +- core/pom.xml | 2 +- examples/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml| 2 +- external/kafka-assembly/pom.xml | 2 +- external/kafka/pom.xml| 2 +- external/mqtt/pom.xml | 2 +- external/twitter/pom.xml | 2 +- external/zeromq/pom.xml | 2 +- extras/java8-tests/pom.xml| 2 +- extras/kinesis-asl/pom.xml| 2 +- extras/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml| 2 +- mllib/pom.xml | 2 +- network/common/pom.xml| 2 +- network/shuffle/pom.xml | 2 +- network/yarn/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- yarn/pom.xml | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b012ed18/assembly/pom.xml -- diff --git a/assembly/pom.xml b/assembly/pom.xml index 0764d2d..c1bcdbb 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.0-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/b012ed18/bagel/pom.xml -- diff --git a/bagel/pom.xml b/bagel/pom.xml index 4f73cf7..510e926 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.0-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/b012ed18/core/pom.xml -- diff --git a/core/pom.xml b/core/pom.xml index 5612149..c993781 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.0-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/b012ed18/examples/pom.xml -- diff --git a/examples/pom.xml b/examples/pom.xml index f7d6030..8caad2b 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.0-SNAPSHOT/version relativePath../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/b012ed18/external/flume-sink/pom.xml -- diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 45aa775..0706f1e 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.0-SNAPSHOT/version relativePath../../pom.xml/relativePath /parent http://git-wip-us.apache.org/repos/asf/spark/blob/b012ed18/external/flume/pom.xml -- diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 455304f..1f26813 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ parent groupIdorg.apache.spark/groupId artifactIdspark-parent/artifactId -version1.3.0/version +version1.3.0-SNAPSHOT/version
