Merge remote-tracking branch 'upstream/master' into code-style
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/a3da468d Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/a3da468d Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/a3da468d Branch: refs/heads/master Commit: a3da468d8b99565a966745e09830eaa768a9c267 Parents: c2852cf fdaabdc Author: Frank Dai <soulmach...@gmail.com> Authored: Tue Jan 14 15:29:17 2014 +0800 Committer: Frank Dai <soulmach...@gmail.com> Committed: Tue Jan 14 15:29:17 2014 +0800 ---------------------------------------------------------------------- bin/compute-classpath.sh | 2 + .../scala/org/apache/spark/Accumulators.scala | 4 +- .../scala/org/apache/spark/Aggregator.scala | 14 +- .../scala/org/apache/spark/FutureAction.scala | 8 +- .../apache/spark/InterruptibleIterator.scala | 2 +- .../main/scala/org/apache/spark/Logging.scala | 2 +- .../org/apache/spark/broadcast/Broadcast.scala | 1 + .../spark/broadcast/BroadcastFactory.scala | 2 +- .../apache/spark/broadcast/HttpBroadcast.scala | 5 +- .../spark/broadcast/TorrentBroadcast.scala | 6 +- .../scala/org/apache/spark/deploy/Client.scala | 3 +- .../spark/deploy/worker/CommandUtils.scala | 3 +- .../org/apache/spark/executor/Executor.scala | 2 +- .../org/apache/spark/executor/TaskMetrics.scala | 10 + .../main/scala/org/apache/spark/package.scala | 3 + .../org/apache/spark/rdd/CoGroupedRDD.scala | 3 + .../scala/org/apache/spark/rdd/HadoopRDD.scala | 2 +- .../org/apache/spark/rdd/PairRDDFunctions.scala | 12 +- .../scala/org/apache/spark/rdd/PipedRDD.scala | 3 +- .../main/scala/org/apache/spark/rdd/RDD.scala | 5 + .../apache/spark/scheduler/SparkListener.scala | 13 +- .../spark/scheduler/TaskSchedulerImpl.scala | 5 +- .../org/apache/spark/storage/BlockManager.scala | 3 + .../spark/storage/BlockObjectWriter.scala | 4 +- .../org/apache/spark/storage/StorageLevel.scala | 4 + .../apache/spark/ui/jobs/ExecutorSummary.scala | 2 + .../apache/spark/ui/jobs/ExecutorTable.scala | 4 + .../spark/ui/jobs/JobProgressListener.scala | 14 + .../org/apache/spark/ui/jobs/StagePage.scala | 53 +- .../apache/spark/util/CompletionIterator.scala | 11 +- .../org/apache/spark/util/MetadataCleaner.scala | 8 +- .../spark/util/RateLimitedOutputStream.scala | 79 -- .../apache/spark/util/collection/BitSet.scala | 87 +- .../util/collection/ExternalAppendOnlyMap.scala | 72 +- .../spark/util/collection/OpenHashSet.scala | 23 +- .../util/RateLimitedOutputStreamSuite.scala | 40 - .../collection/ExternalAppendOnlyMapSuite.scala | 77 +- docs/_config.yml | 2 +- docs/_layouts/global.html | 8 +- docs/_plugins/copy_api_dirs.rb | 2 +- docs/api.md | 1 + docs/bagel-programming-guide.md | 10 +- docs/configuration.md | 11 +- docs/graphx-programming-guide.md | 1003 ++++++++++++++++++ docs/img/data_parallel_vs_graph_parallel.png | Bin 0 -> 432725 bytes docs/img/edge-cut.png | Bin 0 -> 12563 bytes docs/img/edge_cut_vs_vertex_cut.png | Bin 0 -> 79745 bytes docs/img/graph_analytics_pipeline.png | Bin 0 -> 427220 bytes docs/img/graph_parallel.png | Bin 0 -> 92288 bytes docs/img/graphx_figures.pptx | Bin 0 -> 1123363 bytes docs/img/graphx_logo.png | Bin 0 -> 40324 bytes docs/img/graphx_performance_comparison.png | Bin 0 -> 166343 bytes docs/img/property_graph.png | Bin 0 -> 225151 bytes docs/img/tables_and_graphs.png | Bin 0 -> 166265 bytes docs/img/triplet.png | Bin 0 -> 31489 bytes docs/img/vertex-cut.png | Bin 0 -> 12246 bytes docs/img/vertex_routing_edge_tables.png | Bin 0 -> 570007 bytes docs/index.md | 4 +- docs/mllib-guide.md | 19 +- docs/python-programming-guide.md | 8 +- .../examples/graphx/LiveJournalPageRank.scala | 49 + graphx/data/followers.txt | 8 + graphx/data/users.txt | 7 + graphx/pom.xml | 67 ++ .../scala/org/apache/spark/graphx/Edge.scala | 45 + .../org/apache/spark/graphx/EdgeDirection.scala | 44 + .../scala/org/apache/spark/graphx/EdgeRDD.scala | 102 ++ .../org/apache/spark/graphx/EdgeTriplet.scala | 49 + .../scala/org/apache/spark/graphx/Graph.scala | 405 +++++++ .../spark/graphx/GraphKryoRegistrator.scala | 31 + .../org/apache/spark/graphx/GraphLoader.scala | 72 ++ .../org/apache/spark/graphx/GraphOps.scala | 301 ++++++ .../apache/spark/graphx/PartitionStrategy.scala | 103 ++ .../scala/org/apache/spark/graphx/Pregel.scala | 139 +++ .../org/apache/spark/graphx/VertexRDD.scala | 347 ++++++ .../spark/graphx/impl/EdgePartition.scala | 220 ++++ .../graphx/impl/EdgePartitionBuilder.scala | 45 + .../spark/graphx/impl/EdgeTripletIterator.scala | 42 + .../apache/spark/graphx/impl/GraphImpl.scala | 379 +++++++ .../spark/graphx/impl/MessageToPartition.scala | 98 ++ .../graphx/impl/ReplicatedVertexView.scala | 195 ++++ .../apache/spark/graphx/impl/RoutingTable.scala | 65 ++ .../apache/spark/graphx/impl/Serializers.scala | 395 +++++++ .../spark/graphx/impl/VertexPartition.scala | 261 +++++ .../org/apache/spark/graphx/impl/package.scala | 7 + .../org/apache/spark/graphx/lib/Analytics.scala | 136 +++ .../spark/graphx/lib/ConnectedComponents.scala | 38 + .../org/apache/spark/graphx/lib/PageRank.scala | 147 +++ .../apache/spark/graphx/lib/SVDPlusPlus.scala | 138 +++ .../lib/StronglyConnectedComponents.scala | 94 ++ .../apache/spark/graphx/lib/TriangleCount.scala | 76 ++ .../scala/org/apache/spark/graphx/package.scala | 18 + .../spark/graphx/util/BytecodeUtils.scala | 117 ++ .../spark/graphx/util/GraphGenerators.scala | 218 ++++ .../collection/PrimitiveKeyOpenHashMap.scala | 153 +++ graphx/src/test/resources/log4j.properties | 28 + .../org/apache/spark/graphx/GraphOpsSuite.scala | 66 ++ .../org/apache/spark/graphx/GraphSuite.scala | 273 +++++ .../apache/spark/graphx/LocalSparkContext.scala | 28 + .../org/apache/spark/graphx/PregelSuite.scala | 41 + .../apache/spark/graphx/SerializerSuite.scala | 183 ++++ .../apache/spark/graphx/VertexRDDSuite.scala | 85 ++ .../spark/graphx/impl/EdgePartitionSuite.scala | 76 ++ .../graphx/impl/VertexPartitionSuite.scala | 113 ++ .../graphx/lib/ConnectedComponentsSuite.scala | 113 ++ .../apache/spark/graphx/lib/PageRankSuite.scala | 119 +++ .../spark/graphx/lib/SVDPlusPlusSuite.scala | 31 + .../lib/StronglyConnectedComponentsSuite.scala | 57 + .../spark/graphx/lib/TriangleCountSuite.scala | 70 ++ .../spark/graphx/util/BytecodeUtilsSuite.scala | 93 ++ mllib/data/sample_naive_bayes_data.txt | 6 + .../spark/mllib/api/python/PythonMLLibAPI.scala | 17 + .../classification/LogisticRegression.scala | 4 +- .../spark/mllib/classification/NaiveBayes.scala | 65 +- .../apache/spark/mllib/classification/SVM.scala | 2 + .../spark/mllib/regression/LabeledPoint.scala | 6 +- .../apache/spark/mllib/regression/Lasso.scala | 4 +- .../mllib/regression/LinearRegression.scala | 2 + .../mllib/regression/RidgeRegression.scala | 2 + .../classification/JavaNaiveBayesSuite.java | 72 ++ pom.xml | 5 +- project/SparkBuild.scala | 21 +- python/pyspark/mllib/_common.py | 2 +- python/pyspark/mllib/classification.py | 77 +- python/pyspark/mllib/clustering.py | 11 +- python/pyspark/mllib/recommendation.py | 10 +- python/pyspark/mllib/regression.py | 35 +- python/pyspark/worker.py | 4 + python/run-tests | 5 + .../streaming/api/java/JavaDStreamLike.scala | 3 +- .../spark/streaming/dstream/DStream.scala | 60 +- .../dstream/DStreamCheckpointData.scala | 2 +- .../streaming/dstream/FileInputDStream.scala | 82 +- .../util/RateLimitedOutputStream.scala | 79 ++ .../spark/streaming/util/RawTextSender.scala | 13 +- .../spark/streaming/BasicOperationsSuite.scala | 72 +- .../util/RateLimitedOutputStreamSuite.scala | 40 + 137 files changed, 7959 insertions(+), 388 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/a3da468d/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/a3da468d/mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/a3da468d/mllib/src/main/scala/org/apache/spark/mllib/regression/LinearRegression.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/a3da468d/mllib/src/main/scala/org/apache/spark/mllib/regression/RidgeRegression.scala ----------------------------------------------------------------------
