merge master into akka-streams branch Be sure to do all of the following to help us incorporate your contribution quickly and easily:
- [x] Make sure the commit message is formatted like: `[GEARPUMP-<Jira issue #>] Meaningful description of pull request` - [x] Make sure tests pass via `sbt clean test`. - [x] Make sure old documentation affected by the pull request has been updated and new documentation added for new functionality. Author: manuzhang <[email protected]> Author: huafengw <[email protected]> Author: Kam Kasravi <[email protected]> Author: darionyaphet <[email protected]> Author: Roshanson <[email protected]> Author: Willem Jiang <[email protected]> Author: Christoph Seifert <[email protected]> Author: [email protected] <[email protected]> Closes #136 from manuzhang/akka-streams. Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/cc0578e5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/cc0578e5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/cc0578e5 Branch: refs/heads/akka-streams Commit: cc0578e5d5d33bbab719c32744bceb2051342a13 Parents: f1bec67 Author: manuzhang <[email protected]> Authored: Fri Jan 20 16:47:30 2017 +0800 Committer: huafengw <[email protected]> Committed: Fri Jan 20 16:47:30 2017 +0800 ---------------------------------------------------------------------- .github/PULL_REQUEST_TEMPLATE.md | 8 + .travis.yml | 36 +- CHANGELOG.md | 37 + CONTRIBUTING.md | 2 +- LICENSE | 22 - LICENSE.bin | 344 ++++++ NOTICE | 11 +- README.md | 6 +- ReleaseProcess.md | 29 +- conf/gear.conf | 5 + .../gearpump/transport/netty/MessageBatch.java | 2 +- .../org.apache.gearpump.jarstore.JarStore | 19 + .../apache/gearpump/cluster/DaemonMessage.scala | 50 + .../apache/gearpump/cluster/UserConfig.scala | 21 +- .../gearpump/cluster/client/ClientContext.scala | 92 +- .../gearpump/cluster/client/MasterClient.scala | 81 -- .../cluster/client/RunningApplication.scala | 52 + .../cluster/embedded/EmbeddedCluster.scala | 95 ++ .../apache/gearpump/cluster/main/Local.scala | 89 ++ .../apache/gearpump/cluster/main/Master.scala | 236 ++++ .../apache/gearpump/cluster/main/Worker.scala | 70 ++ .../gearpump/cluster/master/AppManager.scala | 354 ++++++ .../cluster/master/InMemoryKVService.scala | 122 ++ .../apache/gearpump/cluster/master/Master.scala | 311 +++++ .../cluster/scheduler/PriorityScheduler.scala | 154 +++ .../gearpump/cluster/scheduler/Scheduler.scala | 77 ++ .../worker/DefaultExecutorProcessLauncher.scala | 40 + .../apache/gearpump/cluster/worker/Worker.scala | 579 ++++++++++ .../apache/gearpump/metrics/AkkaReporter.scala | 2 +- .../org/apache/gearpump/metrics/Counter.scala | 4 +- .../org/apache/gearpump/metrics/Histogram.scala | 4 +- .../apache/gearpump/metrics/JvmMetricsSet.scala | 4 +- .../org/apache/gearpump/metrics/Meter.scala | 2 +- .../org/apache/gearpump/metrics/Metrics.scala | 2 +- .../metrics/MetricsReporterService.scala | 7 +- .../partitioner/BroadcastPartitioner.scala | 36 - .../partitioner/CoLocationPartitioner.scala | 31 - .../gearpump/partitioner/HashPartitioner.scala | 32 - .../gearpump/partitioner/Partitioner.scala | 109 -- .../ShuffleGroupingPartitioner.scala | 45 - .../partitioner/ShufflePartitioner.scala | 44 - .../serializer/FastKryoSerializer.scala | 11 +- .../serializer/GearpumpSerialization.scala | 3 +- .../org/apache/gearpump/util/ActorUtil.scala | 16 +- .../org/apache/gearpump/util/Constants.scala | 10 - .../org/apache/gearpump/util/FileUtils.scala | 2 +- .../scala/org/apache/gearpump/util/Util.scala | 10 +- .../org.apache.gearpump.jarstore.JarStore | 19 + .../apache/gearpump/cluster/MiniCluster.scala | 73 ++ .../cluster/appmaster/AppManagerSpec.scala | 182 +++ .../appmaster/InMemoryKVServiceSpec.scala | 69 ++ .../cluster/client/RunningApplicationSpec.scala | 99 ++ .../apache/gearpump/cluster/main/MainSpec.scala | 188 +++ .../cluster/main/MasterWatcherSpec.scala | 43 + .../scheduler/PrioritySchedulerSpec.scala | 230 ++++ .../gearpump/cluster/worker/WorkerSpec.scala | 128 +++ .../gearpump/jarstore/FileServerSpec.scala | 7 +- .../apache/gearpump/metrics/MetricsSpec.scala | 3 +- .../gearpump/partitioner/PartitionerSpec.scala | 81 -- .../gearpump/serializer/SerializerSpec.scala | 21 +- .../apache/gearpump/util/FileUtilsSpec.scala | 4 +- .../org.apache.gearpump.jarstore.JarStore | 20 - .../apache/gearpump/cluster/DaemonMessage.scala | 51 - .../cluster/embedded/EmbeddedCluster.scala | 95 -- .../apache/gearpump/cluster/main/Local.scala | 90 -- .../apache/gearpump/cluster/main/Master.scala | 236 ---- .../apache/gearpump/cluster/main/Worker.scala | 71 -- .../gearpump/cluster/master/AppManager.scala | 355 ------ .../cluster/master/InMemoryKVService.scala | 122 -- .../apache/gearpump/cluster/master/Master.scala | 311 ----- .../cluster/scheduler/PriorityScheduler.scala | 156 --- .../gearpump/cluster/scheduler/Scheduler.scala | 79 -- .../worker/DefaultExecutorProcessLauncher.scala | 41 - .../apache/gearpump/cluster/worker/Worker.scala | 581 ---------- .../gearpump/jarstore/dfs/DFSJarStore.scala | 67 -- .../org.apache.gearpump.jarstore.JarStore | 20 - .../apache/gearpump/cluster/MiniCluster.scala | 74 -- .../apache/gearpump/cluster/main/MainSpec.scala | 190 --- .../cluster/main/MasterWatcherSpec.scala | 44 - .../cluster/master/AppManagerSpec.scala | 184 --- .../cluster/master/InMemoryKVServiceSpec.scala | 69 -- .../scheduler/PrioritySchedulerSpec.scala | 232 ---- .../gearpump/cluster/worker/WorkerSpec.scala | 129 --- dev-tools/build | 72 ++ dev-tools/create_apache_bin_release.sh | 115 ++ dev-tools/create_apache_source_release.sh | 5 + dev-tools/dependencies.sh | 51 + docs/build_doc.sh | 23 +- docs/contents/api/java.md | 1 + docs/contents/api/scala.md | 1 + .../deployment/deployment-configuration.md | 84 ++ docs/contents/deployment/deployment-docker.md | 5 + docs/contents/deployment/deployment-ha.md | 75 ++ docs/contents/deployment/deployment-local.md | 34 + .../deployment/deployment-msg-delivery.md | 60 + .../deployment/deployment-resource-isolation.md | 112 ++ docs/contents/deployment/deployment-security.md | 80 ++ .../deployment/deployment-standalone.md | 59 + .../deployment/deployment-ui-authentication.md | 290 +++++ docs/contents/deployment/deployment-yarn.md | 135 +++ .../deployment/get-gearpump-distribution.md | 83 ++ .../contents/deployment/hardware-requirement.md | 30 + docs/contents/dev/dev-connectors.md | 237 ++++ docs/contents/dev/dev-custom-serializer.md | 137 +++ docs/contents/dev/dev-ide-setup.md | 29 + docs/contents/dev/dev-non-streaming-example.md | 133 +++ docs/contents/dev/dev-rest-api.md | 1083 ++++++++++++++++++ docs/contents/dev/dev-storm.md | 214 ++++ docs/contents/dev/dev-write-1st-app.md | 370 ++++++ docs/contents/img/actor_hierarchy.png | Bin 0 -> 109855 bytes docs/contents/img/checkpoint_equation.png | Bin 0 -> 1663 bytes .../img/checkpoint_interval_equation.png | Bin 0 -> 16637 bytes docs/contents/img/checkpointing.png | Bin 0 -> 104471 bytes docs/contents/img/checkpointing_interval.png | Bin 0 -> 44284 bytes docs/contents/img/clock.png | Bin 0 -> 33147 bytes docs/contents/img/dag.png | Bin 0 -> 18263 bytes docs/contents/img/dashboard.gif | Bin 0 -> 152314 bytes docs/contents/img/dashboard.png | Bin 0 -> 29434 bytes docs/contents/img/dashboard_3.png | Bin 0 -> 42684 bytes docs/contents/img/download.jpg | Bin 0 -> 3999 bytes docs/contents/img/dynamic.png | Bin 0 -> 40091 bytes docs/contents/img/exact.png | Bin 0 -> 104471 bytes docs/contents/img/failures.png | Bin 0 -> 95078 bytes docs/contents/img/flow_control.png | Bin 0 -> 61928 bytes docs/contents/img/flowcontrol.png | Bin 0 -> 61928 bytes docs/contents/img/ha.png | Bin 0 -> 47152 bytes docs/contents/img/kafka_wordcount.png | Bin 0 -> 14520 bytes docs/contents/img/layout.png | Bin 0 -> 126947 bytes docs/contents/img/logo.png | Bin 0 -> 2053 bytes docs/contents/img/logo.svg | 71 ++ docs/contents/img/logo2.png | Bin 0 -> 7970 bytes docs/contents/img/messageLoss.png | Bin 0 -> 37166 bytes docs/contents/img/netty_transport.png | Bin 0 -> 65010 bytes docs/contents/img/replay.png | Bin 0 -> 37255 bytes docs/contents/img/shuffle.png | Bin 0 -> 23550 bytes docs/contents/img/storm_gearpump_cluster.png | Bin 0 -> 30930 bytes docs/contents/img/storm_gearpump_dag.png | Bin 0 -> 54000 bytes docs/contents/img/submit.png | Bin 0 -> 32954 bytes docs/contents/img/submit2.png | Bin 0 -> 51933 bytes docs/contents/img/through_vs_message_size.png | Bin 0 -> 20965 bytes docs/contents/index.md | 35 + docs/contents/introduction/basic-concepts.md | 46 + docs/contents/introduction/commandline.md | 84 ++ docs/contents/introduction/features.md | 67 ++ .../contents/introduction/gearpump-internals.md | 228 ++++ docs/contents/introduction/message-delivery.md | 47 + .../contents/introduction/performance-report.md | 34 + .../introduction/submit-your-1st-application.md | 39 + docs/docs/api/java.md | 1 - docs/docs/api/scala.md | 1 - .../docs/deployment/deployment-configuration.md | 84 -- docs/docs/deployment/deployment-docker.md | 5 - docs/docs/deployment/deployment-ha.md | 75 -- docs/docs/deployment/deployment-local.md | 34 - docs/docs/deployment/deployment-msg-delivery.md | 60 - .../deployment/deployment-resource-isolation.md | 112 -- docs/docs/deployment/deployment-security.md | 80 -- docs/docs/deployment/deployment-standalone.md | 59 - .../deployment/deployment-ui-authentication.md | 290 ----- docs/docs/deployment/deployment-yarn.md | 135 --- .../deployment/get-gearpump-distribution.md | 83 -- docs/docs/deployment/hardware-requirement.md | 30 - docs/docs/dev/dev-connectors.md | 237 ---- docs/docs/dev/dev-custom-serializer.md | 137 --- docs/docs/dev/dev-ide-setup.md | 29 - docs/docs/dev/dev-non-streaming-example.md | 133 --- docs/docs/dev/dev-rest-api.md | 1083 ------------------ docs/docs/dev/dev-storm.md | 214 ---- docs/docs/dev/dev-write-1st-app.md | 370 ------ docs/docs/img/actor_hierarchy.png | Bin 109855 -> 0 bytes docs/docs/img/checkpoint_equation.png | Bin 1663 -> 0 bytes docs/docs/img/checkpoint_interval_equation.png | Bin 16637 -> 0 bytes docs/docs/img/checkpointing.png | Bin 104471 -> 0 bytes docs/docs/img/checkpointing_interval.png | Bin 44284 -> 0 bytes docs/docs/img/clock.png | Bin 33147 -> 0 bytes docs/docs/img/dag.png | Bin 18263 -> 0 bytes docs/docs/img/dashboard.gif | Bin 152314 -> 0 bytes docs/docs/img/dashboard.png | Bin 29434 -> 0 bytes docs/docs/img/dashboard_3.png | Bin 42684 -> 0 bytes docs/docs/img/download.jpg | Bin 3999 -> 0 bytes docs/docs/img/dynamic.png | Bin 40091 -> 0 bytes docs/docs/img/exact.png | Bin 104471 -> 0 bytes docs/docs/img/failures.png | Bin 95078 -> 0 bytes docs/docs/img/flow_control.png | Bin 61928 -> 0 bytes docs/docs/img/flowcontrol.png | Bin 61928 -> 0 bytes docs/docs/img/ha.png | Bin 47152 -> 0 bytes docs/docs/img/kafka_wordcount.png | Bin 14520 -> 0 bytes docs/docs/img/layout.png | Bin 126947 -> 0 bytes docs/docs/img/logo.png | Bin 2053 -> 0 bytes docs/docs/img/logo.svg | 71 -- docs/docs/img/logo2.png | Bin 7970 -> 0 bytes docs/docs/img/messageLoss.png | Bin 37166 -> 0 bytes docs/docs/img/netty_transport.png | Bin 65010 -> 0 bytes docs/docs/img/replay.png | Bin 37255 -> 0 bytes docs/docs/img/shuffle.png | Bin 23550 -> 0 bytes docs/docs/img/storm_gearpump_cluster.png | Bin 30930 -> 0 bytes docs/docs/img/storm_gearpump_dag.png | Bin 54000 -> 0 bytes docs/docs/img/submit.png | Bin 32954 -> 0 bytes docs/docs/img/submit2.png | Bin 51933 -> 0 bytes docs/docs/img/through_vs_message_size.png | Bin 20965 -> 0 bytes docs/docs/index.md | 35 - docs/docs/introduction/basic-concepts.md | 46 - docs/docs/introduction/commandline.md | 84 -- docs/docs/introduction/features.md | 67 -- docs/docs/introduction/gearpump-internals.md | 228 ---- docs/docs/introduction/message-delivery.md | 47 - docs/docs/introduction/performance-report.md | 34 - .../introduction/submit-your-1st-application.md | 39 - docs/mkdocs.yml | 2 + docs/version.yml | 2 +- .../distributedshell/DistributedShell.scala | 4 +- .../distributeservice/DistributeService.scala | 4 +- .../pagerank/PageRankApplication.scala | 2 +- .../streaming/examples/complexdag/Dag.scala | 2 +- .../examples/fsio/SequenceFileIO.scala | 2 +- .../streaming/examples/hbase/HBaseConn.scala | 67 ++ .../streaming/examples/hbase/Split.scala | 47 + .../examples/kafka/KafkaReadWrite.scala | 2 +- .../examples/kafka/dsl/KafkaReadWrite.scala | 4 +- .../kafka/wordcount/KafkaWordCount.scala | 2 +- .../gearpump/streaming/examples/sol/SOL.scala | 2 +- .../examples/state/MessageCountApp.scala | 2 +- .../examples/state/WindowAverageApp.scala | 2 +- .../examples/wordcountjava/WordCount.java | 5 +- .../examples/wordcountjava/dsl/WordCount.java | 117 +- .../examples/wordcount/WordCount.scala | 2 +- .../wordcount/dsl/WindowedWordCount.scala | 2 +- .../examples/wordcount/dsl/WordCount.scala | 4 +- .../gearpump/akkastream/GearAttributes.scala | 2 +- .../akkastream/GearpumpMaterializer.scala | 53 +- .../GearpumpMaterializerSession.scala | 2 +- .../gearpump/akkastream/example/Test.scala | 2 +- .../gearpump/akkastream/example/Test10.scala | 2 +- .../gearpump/akkastream/example/Test11.scala | 6 +- .../gearpump/akkastream/example/Test12.scala | 8 +- .../gearpump/akkastream/example/Test13.scala | 2 +- .../gearpump/akkastream/example/Test14.scala | 2 +- .../gearpump/akkastream/example/Test15.scala | 2 +- .../gearpump/akkastream/example/Test16.scala | 7 +- .../gearpump/akkastream/example/Test2.scala | 2 +- .../gearpump/akkastream/example/Test3.scala | 4 +- .../gearpump/akkastream/example/Test4.scala | 4 +- .../gearpump/akkastream/example/Test5.scala | 2 +- .../gearpump/akkastream/example/Test6.scala | 4 +- .../gearpump/akkastream/example/Test7.scala | 2 +- .../gearpump/akkastream/example/Test8.scala | 2 +- .../gearpump/akkastream/example/Test9.scala | 2 +- .../akkastream/example/WikipediaApp.scala | 2 +- .../akkastream/graph/GraphPartitioner.scala | 2 +- .../gearpump/akkastream/graph/LocalGraph.scala | 2 +- .../gearpump/akkastream/graph/RemoteGraph.scala | 4 +- .../gearpump/akkastream/graph/SubGraph.scala | 2 +- .../materializer/LocalMaterializerImpl.scala | 64 +- .../materializer/RemoteMaterializerImpl.scala | 40 +- .../akkastream/module/BridgeModule.scala | 2 +- .../akkastream/module/DummyModule.scala | 2 +- .../akkastream/module/GearpumpTaskModule.scala | 2 +- .../akkastream/module/GroupByModule.scala | 2 +- .../akkastream/module/ReduceModule.scala | 2 +- .../gearpump/akkastream/scaladsl/Api.scala | 2 +- .../gearpump/akkastream/task/BalanceTask.scala | 2 +- .../gearpump/akkastream/task/BatchTask.scala | 2 +- .../akkastream/task/BroadcastTask.scala | 2 +- .../gearpump/akkastream/task/ConcatTask.scala | 2 +- .../akkastream/task/DelayInitialTask.scala | 2 +- .../akkastream/task/DropWithinTask.scala | 2 +- .../akkastream/task/FlattenMergeTask.scala | 2 +- .../gearpump/akkastream/task/FoldTask.scala | 2 +- .../gearpump/akkastream/task/GraphTask.scala | 2 +- .../akkastream/task/GroupedWithinTask.scala | 2 +- .../akkastream/task/InterleaveTask.scala | 2 +- .../gearpump/akkastream/task/MapAsyncTask.scala | 2 +- .../gearpump/akkastream/task/MergeTask.scala | 4 +- .../akkastream/task/SingleSourceTask.scala | 2 +- .../akkastream/task/SinkBridgeTask.scala | 2 +- .../akkastream/task/SourceBridgeTask.scala | 2 +- .../akkastream/task/StatefulMapConcatTask.scala | 2 +- .../akkastream/task/TakeWithinTask.scala | 2 +- .../gearpump/akkastream/task/ThrottleTask.scala | 2 +- .../akkastream/task/TickSourceTask.scala | 2 +- .../gearpump/akkastream/task/Unzip2Task.scala | 2 +- .../gearpump/akkastream/task/Zip2Task.scala | 2 +- .../akkastream/util/MaterializedValueOps.scala | 2 +- .../gearpump/cluster/utils/SystemOperation.java | 7 +- .../cluster/worker/CGroupProcessLauncher.scala | 2 +- .../apache/gearpump/redis/RedisMessage.scala | 148 ++- .../org/apache/gearpump/redis/RedisSink.scala | 27 +- .../experiments/storm/main/GearpumpNimbus.scala | 2 +- .../storm/partitioner/StormPartitioner.scala | 2 +- .../experiments/storm/util/GraphBuilder.scala | 2 +- .../partitioner/StormPartitionerSpec.scala | 2 +- .../gearpump/experiments/yarn/Constants.scala | 1 + .../experiments/yarn/appmaster/Command.scala | 2 +- .../experiments/yarn/client/LaunchCluster.scala | 5 + .../yarn/appmaster/CommandSpec.scala | 95 -- .../yarn/client/LaunchClusterSpec.scala | 2 +- .../gearpump/external/hbase/HBaseSink.scala | 20 +- .../external/hbase/dsl/HBaseDSLSink.scala | 5 +- .../gearpump/streaming/kafka/dsl/KafkaDSL.scala | 12 +- .../org.apache.gearpump.jarstore.JarStore | 19 + .../gearpump/jarstore/dfs/DFSJarStore.scala | 66 ++ integrationtest/autorun/autorun.sh | 19 +- .../gearpump/integrationtest/TestSpecBase.scala | 2 +- .../checklist/CommandLineSpec.scala | 2 +- .../checklist/ConnectorKafkaSpec.scala | 2 +- .../checklist/DynamicDagSpec.scala | 4 +- .../integrationtest/checklist/ExampleSpec.scala | 10 +- .../checklist/MessageDeliverySpec.scala | 2 +- .../checklist/StormCompatibilitySpec.scala | 11 +- licenses/LICENSE-AnchorJS.txt | 21 - licenses/LICENSE-activation.txt | 93 ++ licenses/LICENSE-angular-loading-bar.txt | 20 + licenses/LICENSE-angular-motion.txt | 8 + licenses/LICENSE-angular-smart-table.txt | 7 + licenses/LICENSE-angular-strap.txt | 9 + licenses/LICENSE-angular-touch.txt | 9 + licenses/LICENSE-angular-ui-router.txt | 21 + licenses/LICENSE-angular.txt | 9 + licenses/LICENSE-angularjs.txt | 22 + licenses/LICENSE-aopalliance.txt | 4 + licenses/LICENSE-asm.txt | 29 + licenses/LICENSE-bootstrap-additions.txt | 9 + licenses/LICENSE-bootstrap.txt | 3 +- licenses/LICENSE-clipboard.js.txt | 7 + licenses/LICENSE-clojure.txt | 3 + licenses/LICENSE-d3js.txt | 27 + licenses/LICENSE-derive.txt | 7 + licenses/LICENSE-font-awesome.txt | 11 + licenses/LICENSE-glyphicons.txt | 21 - licenses/LICENSE-htmlcleaner.txt | 33 + licenses/LICENSE-jawn-parser.txt | 8 + licenses/LICENSE-jaxb-api.txt | 210 ++++ licenses/LICENSE-jaxb-impl.txt | 210 ++++ licenses/LICENSE-jcodings.txt | 18 + licenses/LICENSE-jedis.txt | 22 + licenses/LICENSE-jersey-guice.txt | 210 ++++ licenses/LICENSE-jline.txt | 3 + licenses/LICENSE-joni.txt | 8 + licenses/LICENSE-jopt-simple.txt | 22 + licenses/LICENSE-jquery-cookie.txt | 20 + licenses/LICENSE-jquery.txt | 57 +- licenses/LICENSE-jsch.txt | 30 + licenses/LICENSE-jsp-api.txt | 308 +++++ licenses/LICENSE-junit.txt | 3 + licenses/LICENSE-kryo.txt | 10 + licenses/LICENSE-lodash.txt | 47 + licenses/LICENSE-mimepull.txt | 210 ++++ licenses/LICENSE-minlog.txt | 10 + licenses/LICENSE-modernizr.txt | 21 - licenses/LICENSE-momentjs.txt | 22 + licenses/LICENSE-ng-file-upload.txt | 20 + licenses/LICENSE-normalize.txt | 21 - licenses/LICENSE-paranamer.txt | 28 + licenses/LICENSE-polyfill.txt | 21 - licenses/LICENSE-protobuf-java.txt | 42 + licenses/LICENSE-reactive-streams.txt | 9 + licenses/LICENSE-reflectasm.txt | 10 + licenses/LICENSE-scala-java8-compat.txt | 14 + licenses/LICENSE-scala-parser-combinators.txt | 14 + licenses/LICENSE-scala-reflect.txt | 14 + licenses/LICENSE-scala-stm.txt | 23 + licenses/LICENSE-scribejava-apis.txt | 22 + licenses/LICENSE-scribejava-core.txt | 22 + licenses/LICENSE-servlet-api.txt | 308 +++++ licenses/LICENSE-slf4j.txt | 21 + licenses/LICENSE-stax-api.txt | 308 +++++ licenses/LICENSE-ui-select.txt | 20 + licenses/LICENSE-upickle.txt | 7 + licenses/LICENSE-xmlenc.txt | 27 + licenses/LICENSE-xz.txt | 1 + notes/0.8.2.markdown | 41 + project/Build.scala | 570 --------- project/BuildDashboard.scala | 119 ++ project/BuildExample.scala | 236 ---- project/BuildExamples.scala | 209 ++++ project/BuildExperiments.scala | 119 ++ project/BuildExternals.scala | 124 ++ project/BuildGearpump.scala | 253 ++++ project/BuildIntegrationTest.scala | 93 -- project/BuildIntegrationTests.scala | 93 ++ project/BuildShaded.scala | 142 --- project/Dependencies.scala | 102 ++ project/Docs.scala | 61 + project/Pack.scala | 34 +- project/build.properties | 2 +- project/license.sbt | 1 - project/plugins.sbt | 5 +- scalastyle-config.xml | 240 ++++ .../gearpump/services/AppMasterService.scala | 10 +- .../gearpump/services/MasterService.scala | 13 +- .../gearpump/services/SecurityService.scala | 2 +- .../gearpump/streaming/javaapi/Graph.java | 2 +- .../javaapi/dsl/functions/FilterFunction.java | 30 - .../javaapi/dsl/functions/FlatMapFunction.java | 32 - .../javaapi/dsl/functions/GroupByFunction.java | 31 - .../javaapi/dsl/functions/MapFunction.java | 31 - .../javaapi/dsl/functions/ReduceFunction.java | 30 - .../apache/gearpump/streaming/Constants.scala | 10 + .../org/apache/gearpump/streaming/DAG.scala | 2 +- .../gearpump/streaming/StreamApplication.scala | 2 +- .../streaming/appmaster/AppMaster.scala | 2 +- .../streaming/appmaster/ClockService.scala | 2 +- .../streaming/appmaster/DagManager.scala | 2 +- .../streaming/appmaster/JarScheduler.scala | 2 +- .../apache/gearpump/streaming/dsl/Stream.scala | 245 ---- .../gearpump/streaming/dsl/StreamApp.scala | 109 -- .../dsl/api/functions/FilterFunction.scala | 42 + .../dsl/api/functions/MapFunction.scala | 43 + .../dsl/api/functions/ReduceFunction.scala | 42 + .../streaming/dsl/javaapi/JavaStream.scala | 22 +- .../streaming/dsl/javaapi/JavaStreamApp.scala | 9 +- .../dsl/javaapi/functions/FlatMapFunction.scala | 32 + .../dsl/javaapi/functions/GroupByFunction.scala | 28 + .../dsl/partitioner/GroupByPartitioner.scala | 2 +- .../apache/gearpump/streaming/dsl/plan/OP.scala | 17 +- .../gearpump/streaming/dsl/plan/Planner.scala | 2 +- .../plan/functions/SingleInputFunction.scala | 65 +- .../streaming/dsl/scalaapi/Stream.scala | 287 +++++ .../streaming/dsl/scalaapi/StreamApp.scala | 109 ++ .../scalaapi/functions/FlatMapFunction.scala | 103 ++ .../functions/SerializableFunction.scala | 32 + .../streaming/dsl/task/CountTriggerTask.scala | 1 - .../dsl/task/EventTimeTriggerTask.scala | 1 - .../dsl/task/ProcessingTimeTriggerTask.scala | 1 - .../streaming/dsl/task/TransformTask.scala | 15 +- .../dsl/window/impl/WindowRunner.scala | 72 +- .../streaming/metrics/ProcessorAggregator.scala | 3 +- .../partitioner/BroadcastPartitioner.scala | 36 + .../partitioner/CoLocationPartitioner.scala | 31 + .../streaming/partitioner/HashPartitioner.scala | 32 + .../streaming/partitioner/Partitioner.scala | 108 ++ .../ShuffleGroupingPartitioner.scala | 45 + .../partitioner/ShufflePartitioner.scala | 44 + .../streaming/source/DataSourceTask.scala | 2 + .../streaming/task/SerializerResolver.scala | 2 +- .../gearpump/streaming/task/Subscriber.scala | 2 +- .../gearpump/streaming/task/Subscription.scala | 4 +- .../gearpump/streaming/task/TaskActor.scala | 13 +- .../org/apache/gearpump/streaming/DAGSpec.scala | 2 +- .../streaming/appmaster/AppMasterSpec.scala | 2 +- .../streaming/appmaster/ClockServiceSpec.scala | 2 +- .../streaming/appmaster/DagManagerSpec.scala | 2 +- .../streaming/appmaster/JarSchedulerSpec.scala | 2 +- .../streaming/appmaster/TaskManagerSpec.scala | 2 +- .../streaming/appmaster/TaskSchedulerSpec.scala | 2 +- .../gearpump/streaming/dsl/StreamAppSpec.scala | 72 -- .../gearpump/streaming/dsl/StreamSpec.scala | 128 --- .../gearpump/streaming/dsl/plan/OpSpec.scala | 18 +- .../streaming/dsl/plan/PlannerSpec.scala | 17 +- .../functions/SingleInputFunctionSpec.scala | 206 ++-- .../streaming/dsl/scalaapi/StreamAppSpec.scala | 73 ++ .../streaming/dsl/scalaapi/StreamSpec.scala | 129 +++ .../streaming/dsl/task/TransformTaskSpec.scala | 75 ++ .../streaming/partitioner/PartitionerSpec.scala | 80 ++ .../streaming/source/DataSourceTaskSpec.scala | 12 +- .../streaming/task/SubscriberSpec.scala | 2 +- .../streaming/task/SubscriptionSpec.scala | 2 +- .../gearpump/streaming/task/TaskActorSpec.scala | 2 +- version.sbt | 2 +- 459 files changed, 14337 insertions(+), 10304 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/.github/PULL_REQUEST_TEMPLATE.md ---------------------------------------------------------------------- diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..175c743 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,8 @@ +Be sure to do all of the following to help us incorporate your contribution +quickly and easily: + + - [ ] Make sure the commit message is formatted like: + `[GEARPUMP-<Jira issue #>] Meaningful description of pull request` + - [ ] Make sure tests pass via `sbt clean test`. + - [ ] Make sure old documentation affected by the pull request has been updated and new documentation added for new functionality. + http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index e7e8abd..e47393c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,7 @@ language: - java - scala -sudo: false -before_script: -- rm -rf $HOME/.m2 -- rm -rf $HOME/.ivy2 -- mkdir -p $HOME/.sbt/launchers/0.13.11/ -- curl -L -o $HOME/.sbt/launchers/0.13.11/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.11/sbt-launch.jar -script: -- echo "TRAVIS_PULL_REQUEST" $TRAVIS_PULL_REQUEST -- echo "TRAVIS_BRANCH" $TRAVIS_BRANCH -- echo "TRAVIS_TAG" $TRAVIS_TAG -- echo "repo" $TRAVIS_REPO_SLUG -- set -o pipefail; skipLogs="Resolving |Compiling |Done updating|Updating |scoverage|coverage-report"; - if [[ $TRAVIS_PULL_REQUEST != "false" || $TRAVIS_REPO_SLUG != "apache/incubator-gearpump" ]]; - then sbt -jvm-opts project/travis/jvmopts clean +assembly scalastyle test:scalastyle it:scalastyle unidoc coverage +test | grep -v -E "$skipLogs"; - elif [[ $TRAVIS_BRANCH == "master" ]]; - then sbt -jvm-opts project/travis/jvmopts clean +assembly +publish | grep -v -E "$skipLogs"; sbt -jvm-opts project/travis/jvmopts scalastyle test:scalastyle it:scalastyle unidoc coverage +test | grep -v -E "$skipLogs"; - elif [[ $TRAVIS_TAG != "" ]]; - then sbt -jvm-opts project/travis/jvmopts clean +assembly +packArchiveZip | grep -v -E "$skipLogs"; - fi +sudo: required jdk: - oraclejdk8 addons: @@ -30,10 +12,24 @@ scala: - 2.11.8 cache: directories: - - $HOME/.m2 + - $HOME/.m2/repository + - $HOME/.sbt - $HOME/.ivy2 git: depth: 1 +script: +- echo "TRAVIS_PULL_REQUEST" $TRAVIS_PULL_REQUEST +- echo "TRAVIS_BRANCH" $TRAVIS_BRANCH +- echo "TRAVIS_TAG" $TRAVIS_TAG +- echo "repo" $TRAVIS_REPO_SLUG +- set -o pipefail; skipLogs="Resolving |Compiling |Done updating|Updating |scoverage|coverage-report"; + if [[ $TRAVIS_PULL_REQUEST != "false" || $TRAVIS_REPO_SLUG != "apache/incubator-gearpump" ]]; + then sbt -jvm-opts project/travis/jvmopts clean +assembly scalastyle test:scalastyle it:scalastyle unidoc coverage +test | grep -v -E "$skipLogs"; + elif [[ $TRAVIS_BRANCH == "master" ]]; + then sbt -jvm-opts project/travis/jvmopts clean +assembly +publish | grep -v -E "$skipLogs"; sbt -jvm-opts project/travis/jvmopts scalastyle test:scalastyle it:scalastyle unidoc coverage +test | grep -v -E "$skipLogs"; + elif [[ $TRAVIS_TAG != "" ]]; + then sbt -jvm-opts project/travis/jvmopts clean +assembly +packArchiveZip | grep -v -E "$skipLogs"; + fi after_success: - if [[ $TRAVIS_PULL_REQUEST != "false" || $TRAVIS_REPO_SLUG != "apache/incubator-gearpump" || $TRAVIS_BRANCH == "master" ]]; then bash <(curl -s https://codecov.io/bash); http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/CHANGELOG.md ---------------------------------------------------------------------- diff --git a/CHANGELOG.md b/CHANGELOG.md index 564899e..f2522ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,40 @@ +Apache Gearpump 0.8.2 +================== +Change log: +------------------ +- GEARPUMP-241 Update CHANGELOG.md to reflect all JIRA's +- GEARPUMP-240 Correct licenses +- GEARPUMP-239 Correct binary and source licenses subdirectory +- GEARPUMP-238 Correct release naming and packaging +- GEARPUMP-237 do not manually download sbt launcher jar on travis +- GEARPUMP-234 Update LICENSE and related files for 0.8.2-incubating binary release +- GEARPUMP-230 Removed unused code from AppMasterService +- GEARPUMP-227 remove unneeded license files +- GEARPUMP-226 refactor build files +- GEARPUMP-225 move partitioner from gearpump-core to module gearpump-streaming +- GEARPUMP-224 Merge gearpump-daemon to gearpump-core +- GEARPUMP-222 DFSJarStore is loaded even when local file path is configured +- GEARPUMP-218 add shaded library as transitive dependencies +- GEARPUMP-215 Gearpump Redis Integration - RedisStorage +- GEARPUMP-213 build docs with MkDocs +- GEARPUMP-212 Make Gearpump's shaded artifacts publishable +- GEARPUMP-208 Demo site goes down a lot. Also references gearpump.io when connection is refused +- GEARPUMP-207 remove hadoop dependency out of application path from dashboard +- GEARPUMP-205 remove hdfs dependency from command gear's classpath +- GEARPUMP-204 Add unit test for external_hbase module +- GEARPUMP-203 Use DataSourceTask / DataSinkTask for DSL +- GEARPUMP-201 integration test failure +- GEARPUMP-197 Busy loop in FetchThread when incoming queue is full +- GEARPUMP-189 Update project/Build.scala to enable publishing releases to https://repository.apache.org/content/repositories/releases/org/apache/gearpump/ +- GEARPUMP-188 use java.time.Instant for Task start time +- GEARPUMP-185 Yarn kills Apache Gearpump's worker if the memory usage is too high +- GEARPUMP-171 Fix some typos +- GEARPUMP-48 Improve Java interoperability +- GEARPUMP-37 Sometimes application clock is "not started" when the data stream is very slow +- GEARPUMP-32 Minimum clock of source Tasks maybe inaccurate +- GEARPUMP-23 add window DSL +- GEARPUMP-15 Define initial Apache branded home website + Apache Gearpump 0.8.1 ================== Change log: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/CONTRIBUTING.md ---------------------------------------------------------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9fd3859..a769f6d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -267,7 +267,7 @@ To build Gearpump document use <a name="ide-setup"></a> ## IDE setup IDE environment can be set up on either Windows, Linux and Mac platform. You can choose the one you prefer. -The IDE setup guide can be found at [Gearpump website](http://gearpump.apache.org/releases/latest/dev-ide-setup.html). +The IDE setup guide can be found at [Gearpump website](http://gearpump.apache.org/releases/latest/dev/dev-ide-setup/index.html). It is highly recommended to perform [package build](#build) before IDE setup. http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/LICENSE ---------------------------------------------------------------------- diff --git a/LICENSE b/LICENSE index 5bc72ee..7e555a5 100644 --- a/LICENSE +++ b/LICENSE @@ -209,11 +209,6 @@ Apache Gearpump subcomponents: code for these subcomponents is subject to the terms and conditions of the following licenses. - - This product bundles AnchorJS 1.1.1, copyright Bryan Braun, - which is available under a MIT license. - For details see licenses/LICENSE-AnchorJS.txt - This product bundles Bootstrap 3.3.5, copyright Twitter Inc., which is available under a MIT license. For details see licenses/LICENSE-bootstrap.txt @@ -221,23 +216,6 @@ Apache Gearpump subcomponents: This product bundles jquery 2.1.4, copyright jQuery Foundation, Inc., which is available under a MIT license. For details see licenses/LICENSE-jquery.txt - - This product bundles modernizr 2.6.1, copyright Faruk Ates, Paul Irish, - Alex Sexton, Ryan Seddon, Patrick Kettner, Stu Cox and Richard Herrera, - which is available under a MIT license. - For details see licenses/LICENSE-modernizr.txt - - This product bundles normalize 3.0.3, copyright Nicolas Gallagher and - Jonathan Neal, which is available under a MIT license. - For details, see licenses/normalize.MIT. - - This product bundles Glyphicons Halflings Regular, copyright - Jan Kovarik, which is available under a MIT license. - For details, see licenses/glyphicons.MIT. - - This product bundles polyfill, copyright Scott Jehl, Paul Irish and - Nicholas Zakas, which is available under a MIT license. - For details, see licenses/glyphicons.MIT. http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/LICENSE.bin ---------------------------------------------------------------------- diff --git a/LICENSE.bin b/LICENSE.bin new file mode 100644 index 0000000..52b09fe --- /dev/null +++ b/LICENSE.bin @@ -0,0 +1,344 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +======================================================================== + +Apache Gearpump subcomponents: + + Apache Gearpump includes a number of subcomponents with + separate copyright notices and license terms. Your use of the source + code for these subcomponents is subject to the terms and + conditions of the following licenses. + +======================================================================== +BSD licenses +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +com.jcraft:jsch:0.1.42 - http://www.jcraft.com/jsch/ +com.thoughtworks.paranamer:paranamer:2.3 - https://github.com/paul-hammant/paranamer +jline:jline:0.9.94 - http://jline.sourceforge.net/ +net.sourceforge.htmlcleaner:htmlcleaner:2.2 - http://htmlcleaner.sourceforge.net/download.php +asm.asm:3.1 - http://asm.ow2.org/ +org.ow2.asm:asm:4.0 - http://asm.ow2.org/ +org.ow2.asm:asm:5.0.3 - http://asm.ow2.org/ +org.scala-stm:scala-stm_2.11:0.7 - https://nbronson.github.io/scala-stm/releases.html +xmlenc:xmlenc:0.52 - http://xmlenc.sourceforge.net/ + +======================================================================== +BSD 3-clause licenses +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +org.scala-lang:scala-reflect:2.11.8 - http://www.scala-lang.org/download/#License +org.scala-lang.modules:scala-java8-compat_2.11:0.7.0 - http://www.scala-lang.org/download/#License +org.scala-lang.modules:scala-parser-combinators_2.11:1.0.2 - http://www.scala-lang.org/download/#License +org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4 - http://www.scala-lang.org/download/#License +org.scala-lang.modules:scala-xml_2.11:1.0.2 - http://www.scala-lang.org/download/#License +org.webjars:d3js:3.5.6 - https://d3js.org/ + +======================================================================== +New BSD licenses +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +com.esotericsoftware.kryo:kryo:2.21 - https://github.com/EsotericSoftware/kryo/ +com.esotericsoftware.minlog:minlog:1.2 - https://github.com/EsotericSoftware/minlog +com.esotericsoftware.reflectasm:reflectasm:1.07 - https://github.com/EsotericSoftware/reflectasm +com.esotericsoftware:kryo:3.0.3 - https://github.com/EsotericSoftware/kryo/ +com.esotericsoftware:minlog:1.3.0 - https://github.com/EsotericSoftware/minlog +com.esotericsoftware:reflectasm:1.10.1 - https://github.com/EsotericSoftware/reflectasm +com.google.protobuf:protobuf-java:2.5.0 - https://developers.google.com/protocol-buffers/ + +======================================================================== +CC BY 3.0 license +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +org.webjars:font-awesome:4.5.0 - http://fontawesome.io/ + +======================================================================== +CC0 license +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +org.reactivestreams:reactive-streams:1.0.0 - http://www.reactive-streams.org/ + +======================================================================== +CDDL 1.1 license +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +com.sun.jersey.contribs:jersey-guice:1.9 - https://jersey.java.net/ +com.sun.jersey:jersey-client:1.9 - https://jersey.java.net/ +com.sun.jersey:jersey-core:1.9 - https://jersey.java.net/ +com.sun.jersey:jersey-json:1.9 - https://jersey.java.net/ +com.sun.jersey:jersey-server:1.9 - https://jersey.java.net/ +com.sun.xml.bind:jaxb-impl:2.2.3-1 - https://jaxb.java.net/ +javax.xml.bind:jaxb-api:2.2.2 - https://jaxb.java.net/ +org.jvnet.mimepull:mimepull:1.9.4 - https://mimepull.java.net/ + +======================================================================== +Common Development and Distribution License (CDDL) v1.0 +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +javax.activation:activation:1.1 - http://www.oracle.com/technetwork/java/index.html +javax.servlet:servlet-api:2.5 - http://www.oracle.com/technetwork/java/index.html +javax.xml.stream:stax-api:1.0-2 - https://sjsxp.java.net/ +javax.servlet.jsp:jsp-api:2.1 - http://www.oracle.com/technetwork/java/index.html + +======================================================================== +Common Public License Version 1.0 +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +junit:junit:3.8.1 - http://junit.org + +======================================================================== +Eclipse Public License 1.0 +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +org.clojure:clojure:1.6.0 - http://clojure.org/ + +======================================================================== +MIT licenses +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +com.github.scribejava:scribejava-apis:2.4.0 - https://github.com/scribejava/scribejava +com.github.scribejava:scribejava-core:2.4.0 - https://github.com/scribejava/scribejava +com.lihaoyi:derive_2.11:0.3.4 - https://github.com/lihaoyi/upickle +com.lihaoyi:upickle_2.11:0.3.4 - https://github.com/lihaoyi/upickle +org.jruby.jcodings:jcodings:1.0.8 - https://github.com/jruby/jcodings +org.jruby.joni:joni:2.1.2 - https://github.com/jruby/joni +org.slf4j:slf4j-api:1.7.16 - http://www.slf4j.org/ +org.slf4j:slf4j-api:1.7.12 - http://www.slf4j.org/ +org.slf4j:slf4j-log4j12:1.7.16 - http://www.slf4j.org/ +org.spire-math:jawn-parser_2.11:0.7.0 - https://github.com/non/jawn/ +org.webjars:angular-ui-router:0.2.15 - https://github.com/angular-ui/ui-router +org.webjars:angularjs:1.4.9 - https://angularjs.org +org.webjars.bower:angular-loading-bar:0.8.0 - https://chieffancypants.github.io/angular-loading-bar/ +org.webjars.bower:angular-motion:0.4.3 - http://www.webjars.org/ +org.webjars.bower:angular-smart-table:2.1.6 - https://lorenzofox3.github.io/smart-table-website/ +org.webjars.bower:angular-strap:2.3.5 - http://mgcrea.github.io/angular-strap/ +org.webjars.bower:bootstrap:0.3.3.5 = https://getbootstrap.com/ +org.webjars.bower:bootstrap-additions:0.3.1 = https://github.com/mgcrea/bootstrap-additions +org.webjars.bower:clipboard.js:0.1.1 - https://clipboardjs.com/ +org.webjars.bower:ng-file-upload:5.0.9 - https://github.com/danialfarid/ng-file-upload +org.webjars:lodash:3.10.1 - https://lodash.com/ +org.webjars:jquery:2.2.0 - https://jquery.com/ +org.webjars:jquery-cookie:1.4.1 - https://github.com/js-cookie/js-cookie +org.webjars:momentjs:2.10.6 - http://momentjs.com/ +org.webjars.npm:angular-touch:1.5.0 - http://angularjs.org +org.webjars.npm:ui-select:0.14.2 - http://angular-ui.github.io/ui-select/ +redis.clients:jedis:2.9.0 - https://github.com/xetorthio/jedis +net.sf.jopt-simple:jopt-simple:3.2 - https://pholser.github.io/jopt-simple/ + +======================================================================== +Public Domain +The text of each license is also included at licenses/LICENSE-[project].txt. +See project link for details. +======================================================================== +[info] aopalliance:aopalliance:1.0 - http://aopalliance.sourceforge.net/ +[info] org.tukaani:xz:1.0 - http://tukaani.org/xz/java.html http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/NOTICE ---------------------------------------------------------------------- diff --git a/NOTICE b/NOTICE index da0bf4f..ca86fe0 100644 --- a/NOTICE +++ b/NOTICE @@ -1,14 +1,5 @@ -Apache Gearpump +Apache Gearpump (incubating) Copyright 2015-2016 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). - -======================================================================= - -Apache Gearpump contains subcomponents with separate copyright notices and -license terms. Your use of the source code for the these subcomponents -is subject to the terms and conditions of their respective licenses. - -See the LICENSE file for a list of subcomponents and dependencies and -their respective licenses. http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 7258155..9edd0f7 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,8 @@ -## Apache Gearpump (incubating) [](http://gearpump.apache.org/downloads.html) [](https://github.com/apache/incubator-gearpump/blob/master/LICENSE) [](https://travis-ci.org/apache/incubator-gearpump?branch=master) [](https://codecov.io/github/apache/incubator-gearpump?branch=master) - -[](http://gearpump.apache.org/downloads.html) +## Apache Gearpump (incubating) [](http://gearpump.apache.org/downloads.html) [](https://github.com/apache/incubator-gearpump/blob/master/LICENSE) [](https://travis-ci.org/apache/incubator-gearpump?branch=master) [](https://codecov.io/github/apache/incubator-gearpump?branch=master) Gearpump is a lightweight real-time big data streaming engine. It is inspired by recent advances in the [Akka](https://github.com/akka/akka) framework and a desire to improve on existing streaming frameworks. - - The name Gearpump is a reference to the engineering term "gear pump", which is a super simple pump that consists of only two gears, but is very powerful at streaming water.  http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/ReleaseProcess.md ---------------------------------------------------------------------- diff --git a/ReleaseProcess.md b/ReleaseProcess.md index 4ecf460..e1b188a 100644 --- a/ReleaseProcess.md +++ b/ReleaseProcess.md @@ -11,17 +11,32 @@ Step0: Function verification Checklist Step1: Pre-release =================== -1. Modify CHANGELOG.md to add JIRA's -2. Update version in docs/_config.yml +1. Modify CHANGELOG.md to add JIRA's for the release +2. Update version in docs/version.yml 3. Bump the gearpump version in version.sbt - ```scala version in ThisBuild := "RELEASE_VERSION" ``` - -4. Run dev-tools/create_apache_source_release.sh $GPG_KEY $GPG_PASSPHRASE -5. svn add gearpump-* to https://dist.apache.org/repos/dist/dev/incubator/gearpump/RELEASE_VERSION-incubating/RC[0-9] -6. svn add KEYS to https://dist.apache.org/repos/dist/dev/incubator/gearpump/ +4. Run dev-tools/dependencies.sh + This will generate a LICENSE.dependencies file that lists all dependencies including Apache. + Make sure this agrees with the LICENSE and license/* files. + Eventually we'll have something like a verify option so the inspection isn't manual. +5. Run dev-tools/create_apache_source_release.sh $GPG_KEY $GPG_PASSPHRASE + This will provide the source artifacts that need to be uploaded in step 6. below +6. Upload to svn + Run 'svn checkout https://dist.apache.org/repos/dist/dev/incubator/gearpump' + Run 'svn mkdir RELEASE_VERSION-incubating' + Run 'svn mkdir RELEASE_VERSION-incubating/RC[0-9]' + cp the gearpump* files generated from 5. to RELEASE_VERSION-incubating/RC[0-9] + Run 'svn add RELEASE_VERSION-incubating/RC[0-9]/gearpump*' + Run 'svn commit' +7. Run dev-tools/create_apache_bin_release.sh $GPG_KEY $GPG_PASSPHRASE + This will provide the binary artifacts that need to be uploaded in step 8. below +8. svn add gearpump-* to https://dist.apache.org/repos/dist/dev/incubator/gearpump/RELEASE_VERSION-incubating/RC[0-9] +9. svn add KEYS to https://dist.apache.org/repos/dist/dev/incubator/gearpump/ + This only needs to be done if we are adding new committers for this release +10.Create a tag for the RC release by ```git tag RELEASE_VERION-RC[0-9]``` +11.Push this tag upstream and merge Step2: Release ================== http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/conf/gear.conf ---------------------------------------------------------------------- diff --git a/conf/gear.conf b/conf/gear.conf index fa0a3e9..52c67b8 100644 --- a/conf/gear.conf +++ b/conf/gear.conf @@ -282,6 +282,11 @@ gearpump { ### Configuration for yarn module ########################### yarn { + ## By default the owner of yarn application(Here is a Gearpump cluster) is the one who + ## launches yarnclient. + ## Now you delegat to other users by setting following config. + # user = "guest" + client { package-path = "/usr/lib/gearpump/gearpump.zip" } http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java b/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java index d9de871..13ce9fa 100644 --- a/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java +++ b/core/src/main/java/org/apache/gearpump/transport/netty/MessageBatch.java @@ -18,7 +18,7 @@ package org.apache.gearpump.transport.netty; -import org.apache.gearpump.google.common.io.Closeables; +import com.google.common.io.Closeables; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBufferOutputStream; import org.jboss.netty.buffer.ChannelBuffers; http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore ---------------------------------------------------------------------- diff --git a/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore new file mode 100644 index 0000000..c64d444 --- /dev/null +++ b/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.gearpump.jarstore.local.LocalJarStore \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/core/src/main/scala/org/apache/gearpump/cluster/DaemonMessage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/cluster/DaemonMessage.scala b/core/src/main/scala/org/apache/gearpump/cluster/DaemonMessage.scala new file mode 100644 index 0000000..1e94132 --- /dev/null +++ b/core/src/main/scala/org/apache/gearpump/cluster/DaemonMessage.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gearpump.cluster + +import akka.actor.ActorRef +import org.apache.gearpump.cluster.master.Master.MasterInfo +import org.apache.gearpump.cluster.scheduler.Resource +import org.apache.gearpump.cluster.worker.WorkerId + +/** + * Cluster Bootup Flow + */ +object WorkerToMaster { + + /** When an worker is started, it sends RegisterNewWorker */ + case object RegisterNewWorker + + /** When worker lose connection with master, it tries to register itself again with old Id. */ + case class RegisterWorker(workerId: WorkerId) + + /** Worker is responsible to broadcast its current status to master */ + case class ResourceUpdate(worker: ActorRef, workerId: WorkerId, resource: Resource) +} + +object MasterToWorker { + + /** Master confirm the reception of RegisterNewWorker or RegisterWorker */ + case class WorkerRegistered(workerId: WorkerId, masterInfo: MasterInfo) + + /** Worker have not received reply from master */ + case class UpdateResourceFailed(reason: String = null, ex: Throwable = null) + + /** Master is synced with worker on resource slots managed by current worker */ + case object UpdateResourceSucceed +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala b/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala index 393d5f7..1946cfb 100644 --- a/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala +++ b/core/src/main/scala/org/apache/gearpump/cluster/UserConfig.scala @@ -20,16 +20,13 @@ package org.apache.gearpump.cluster import akka.actor.{ActorSystem, ExtendedActorSystem} import akka.serialization.JavaSerializer -import org.apache.gearpump.google.common.io.BaseEncoding -import org.apache.gearpump.util.LogUtil +import com.google.common.io.BaseEncoding -import scala.util.{Failure, Success, Try} /** * Immutable configuration */ final class UserConfig(private val _config: Map[String, String]) extends Serializable { - private val LOG = LogUtil.getLogger(getClass) def withBoolean(key: String, value: Boolean): UserConfig = { new UserConfig(_config + (key -> value.toString)) @@ -138,18 +135,10 @@ final class UserConfig(private val _config: Map[String, String]) extends Seriali if (null == value) { this } else { - Try({ - val serializer = new JavaSerializer(system.asInstanceOf[ExtendedActorSystem]) - val bytes = serializer.toBinary(value) - BaseEncoding.base64().encode(bytes) - }) match { - case Success(enc) => - this.withString(key, enc) - case Failure(throwable) => - LOG.error(s"Could not serialize value with key $key ${throwable.getMessage}") - this - } - + val serializer = new JavaSerializer(system.asInstanceOf[ExtendedActorSystem]) + val bytes = serializer.toBinary(value) + val encoded = BaseEncoding.base64().encode(bytes) + this.withString(key, encoded) } } // scalastyle:on line.size.limit http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/core/src/main/scala/org/apache/gearpump/cluster/client/ClientContext.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/cluster/client/ClientContext.scala b/core/src/main/scala/org/apache/gearpump/cluster/client/ClientContext.scala old mode 100644 new mode 100755 index 0cba079..48b95d8 --- a/core/src/main/scala/org/apache/gearpump/cluster/client/ClientContext.scala +++ b/core/src/main/scala/org/apache/gearpump/cluster/client/ClientContext.scala @@ -19,23 +19,25 @@ package org.apache.gearpump.cluster.client import java.util.concurrent.TimeUnit -import scala.collection.JavaConverters._ -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, Future} -import scala.util.Try import akka.actor.{ActorRef, ActorSystem} import akka.util.Timeout import com.typesafe.config.{Config, ConfigValueFactory} -import org.slf4j.Logger - -import org.apache.gearpump.cluster.MasterToAppMaster.{AppMastersData, ReplayFromTimestampWindowTrailingEdge} -import org.apache.gearpump.cluster.MasterToClient.ReplayApplicationResult +import org.apache.gearpump.cluster.ClientToMaster.{ResolveAppId, ShutdownApplication, SubmitApplication} +import org.apache.gearpump.cluster.MasterToAppMaster.{AppMastersData, AppMastersDataRequest, ReplayFromTimestampWindowTrailingEdge} +import org.apache.gearpump.cluster.MasterToClient._ import org.apache.gearpump.cluster._ import org.apache.gearpump.cluster.master.MasterProxy -import org.apache.gearpump.jarstore.{JarStoreClient, JarStoreServer} +import org.apache.gearpump.jarstore.JarStoreClient import org.apache.gearpump.util.Constants._ import org.apache.gearpump.util.{ActorUtil, Constants, LogUtil, Util} +import org.slf4j.Logger + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.collection.JavaConverters._ +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} +import scala.util.{Failure, Success, Try} /** * ClientContext is a user facing util to submit/manage an application. @@ -43,7 +45,6 @@ import org.apache.gearpump.util.{ActorUtil, Constants, LogUtil, Util} * TODO: add interface to query master here */ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { - def this(system: ActorSystem) = { this(system.settings.config, system, null) } @@ -53,20 +54,20 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { } private val LOG: Logger = LogUtil.getLogger(getClass) - private implicit val timeout = Timeout(5, TimeUnit.SECONDS) - implicit val system = Option(sys).getOrElse(ActorSystem(s"client${Util.randInt()}", config)) LOG.info(s"Starting system ${system.name}") - val shouldCleanupSystem = Option(sys).isEmpty - private val jarStoreClient = new JarStoreClient(config, system) + private val masterClientTimeout = { + val timeout = Try(config.getInt(Constants.GEARPUMP_MASTERCLIENT_TIMEOUT)).getOrElse(90) + Timeout(timeout, TimeUnit.SECONDS) + } private lazy val master: ActorRef = { val masters = config.getStringList(Constants.GEARPUMP_CLUSTER_MASTERS).asScala .flatMap(Util.parseHostList) val master = Option(_master).getOrElse(system.actorOf(MasterProxy.props(masters), s"masterproxy${system.name}")) - LOG.info(s"Creating master proxy ${master} for master list: $masters") + LOG.info(s"Creating master proxy $master for master list: $masters") master } @@ -75,26 +76,25 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { * defined. Otherwise, it assumes the jar is on the target runtime classpath, thus will * not send the jar across the wire. */ - def submit(app: Application): Int = { + def submit(app: Application): RunningApplication = { submit(app, System.getProperty(GEARPUMP_APP_JAR)) } - def submit(app: Application, jar: String): Int = { - submit(app, jar, getExecutorNum()) + def submit(app: Application, jar: String): RunningApplication = { + submit(app, jar, getExecutorNum) } - def submit(app: Application, jar: String, executorNum: Int): Int = { - val client = getMasterClient + def submit(app: Application, jar: String, executorNum: Int): RunningApplication = { val appName = checkAndAddNamePrefix(app.name, System.getProperty(GEARPUMP_APP_NAME_PREFIX)) val submissionConfig = getSubmissionConfig(config) .withValue(APPLICATION_EXECUTOR_NUMBER, ConfigValueFactory.fromAnyRef(executorNum)) val appDescription = AppDescription(appName, app.appMaster.getName, app.userConfig, submissionConfig) val appJar = Option(jar).map(loadFile) - client.submitApplication(appDescription, appJar) + submitApplication(SubmitApplication(appDescription, appJar)) } - private def getExecutorNum(): Int = { + private def getExecutorNum: Int = { Try(System.getProperty(APPLICATION_EXECUTOR_NUMBER).toInt).getOrElse(1) } @@ -102,8 +102,11 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { ClusterConfig.filterOutDefaultConfig(config) } + def listApps: AppMastersData = { + ActorUtil.askActor[AppMastersData](master, AppMastersDataRequest, masterClientTimeout) + } + def replayFromTimestampWindowTrailingEdge(appId: Int): ReplayApplicationResult = { - import scala.concurrent.ExecutionContext.Implicits.global val result = Await.result( ActorUtil.askAppMaster[ReplayApplicationResult](master, appId, ReplayFromTimestampWindowTrailingEdge(appId)), Duration.Inf) @@ -111,27 +114,29 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { } def askAppMaster[T](appId: Int, msg: Any): Future[T] = { - import scala.concurrent.ExecutionContext.Implicits.global ActorUtil.askAppMaster[T](master, appId, msg) } - def listApps: AppMastersData = { - val client = getMasterClient - client.listApplications - } - def shutdown(appId: Int): Unit = { - val client = getMasterClient - client.shutdownApplication(appId) + val result = ActorUtil.askActor[ShutdownApplicationResult](master, + ShutdownApplication(appId), masterClientTimeout) + result.appId match { + case Success(_) => + case Failure(ex) => throw ex + } } def resolveAppID(appId: Int): ActorRef = { - val client = getMasterClient - client.resolveAppId(appId) + val result = ActorUtil.askActor[ResolveAppIdResult](master, + ResolveAppId(appId), masterClientTimeout) + result.appMaster match { + case Success(appMaster) => appMaster + case Failure(ex) => throw ex + } } def close(): Unit = { - if (shouldCleanupSystem) { + if (sys == null) { LOG.info(s"Shutting down system ${system.name}") system.terminate() } @@ -139,6 +144,10 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { private def loadFile(jarPath: String): AppJar = { val jarFile = new java.io.File(jarPath) + if (!jarFile.exists()) { + val error = s"File $jarPath does not exist and cannot submit application" + throw new Exception(error) + } Util.uploadJar(jarFile, jarStoreClient) } @@ -157,9 +166,18 @@ class ClientContext(config: Config, sys: ActorSystem, _master: ActorRef) { fullName } - private def getMasterClient: MasterClient = { - val timeout = Try(config.getInt(Constants.GEARPUMP_MASTERCLIENT_TIMEOUT)).getOrElse(90) - new MasterClient(master, akka.util.Timeout(timeout, TimeUnit.SECONDS)) + private def submitApplication(submitApplication: SubmitApplication): RunningApplication = { + val result = ActorUtil.askActor[SubmitApplicationResult](master, + submitApplication, masterClientTimeout) + val application = result.appId match { + case Success(appId) => + // scalastyle:off println + Console.println(s"Submit application succeed. The application id is $appId") + // scalastyle:on println + new RunningApplication(appId, master, masterClientTimeout) + case Failure(ex) => throw ex + } + application } } http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/core/src/main/scala/org/apache/gearpump/cluster/client/MasterClient.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/cluster/client/MasterClient.scala b/core/src/main/scala/org/apache/gearpump/cluster/client/MasterClient.scala deleted file mode 100644 index 77ebedf..0000000 --- a/core/src/main/scala/org/apache/gearpump/cluster/client/MasterClient.scala +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.gearpump.cluster.client - -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, Future} -import scala.util.{Failure, Success} - -import akka.actor.ActorRef -import akka.pattern.ask -import akka.util.Timeout - -import org.apache.gearpump.cluster.ClientToMaster._ -import org.apache.gearpump.cluster.MasterToAppMaster.{AppMastersData, AppMastersDataRequest} -import org.apache.gearpump.cluster.MasterToClient.{ResolveAppIdResult, ShutdownApplicationResult, SubmitApplicationResult} -import org.apache.gearpump.cluster.{AppDescription, AppJar} - -/** - * Client to inter-operate with Master node. - * - * NOTE: Stateless, thread safe - */ -class MasterClient(master: ActorRef, timeout: Timeout) { - implicit val masterClientTimeout = timeout - - def submitApplication(app: AppDescription, appJar: Option[AppJar]): Int = { - val result = Await.result( - (master ? SubmitApplication(app, appJar)).asInstanceOf[Future[SubmitApplicationResult]], - Duration.Inf) - val appId = result.appId match { - case Success(appId) => - // scalastyle:off println - Console.println(s"Submit application succeed. The application id is $appId") - // scalastyle:on println - appId - case Failure(ex) => throw ex - } - appId - } - - def resolveAppId(appId: Int): ActorRef = { - val result = Await.result( - (master ? ResolveAppId(appId)).asInstanceOf[Future[ResolveAppIdResult]], Duration.Inf) - result.appMaster match { - case Success(appMaster) => appMaster - case Failure(ex) => throw ex - } - } - - def shutdownApplication(appId: Int): Unit = { - val result = Await.result( - (master ? ShutdownApplication(appId)).asInstanceOf[Future[ShutdownApplicationResult]], - Duration.Inf) - result.appId match { - case Success(_) => - case Failure(ex) => throw ex - } - } - - def listApplications: AppMastersData = { - val result = Await.result( - (master ? AppMastersDataRequest).asInstanceOf[Future[AppMastersData]], Duration.Inf) - result - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/core/src/main/scala/org/apache/gearpump/cluster/client/RunningApplication.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/cluster/client/RunningApplication.scala b/core/src/main/scala/org/apache/gearpump/cluster/client/RunningApplication.scala new file mode 100644 index 0000000..153c824 --- /dev/null +++ b/core/src/main/scala/org/apache/gearpump/cluster/client/RunningApplication.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gearpump.cluster.client + +import akka.pattern.ask +import akka.actor.ActorRef +import akka.util.Timeout +import org.apache.gearpump.cluster.ClientToMaster.{ResolveAppId, ShutdownApplication} +import org.apache.gearpump.cluster.MasterToClient.{ResolveAppIdResult, ShutdownApplicationResult} +import org.apache.gearpump.util.ActorUtil + +import scala.concurrent.Future +import scala.util.{Failure, Success} +import scala.concurrent.ExecutionContext.Implicits.global + +class RunningApplication(val appId: Int, master: ActorRef, timeout: Timeout) { + lazy val appMaster: Future[ActorRef] = resolveAppMaster(appId) + + def shutDown(): Unit = { + val result = ActorUtil.askActor[ShutdownApplicationResult](master, + ShutdownApplication(appId), timeout) + result.appId match { + case Success(_) => + case Failure(ex) => throw ex + } + } + + def askAppMaster[T](msg: Any): Future[T] = { + appMaster.flatMap(_.ask(msg)(timeout).asInstanceOf[Future[T]]) + } + + private def resolveAppMaster(appId: Int): Future[ActorRef] = { + master.ask(ResolveAppId(appId))(timeout). + asInstanceOf[Future[ResolveAppIdResult]].map(_.appMaster.get) + } +} + http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/cc0578e5/core/src/main/scala/org/apache/gearpump/cluster/embedded/EmbeddedCluster.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/gearpump/cluster/embedded/EmbeddedCluster.scala b/core/src/main/scala/org/apache/gearpump/cluster/embedded/EmbeddedCluster.scala new file mode 100644 index 0000000..9bde4d1 --- /dev/null +++ b/core/src/main/scala/org/apache/gearpump/cluster/embedded/EmbeddedCluster.scala @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gearpump.cluster.embedded + +import scala.collection.JavaConverters._ +import scala.concurrent.Await +import scala.concurrent.duration.Duration + +import akka.actor.{ActorRef, ActorSystem, Props} +import com.typesafe.config.{Config, ConfigValueFactory} + +import org.apache.gearpump.cluster.ClusterConfig +import org.apache.gearpump.cluster.client.ClientContext +import org.apache.gearpump.cluster.master.{Master => MasterActor} +import org.apache.gearpump.cluster.worker.{Worker => WorkerActor} +import org.apache.gearpump.util.Constants.{GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, GEARPUMP_CLUSTER_MASTERS, GEARPUMP_METRIC_ENABLED, MASTER} +import org.apache.gearpump.util.{LogUtil, Util} + +/** + * Create a in-process cluster with single worker + */ +class EmbeddedCluster(inputConfig: Config) { + + private val workerCount: Int = 1 + private var _master: ActorRef = null + private var _system: ActorSystem = null + private var _config: Config = null + + private val LOG = LogUtil.getLogger(getClass) + + def start(): Unit = { + val port = Util.findFreePort().get + val akkaConf = getConfig(inputConfig, port) + _config = akkaConf + val system = ActorSystem(MASTER, akkaConf) + + val master = system.actorOf(Props[MasterActor], MASTER) + + 0.until(workerCount).foreach { id => + system.actorOf(Props(classOf[WorkerActor], master), classOf[WorkerActor].getSimpleName + id) + } + this._master = master + this._system = system + + LOG.info("=================================") + LOG.info("Local Cluster is started at: ") + LOG.info(s" 127.0.0.1:$port") + LOG.info(s"To see UI, run command: services -master 127.0.0.1:$port") + } + + private def getConfig(inputConfig: Config, port: Int): Config = { + val config = inputConfig. + withValue("akka.remote.netty.tcp.port", ConfigValueFactory.fromAnyRef(port)). + withValue(GEARPUMP_CLUSTER_MASTERS, + ConfigValueFactory.fromIterable(List(s"127.0.0.1:$port").asJava)). + withValue(GEARPUMP_CLUSTER_EXECUTOR_WORKER_SHARE_SAME_PROCESS, + ConfigValueFactory.fromAnyRef(true)). + withValue(GEARPUMP_METRIC_ENABLED, ConfigValueFactory.fromAnyRef(true)). + withValue("akka.actor.provider", + ConfigValueFactory.fromAnyRef("akka.cluster.ClusterActorRefProvider")) + config + } + + def newClientContext: ClientContext = { + ClientContext(_config, _system, _master) + } + + def stop(): Unit = { + _system.stop(_master) + _system.terminate() + Await.result(_system.whenTerminated, Duration.Inf) + } +} + +object EmbeddedCluster { + def apply(): EmbeddedCluster = { + new EmbeddedCluster(ClusterConfig.master()) + } +} \ No newline at end of file
