STORM-697: Merged in master, resolved conflicts
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/47f4a3c3 Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/47f4a3c3 Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/47f4a3c3 Branch: refs/heads/master Commit: 47f4a3c3d3396b35928578e36b27dbcc49f00752 Parents: 472567e 93043e6 Author: matt.tieman <matt.tie...@inin.com> Authored: Wed Oct 14 17:31:08 2015 -0400 Committer: matt.tieman <matt.tie...@inin.com> Committed: Wed Oct 14 17:31:08 2015 -0400 ---------------------------------------------------------------------- CHANGELOG.md | 14 + README.markdown | 1 + STORM-UI-REST-API.md | 4 +- bin/storm | 20 +- bin/storm.py | 35 +- conf/defaults.yaml | 18 + dev-tools/storm-merge.py | 2 +- docs/DYNAMIC_LOG_LEVEL_SETTINGS.md | 41 + docs/images/dynamic_log_level_settings_1.png | Bin 0 -> 93689 bytes docs/images/dynamic_log_level_settings_2.png | Bin 0 -> 78785 bytes examples/storm-starter/pom.xml | 7 + .../storm/starter/MultipleLoggerTopology.java | 105 + .../starter/ResourceAwareExampleTopology.java | 87 + .../bolt/IntermediateRankingsBoltTest.java | 2 +- .../starter/bolt/RollingCountBoltTest.java | 2 +- .../starter/bolt/TotalRankingsBoltTest.java | 2 +- .../storm/starter/tools/MockTupleHelpers.java | 40 - external/storm-hdfs/pom.xml | 15 + .../apache/storm/hdfs/bolt/TestHdfsBolt.java | 18 +- external/storm-hive/pom.xml | 7 + .../apache/storm/hive/bolt/TestHiveBolt.java | 56 +- .../src/jvm/storm/kafka/PartitionManager.java | 20 +- storm-core/pom.xml | 12 + storm-core/src/clj/backtype/storm/cluster.clj | 28 +- .../backtype/storm/command/set_log_level.clj | 75 + storm-core/src/clj/backtype/storm/converter.clj | 4 +- .../src/clj/backtype/storm/daemon/common.clj | 4 +- .../src/clj/backtype/storm/daemon/drpc.clj | 26 +- .../src/clj/backtype/storm/daemon/nimbus.clj | 232 +- .../clj/backtype/storm/daemon/supervisor.clj | 13 +- .../src/clj/backtype/storm/daemon/worker.clj | 134 +- storm-core/src/clj/backtype/storm/log.clj | 12 +- storm-core/src/clj/backtype/storm/stats.clj | 1282 ++- storm-core/src/clj/backtype/storm/ui/core.clj | 1050 +-- .../src/clj/backtype/storm/ui/helpers.clj | 30 - storm-core/src/jvm/backtype/storm/Config.java | 70 + .../jvm/backtype/storm/ConfigValidation.java | 22 + .../src/jvm/backtype/storm/StormSubmitter.java | 35 +- .../storm/coordination/CoordinatedBolt.java | 2 +- .../backtype/storm/generated/Assignment.java | 194 +- .../storm/generated/BoltAggregateStats.java | 704 ++ .../jvm/backtype/storm/generated/BoltStats.java | 442 +- .../storm/generated/ClusterSummary.java | 110 +- .../storm/generated/ClusterWorkerHeartbeat.java | 54 +- .../storm/generated/CommonAggregateStats.java | 902 ++ .../generated/ComponentAggregateStats.java | 752 ++ .../storm/generated/ComponentPageInfo.java | 2194 +++++ .../backtype/storm/generated/ComponentType.java | 62 + .../backtype/storm/generated/Credentials.java | 46 +- .../storm/generated/ExecutorAggregateStats.java | 526 ++ .../backtype/storm/generated/ExecutorStats.java | 170 +- .../storm/generated/LSApprovedWorkers.java | 46 +- .../generated/LSSupervisorAssignments.java | 50 +- .../storm/generated/LSWorkerHeartbeat.java | 38 +- .../storm/generated/LocalAssignment.java | 38 +- .../storm/generated/LocalStateData.java | 50 +- .../jvm/backtype/storm/generated/LogConfig.java | 475 + .../jvm/backtype/storm/generated/LogLevel.java | 836 ++ .../storm/generated/LogLevelAction.java | 65 + .../jvm/backtype/storm/generated/Nimbus.java | 8830 +++++++++++++----- .../jvm/backtype/storm/generated/NodeInfo.java | 34 +- .../storm/generated/RebalanceOptions.java | 46 +- .../storm/generated/SpecificAggregateStats.java | 387 + .../storm/generated/SpoutAggregateStats.java | 407 + .../backtype/storm/generated/SpoutStats.java | 254 +- .../jvm/backtype/storm/generated/StormBase.java | 94 +- .../storm/generated/SupervisorInfo.java | 276 +- .../storm/generated/SupervisorSummary.java | 168 +- .../backtype/storm/generated/TopologyInfo.java | 162 +- .../storm/generated/TopologyPageInfo.java | 2003 ++++ .../backtype/storm/generated/TopologyStats.java | 1094 +++ .../AbstractDNSToSwitchMapping.java | 95 + .../networktopography/DNSToSwitchMapping.java | 50 + .../DefaultRackDNSToSwitchMapping.java | 35 + .../jvm/backtype/storm/nimbus/NimbusInfo.java | 8 + .../jvm/backtype/storm/scheduler/Cluster.java | 46 +- .../storm/scheduler/SupervisorDetails.java | 61 +- .../backtype/storm/scheduler/Topologies.java | 15 +- .../storm/scheduler/TopologyDetails.java | 330 +- .../storm/scheduler/resource/Component.java | 54 + .../storm/scheduler/resource/RAS_Node.java | 547 ++ .../resource/ResourceAwareScheduler.java | 149 + .../storm/scheduler/resource/ResourceUtils.java | 133 + .../resource/strategies/IStrategy.java | 37 + .../strategies/ResourceAwareStrategy.java | 480 + .../storm/security/auth/ReqContext.java | 7 + .../auth/authorizer/SimpleACLAuthorizer.java | 15 +- .../auth/kerberos/ServerCallbackHandler.java | 2 + .../AlternateRackDNSToSwitchMapping.java | 48 + .../topology/BaseConfigurationDeclarer.java | 31 +- .../ComponentConfigurationDeclarer.java | 3 + .../src/jvm/backtype/storm/tuple/Tuple.java | 8 + .../src/jvm/backtype/storm/tuple/TupleImpl.java | 7 +- .../backtype/storm/utils/DisruptorQueue.java | 11 +- .../src/jvm/backtype/storm/utils/Utils.java | 21 +- .../trident/topology/TridentBoltExecutor.java | 4 +- storm-core/src/py/storm/Nimbus-remote | 28 + storm-core/src/py/storm/Nimbus.py | 1451 ++- storm-core/src/py/storm/ttypes.py | 2577 ++++- storm-core/src/storm.thrift | 127 + storm-core/src/ui/public/css/style.css | 8 + storm-core/src/ui/public/js/script.js | 20 + storm-core/src/ui/public/js/visualization.js | 92 +- .../templates/component-page-template.html | 2 +- .../templates/topology-page-template.html | 120 +- storm-core/src/ui/public/topology.html | 149 +- .../test/clj/backtype/storm/cluster_test.clj | 4 +- .../test/clj/backtype/storm/nimbus_test.clj | 56 +- .../scheduler/multitenant_scheduler_test.clj | 34 +- .../scheduler/resource_aware_scheduler_test.clj | 669 ++ .../test/clj/backtype/storm/scheduler_test.clj | 3 +- .../auth/DefaultHttpCredentialsPlugin_test.clj | 40 +- .../test/clj/backtype/storm/supervisor_test.clj | 14 + .../test/clj/backtype/storm/worker_test.clj | 179 +- .../backtype/storm/utils/MockTupleHelpers.java | 40 + 115 files changed, 27909 insertions(+), 4937 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/47f4a3c3/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java ---------------------------------------------------------------------- diff --cc external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java index ed36fbe,10405fa..980ed8b --- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java +++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java @@@ -135,23 -135,16 +135,23 @@@ public class PartitionManager if (toEmit == null) { return EmitState.NO_EMITTED; } - - Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg); ++ + Iterable<List<Object>> tups; + if (_spoutConfig.tupleMetaData) { + tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg, _partition, toEmit.offset); + } else { + tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg); + } + - if (tups != null) { - if(_spoutConfig.topicAsStreamId) { - for (List<Object> tup : tups) { - collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset)); - } - } else { - for (List<Object> tup : tups) { - collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); - } + if ((tups != null) && tups.iterator().hasNext()) { + if(_spoutConfig.topicAsStreamId) { + for (List<Object> tup : tups) { + collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset)); + } + } else { + for (List<Object> tup : tups) { + collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset)); + } } break; } else {
