Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/STORM-UI-REST-API.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/STORM-UI-REST-API.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/STORM-UI-REST-API.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/STORM-UI-REST-API.md Fri Mar 18 17:56:59 2016 @@ -582,7 +582,7 @@ Sample response: "errorPort": 6701, "errorWorkerLogLink": "http://10.11.1.7:8000/log?file=worker-6701.log";, "errorLapsedSecs": 16, - "error": "java.lang.RuntimeException: java.lang.StringIndexOutOfBoundsException: Some Error\n\tat backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:128)\n\tat backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:99)\n\tat backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:80)\n\tat backtype...more.." + "error": "java.lang.RuntimeException: java.lang.StringIndexOutOfBoundsException: Some Error\n\tat org.apache.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:128)\n\tat org.apache.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:99)\n\tat org.apache.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:80)\n\tat backtype...more.." }], "topologyId": "WordCount3-1-1402960825", "tasks": 5, @@ -1012,6 +1012,6 @@ Sample response: ```json { "error": "Internal Server Error", - "errorMessage": "java.lang.NullPointerException\n\tat clojure.core$name.invoke(core.clj:1505)\n\tat backtype.storm.ui.core$component_page.invoke(core.clj:752)\n\tat backtype.storm.ui.core$fn__7766.invoke(core.clj:782)\n\tat compojure.core$make_route$fn__5755.invoke(core.clj:93)\n\tat compojure.core$if_route$fn__5743.invoke(core.clj:39)\n\tat compojure.core$if_method$fn__5736.invoke(core.clj:24)\n\tat compojure.core$routing$fn__5761.invoke(core.clj:106)\n\tat clojure.core$some.invoke(core.clj:2443)\n\tat compojure.core$routing.doInvoke(core.clj:106)\n\tat clojure.lang.RestFn.applyTo(RestFn.java:139)\n\tat clojure.core$apply.invoke(core.clj:619)\n\tat compojure.core$routes$fn__5765.invoke(core.clj:111)\n\tat ring.middleware.reload$wrap_reload$fn__6880.invoke(reload.clj:14)\n\tat backtype.storm.ui.core$catch_errors$fn__7800.invoke(core.clj:836)\n\tat ring.middleware.keyword_params$wrap_keyword_params$fn__6319.invoke(keyword_params.clj:27)\n\tat ring.middleware.nested_params$wrap_nest ed_params$fn__6358.invoke(nested_params.clj:65)\n\tat ring.middleware.params$wrap_params$fn__6291.invoke(params.clj:55)\n\tat ring.middleware.multipart_params$wrap_multipart_params$fn__6386.invoke(multipart_params.clj:103)\n\tat ring.middleware.flash$wrap_flash$fn__6675.invoke(flash.clj:14)\n\tat ring.middleware.session$wrap_session$fn__6664.invoke(session.clj:43)\n\tat ring.middleware.cookies$wrap_cookies$fn__6595.invoke(cookies.clj:160)\n\tat ring.adapter.jetty$proxy_handler$fn__6112.invoke(jetty.clj:16)\n\tat ring.adapter.jetty.proxy$org.mortbay.jetty.handler.AbstractHandler$0.handle(Unknown Source)\n\tat org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)\n\tat org.mortbay.jetty.Server.handle(Server.java:326)\n\tat org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)\n\tat org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)\n\tat org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)\n\tat org.mortb ay.jetty.HttpParser.parseAvailable(HttpParser.java:212)\n\tat org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)\n\tat org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)\n\tat org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)\n" + "errorMessage": "java.lang.NullPointerException\n\tat clojure.core$name.invoke(core.clj:1505)\n\tat org.apache.storm.ui.core$component_page.invoke(core.clj:752)\n\tat org.apache.storm.ui.core$fn__7766.invoke(core.clj:782)\n\tat compojure.core$make_route$fn__5755.invoke(core.clj:93)\n\tat compojure.core$if_route$fn__5743.invoke(core.clj:39)\n\tat compojure.core$if_method$fn__5736.invoke(core.clj:24)\n\tat compojure.core$routing$fn__5761.invoke(core.clj:106)\n\tat clojure.core$some.invoke(core.clj:2443)\n\tat compojure.core$routing.doInvoke(core.clj:106)\n\tat clojure.lang.RestFn.applyTo(RestFn.java:139)\n\tat clojure.core$apply.invoke(core.clj:619)\n\tat compojure.core$routes$fn__5765.invoke(core.clj:111)\n\tat ring.middleware.reload$wrap_reload$fn__6880.invoke(reload.clj:14)\n\tat org.apache.storm.ui.core$catch_errors$fn__7800.invoke(core.clj:836)\n\tat ring.middleware.keyword_params$wrap_keyword_params$fn__6319.invoke(keyword_params.clj:27)\n\tat ring.middleware.nested_params$wra p_nested_params$fn__6358.invoke(nested_params.clj:65)\n\tat ring.middleware.params$wrap_params$fn__6291.invoke(params.clj:55)\n\tat ring.middleware.multipart_params$wrap_multipart_params$fn__6386.invoke(multipart_params.clj:103)\n\tat ring.middleware.flash$wrap_flash$fn__6675.invoke(flash.clj:14)\n\tat ring.middleware.session$wrap_session$fn__6664.invoke(session.clj:43)\n\tat ring.middleware.cookies$wrap_cookies$fn__6595.invoke(cookies.clj:160)\n\tat ring.adapter.jetty$proxy_handler$fn__6112.invoke(jetty.clj:16)\n\tat ring.adapter.jetty.proxy$org.mortbay.jetty.handler.AbstractHandler$0.handle(Unknown Source)\n\tat org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)\n\tat org.mortbay.jetty.Server.handle(Server.java:326)\n\tat org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)\n\tat org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)\n\tat org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)\n\tat org .mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)\n\tat org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)\n\tat org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)\n\tat org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)\n" } ```
Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Serialization.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Serialization.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Serialization.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Serialization.md Fri Mar 18 17:56:59 2016 @@ -41,7 +41,7 @@ topology.kryo.register: `com.mycompany.CustomType1` and `com.mycompany.CustomType3` will use the `FieldsSerializer`, whereas `com.mycompany.CustomType2` will use `com.mycompany.serializer.CustomType2Serializer` for serialization. -Storm provides helpers for registering serializers in a topology config. The [Config](javadocs/backtype/storm/Config.html) class has a method called `registerSerialization` that takes in a registration to add to the config. +Storm provides helpers for registering serializers in a topology config. The [Config](javadocs/org/apache/storm/Config.html) class has a method called `registerSerialization` that takes in a registration to add to the config. There's an advanced config called `Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS`. If you set this to true, Storm will ignore any serializations that are registered but do not have their code available on the classpath. Otherwise, Storm will throw errors when it can't find a serialization. This is useful if you run many topologies on a cluster that each have different serializations, but you want to declare all the serializations across all topologies in the `storm.yaml` files. Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Structure-of-the-codebase.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Structure-of-the-codebase.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Structure-of-the-codebase.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Structure-of-the-codebase.md Fri Mar 18 17:56:59 2016 @@ -25,8 +25,8 @@ Spouts and bolts have the same Thrift de The `ComponentObject` defines the implementation for the bolt. It can be one of three types: -1. A serialized java object (that implements [IBolt]({{page.git-blob-base}}/storm-core/src/jvm/backtype/storm/task/IBolt.java)) -2. A `ShellComponent` object that indicates the implementation is in another language. Specifying a bolt this way will cause Storm to instantiate a [ShellBolt]({{page.git-blob-base}}/storm-core/src/jvm/backtype/storm/task/ShellBolt.java) object to handle the communication between the JVM-based worker process and the non-JVM-based implementation of the component. +1. A serialized java object (that implements [IBolt]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/task/IBolt.java)) +2. A `ShellComponent` object that indicates the implementation is in another language. Specifying a bolt this way will cause Storm to instantiate a [ShellBolt]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/task/ShellBolt.java) object to handle the communication between the JVM-based worker process and the non-JVM-based implementation of the component. 3. A `JavaObject` structure which tells Storm the classname and constructor arguments to use to instantiate that bolt. This is useful if you want to define a topology in a non-JVM language. This way, you can make use of JVM-based spouts and bolts without having to create and serialize a Java object yourself. `ComponentCommon` defines everything else for this component. This includes: @@ -36,107 +36,107 @@ The `ComponentObject` defines the implem 3. The parallelism for this component 4. The component-specific [configuration](Configuration.html) for this component -Note that the structure spouts also have a `ComponentCommon` field, and so spouts can also have declarations to consume other input streams. Yet the Storm Java API does not provide a way for spouts to consume other streams, and if you put any input declarations there for a spout you would get an error when you tried to submit the topology. The reason that spouts have an input declarations field is not for users to use, but for Storm itself to use. Storm adds implicit streams and bolts to the topology to set up the [acking framework](https://github.com/apache/storm/wiki/Guaranteeing-message-processing), and two of these implicit streams are from the acker bolt to each spout in the topology. The acker sends "ack" or "fail" messages along these streams whenever a tuple tree is detected to be completed or failed. The code that transforms the user's topology into the runtime topology is located [here]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/daemon/common.clj#L279). +Note that the structure spouts also have a `ComponentCommon` field, and so spouts can also have declarations to consume other input streams. Yet the Storm Java API does not provide a way for spouts to consume other streams, and if you put any input declarations there for a spout you would get an error when you tried to submit the topology. The reason that spouts have an input declarations field is not for users to use, but for Storm itself to use. Storm adds implicit streams and bolts to the topology to set up the [acking framework](https://github.com/apache/storm/wiki/Guaranteeing-message-processing), and two of these implicit streams are from the acker bolt to each spout in the topology. The acker sends "ack" or "fail" messages along these streams whenever a tuple tree is detected to be completed or failed. The code that transforms the user's topology into the runtime topology is located [here]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/daemon/common.clj#L279). ### Java interfaces The interfaces for Storm are generally specified as Java interfaces. The main interfaces are: -1. [IRichBolt](javadocs/backtype/storm/topology/IRichBolt.html) -2. [IRichSpout](javadocs/backtype/storm/topology/IRichSpout.html) -3. [TopologyBuilder](javadocs/backtype/storm/topology/TopologyBuilder.html) +1. [IRichBolt](javadocs/org/apache/storm/topology/IRichBolt.html) +2. [IRichSpout](javadocs/org/apache/storm/topology/IRichSpout.html) +3. [TopologyBuilder](javadocs/org/apache/storm/topology/TopologyBuilder.html) The strategy for the majority of the interfaces is to: 1. Specify the interface using a Java interface 2. Provide a base class that provides default implementations when appropriate -You can see this strategy at work with the [BaseRichSpout](javadocs/backtype/storm/topology/base/BaseRichSpout.html) class. +You can see this strategy at work with the [BaseRichSpout](javadocs/org/apache/storm/topology/base/BaseRichSpout.html) class. Spouts and bolts are serialized into the Thrift definition of the topology as described above. -One subtle aspect of the interfaces is the difference between `IBolt` and `ISpout` vs. `IRichBolt` and `IRichSpout`. The main difference between them is the addition of the `declareOutputFields` method in the "Rich" versions of the interfaces. The reason for the split is that the output fields declaration for each output stream needs to be part of the Thrift struct (so it can be specified from any language), but as a user you want to be able to declare the streams as part of your class. What `TopologyBuilder` does when constructing the Thrift representation is call `declareOutputFields` to get the declaration and convert it into the Thrift structure. The conversion happens [at this portion]({{page.git-blob-base}}/storm-core/src/jvm/backtype/storm/topology/TopologyBuilder.java#L205) of the `TopologyBuilder` code. +One subtle aspect of the interfaces is the difference between `IBolt` and `ISpout` vs. `IRichBolt` and `IRichSpout`. The main difference between them is the addition of the `declareOutputFields` method in the "Rich" versions of the interfaces. The reason for the split is that the output fields declaration for each output stream needs to be part of the Thrift struct (so it can be specified from any language), but as a user you want to be able to declare the streams as part of your class. What `TopologyBuilder` does when constructing the Thrift representation is call `declareOutputFields` to get the declaration and convert it into the Thrift structure. The conversion happens [at this portion]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/topology/TopologyBuilder.java#L205) of the `TopologyBuilder` code. ### Implementation Specifying all the functionality via Java interfaces ensures that every feature of Storm is available via Java. Moreso, the focus on Java interfaces ensures that the user experience from Java-land is pleasant as well. -The implementation of Storm, on the other hand, is primarily in Clojure. While the codebase is about 50% Java and 50% Clojure in terms of LOC, most of the implementation logic is in Clojure. There are two notable exceptions to this, and that is the [DRPC](https://github.com/apache/storm/wiki/Distributed-RPC) and [transactional topologies](https://github.com/apache/storm/wiki/Transactional-topologies) implementations. These are implemented purely in Java. This was done to serve as an illustration for how to implement a higher level abstraction on Storm. The DRPC and transactional topologies implementations are in the [backtype.storm.coordination]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/coordination), [backtype.storm.drpc]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/drpc), and [backtype.storm.transactional]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/transactional) packages. +The implementation of Storm, on the other hand, is primarily in Clojure. While the codebase is about 50% Java and 50% Clojure in terms of LOC, most of the implementation logic is in Clojure. There are two notable exceptions to this, and that is the [DRPC](https://github.com/apache/storm/wiki/Distributed-RPC) and [transactional topologies](https://github.com/apache/storm/wiki/Transactional-topologies) implementations. These are implemented purely in Java. This was done to serve as an illustration for how to implement a higher level abstraction on Storm. The DRPC and transactional topologies implementations are in the [org.apache.storm.coordination]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/coordination), [org.apache.storm.drpc]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/drpc), and [org.apache.storm.transactional]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/transactional) packages. Here's a summary of the purpose of the main Java packages and Clojure namespace: #### Java packages -[backtype.storm.coordination]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/coordination): Implements the pieces required to coordinate batch-processing on top of Storm, which both DRPC and transactional topologies use. `CoordinatedBolt` is the most important class here. +[org.apache.storm.coordination]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/coordination): Implements the pieces required to coordinate batch-processing on top of Storm, which both DRPC and transactional topologies use. `CoordinatedBolt` is the most important class here. -[backtype.storm.drpc]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/drpc): Implementation of the DRPC higher level abstraction +[org.apache.storm.drpc]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/drpc): Implementation of the DRPC higher level abstraction -[backtype.storm.generated]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/generated): The generated Thrift code for Storm (generated using [this fork](https://github.com/nathanmarz/thrift) of Thrift, which simply renames the packages to org.apache.thrift7 to avoid conflicts with other Thrift versions) +[org.apache.storm.generated]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/generated): The generated Thrift code for Storm (generated using [this fork](https://github.com/nathanmarz/thrift) of Thrift, which simply renames the packages to org.apache.thrift7 to avoid conflicts with other Thrift versions) -[backtype.storm.grouping]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/grouping): Contains interface for making custom stream groupings +[org.apache.storm.grouping]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/grouping): Contains interface for making custom stream groupings -[backtype.storm.hooks]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/hooks): Interfaces for hooking into various events in Storm, such as when tasks emit tuples, when tuples are acked, etc. User guide for hooks is [here](https://github.com/apache/storm/wiki/Hooks). +[org.apache.storm.hooks]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/hooks): Interfaces for hooking into various events in Storm, such as when tasks emit tuples, when tuples are acked, etc. User guide for hooks is [here](https://github.com/apache/storm/wiki/Hooks). -[backtype.storm.serialization]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/serialization): Implementation of how Storm serializes/deserializes tuples. Built on top of [Kryo](http://code.google.com/p/kryo/). +[org.apache.storm.serialization]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/serialization): Implementation of how Storm serializes/deserializes tuples. Built on top of [Kryo](http://code.google.com/p/kryo/). -[backtype.storm.spout]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/spout): Definition of spout and associated interfaces (like the `SpoutOutputCollector`). Also contains `ShellSpout` which implements the protocol for defining spouts in non-JVM languages. +[org.apache.storm.spout]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/spout): Definition of spout and associated interfaces (like the `SpoutOutputCollector`). Also contains `ShellSpout` which implements the protocol for defining spouts in non-JVM languages. -[backtype.storm.task]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/task): Definition of bolt and associated interfaces (like `OutputCollector`). Also contains `ShellBolt` which implements the protocol for defining bolts in non-JVM languages. Finally, `TopologyContext` is defined here as well, which is provided to spouts and bolts so they can get data about the topology and its execution at runtime. +[org.apache.storm.task]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/task): Definition of bolt and associated interfaces (like `OutputCollector`). Also contains `ShellBolt` which implements the protocol for defining bolts in non-JVM languages. Finally, `TopologyContext` is defined here as well, which is provided to spouts and bolts so they can get data about the topology and its execution at runtime. -[backtype.storm.testing]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/testing): Contains a variety of test bolts and utilities used in Storm's unit tests. +[org.apache.storm.testing]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/testing): Contains a variety of test bolts and utilities used in Storm's unit tests. -[backtype.storm.topology]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/topology): Java layer over the underlying Thrift structure to provide a clean, pure-Java API to Storm (users don't have to know about Thrift). `TopologyBuilder` is here as well as the helpful base classes for the different spouts and bolts. The slightly-higher level `IBasicBolt` interface is here, which is a simpler way to write certain kinds of bolts. +[org.apache.storm.topology]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/topology): Java layer over the underlying Thrift structure to provide a clean, pure-Java API to Storm (users don't have to know about Thrift). `TopologyBuilder` is here as well as the helpful base classes for the different spouts and bolts. The slightly-higher level `IBasicBolt` interface is here, which is a simpler way to write certain kinds of bolts. -[backtype.storm.transactional]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/transactional): Implementation of transactional topologies. +[org.apache.storm.transactional]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/transactional): Implementation of transactional topologies. -[backtype.storm.tuple]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/tuple): Implementation of Storm's tuple data model. +[org.apache.storm.tuple]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/tuple): Implementation of Storm's tuple data model. -[backtype.storm.utils]({{page.git-tree-base}}/storm-core/src/jvm/backtype/storm/tuple): Data structures and miscellaneous utilities used throughout the codebase. +[org.apache.storm.utils]({{page.git-tree-base}}/storm-core/src/jvm/org/apache/storm/tuple): Data structures and miscellaneous utilities used throughout the codebase. #### Clojure namespaces -[backtype.storm.bootstrap]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/bootstrap.clj): Contains a helpful macro to import all the classes and namespaces that are used throughout the codebase. +[org.apache.storm.bootstrap]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/bootstrap.clj): Contains a helpful macro to import all the classes and namespaces that are used throughout the codebase. -[backtype.storm.clojure]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/clojure.clj): Implementation of the Clojure DSL for Storm. +[org.apache.storm.clojure]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/clojure.clj): Implementation of the Clojure DSL for Storm. -[backtype.storm.cluster]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/cluster.clj): All Zookeeper logic used in Storm daemons is encapsulated in this file. This code manages how cluster state (like what tasks are running where, what spout/bolt each task runs as) is mapped to the Zookeeper "filesystem" API. +[org.apache.storm.cluster]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/cluster.clj): All Zookeeper logic used in Storm daemons is encapsulated in this file. This code manages how cluster state (like what tasks are running where, what spout/bolt each task runs as) is mapped to the Zookeeper "filesystem" API. -[backtype.storm.command.*]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/command): These namespaces implement various commands for the `storm` command line client. These implementations are very short. +[org.apache.storm.command.*]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/command): These namespaces implement various commands for the `storm` command line client. These implementations are very short. -[backtype.storm.config]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/config.clj): Implementation of config reading/parsing code for Clojure. Also has utility functions for determining what local path nimbus/supervisor/daemons should be using for various things. e.g. the `master-inbox` function will return the local path that Nimbus should use when jars are uploaded to it. +[org.apache.storm.config]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/config.clj): Implementation of config reading/parsing code for Clojure. Also has utility functions for determining what local path nimbus/supervisor/daemons should be using for various things. e.g. the `master-inbox` function will return the local path that Nimbus should use when jars are uploaded to it. -[backtype.storm.daemon.acker]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/daemon/acker.clj): Implementation of the "acker" bolt, which is a key part of how Storm guarantees data processing. +[org.apache.storm.daemon.acker]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/daemon/acker.clj): Implementation of the "acker" bolt, which is a key part of how Storm guarantees data processing. -[backtype.storm.daemon.common]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/daemon/common.clj): Implementation of common functions used in Storm daemons, like getting the id for a topology based on the name, mapping a user's topology into the one that actually executes (with implicit acking streams and acker bolt added - see `system-topology!` function), and definitions for the various heartbeat and other structures persisted by Storm. +[org.apache.storm.daemon.common]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/daemon/common.clj): Implementation of common functions used in Storm daemons, like getting the id for a topology based on the name, mapping a user's topology into the one that actually executes (with implicit acking streams and acker bolt added - see `system-topology!` function), and definitions for the various heartbeat and other structures persisted by Storm. -[backtype.storm.daemon.drpc]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/daemon/drpc.clj): Implementation of the DRPC server for use with DRPC topologies. +[org.apache.storm.daemon.drpc]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/daemon/drpc.clj): Implementation of the DRPC server for use with DRPC topologies. -[backtype.storm.daemon.nimbus]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/daemon/nimbus.clj): Implementation of Nimbus. +[org.apache.storm.daemon.nimbus]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/daemon/nimbus.clj): Implementation of Nimbus. -[backtype.storm.daemon.supervisor]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/daemon/supervisor.clj): Implementation of Supervisor. +[org.apache.storm.daemon.supervisor]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj): Implementation of Supervisor. -[backtype.storm.daemon.task]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/daemon/task.clj): Implementation of an individual task for a spout or bolt. Handles message routing, serialization, stats collection for the UI, as well as the spout-specific and bolt-specific execution implementations. +[org.apache.storm.daemon.task]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/daemon/task.clj): Implementation of an individual task for a spout or bolt. Handles message routing, serialization, stats collection for the UI, as well as the spout-specific and bolt-specific execution implementations. -[backtype.storm.daemon.worker]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/daemon/worker.clj): Implementation of a worker process (which will contain many tasks within). Implements message transferring and task launching. +[org.apache.storm.daemon.worker]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/daemon/worker.clj): Implementation of a worker process (which will contain many tasks within). Implements message transferring and task launching. -[backtype.storm.event]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/event.clj): Implements a simple asynchronous function executor. Used in various places in Nimbus and Supervisor to make functions execute in serial to avoid any race conditions. +[org.apache.storm.event]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/event.clj): Implements a simple asynchronous function executor. Used in various places in Nimbus and Supervisor to make functions execute in serial to avoid any race conditions. -[backtype.storm.log]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/log.clj): Defines the functions used to log messages to log4j. +[org.apache.storm.log]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/log.clj): Defines the functions used to log messages to log4j. -[backtype.storm.messaging.*]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/messaging): Defines a higher level interface to implementing point to point messaging. In local mode Storm uses in-memory Java queues to do this; on a cluster, it uses ZeroMQ. The generic interface is defined in protocol.clj. +[org.apache.storm.messaging.*]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/messaging): Defines a higher level interface to implementing point to point messaging. In local mode Storm uses in-memory Java queues to do this; on a cluster, it uses ZeroMQ. The generic interface is defined in protocol.clj. -[backtype.storm.stats]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/stats.clj): Implementation of stats rollup routines used when sending stats to ZK for use by the UI. Does things like windowed and rolling aggregations at multiple granularities. +[org.apache.storm.stats]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/stats.clj): Implementation of stats rollup routines used when sending stats to ZK for use by the UI. Does things like windowed and rolling aggregations at multiple granularities. -[backtype.storm.testing]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/testing.clj): Implementation of facilities used to test Storm topologies. Includes time simulation, `complete-topology` for running a fixed set of tuples through a topology and capturing the output, tracker topologies for having fine grained control over detecting when a cluster is "idle", and other utilities. +[org.apache.storm.testing]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/testing.clj): Implementation of facilities used to test Storm topologies. Includes time simulation, `complete-topology` for running a fixed set of tuples through a topology and capturing the output, tracker topologies for having fine grained control over detecting when a cluster is "idle", and other utilities. -[backtype.storm.thrift]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/thrift.clj): Clojure wrappers around the generated Thrift API to make working with Thrift structures more pleasant. +[org.apache.storm.thrift]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/thrift.clj): Clojure wrappers around the generated Thrift API to make working with Thrift structures more pleasant. -[backtype.storm.timer]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/timer.clj): Implementation of a background timer to execute functions in the future or on a recurring interval. Storm couldn't use the [Timer](http://docs.oracle.com/javase/1.4.2/docs/api/java/util/Timer.html) class because it needed integration with time simulation in order to be able to unit test Nimbus and the Supervisor. +[org.apache.storm.timer]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/timer.clj): Implementation of a background timer to execute functions in the future or on a recurring interval. Storm couldn't use the [Timer](http://docs.oracle.com/javase/1.4.2/docs/api/java/util/Timer.html) class because it needed integration with time simulation in order to be able to unit test Nimbus and the Supervisor. -[backtype.storm.ui.*]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/ui): Implementation of Storm UI. Completely independent from rest of code base and uses the Nimbus Thrift API to get data. +[org.apache.storm.ui.*]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/ui): Implementation of Storm UI. Completely independent from rest of code base and uses the Nimbus Thrift API to get data. -[backtype.storm.util]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/util.clj): Contains generic utility functions used throughout the code base. +[org.apache.storm.util]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/util.clj): Contains generic utility functions used throughout the code base. -[backtype.storm.zookeeper]({{page.git-blob-base}}/storm-core/src/clj/backtype/storm/zookeeper.clj): Clojure wrapper around the Zookeeper API and implements some "high-level" stuff like "mkdirs" and "delete-recursive". +[org.apache.storm.zookeeper]({{page.git-blob-base}}/storm-core/src/clj/org/apache/storm/zookeeper.clj): Clojure wrapper around the Zookeeper API and implements some "high-level" stuff like "mkdirs" and "delete-recursive". Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Transactional-topologies.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Transactional-topologies.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Transactional-topologies.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Transactional-topologies.md Fri Mar 18 17:56:59 2016 @@ -81,7 +81,7 @@ Finally, another thing to note is that t ## The basics through example -You build transactional topologies by using [TransactionalTopologyBuilder](javadocs/backtype/storm/transactional/TransactionalTopologyBuilder.html). Here's the transactional topology definition for a topology that computes the global count of tuples from the input stream. This code comes from [TransactionalGlobalCount]({{page.git-blob-base}}/examples/storm-starter/src/jvm/storm/starter/TransactionalGlobalCount.java) in storm-starter. +You build transactional topologies by using [TransactionalTopologyBuilder](javadocs/org/apache/storm/transactional/TransactionalTopologyBuilder.html). Here's the transactional topology definition for a topology that computes the global count of tuples from the input stream. This code comes from [TransactionalGlobalCount]({{page.git-blob-base}}/examples/storm-starter/src/jvm/storm/starter/TransactionalGlobalCount.java) in storm-starter. ```java MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH); @@ -130,9 +130,9 @@ public static class BatchCount extends B } ``` -A new instance of this object is created for every batch that's being processed. The actual bolt this runs within is called [BatchBoltExecutor](https://github.com/apache/storm/blob/0.7.0/src/jvm/backtype/storm/coordination/BatchBoltExecutor.java) and manages the creation and cleanup for these objects. +A new instance of this object is created for every batch that's being processed. The actual bolt this runs within is called [BatchBoltExecutor](https://github.com/apache/storm/blob/0.7.0/src/jvm/org/apache/storm/coordination/BatchBoltExecutor.java) and manages the creation and cleanup for these objects. -The `prepare` method parameterizes this batch bolt with the Storm config, the topology context, an output collector, and the id for this batch of tuples. In the case of transactional topologies, the id will be a [TransactionAttempt](javadocs/backtype/storm/transactional/TransactionAttempt.html) object. The batch bolt abstraction can be used in Distributed RPC as well which uses a different type of id for the batches. `BatchBolt` can actually be parameterized with the type of the id, so if you only intend to use the batch bolt for transactional topologies, you can extend `BaseTransactionalBolt` which has this definition: +The `prepare` method parameterizes this batch bolt with the Storm config, the topology context, an output collector, and the id for this batch of tuples. In the case of transactional topologies, the id will be a [TransactionAttempt](javadocs/org/apache/storm/transactional/TransactionAttempt.html) object. The batch bolt abstraction can be used in Distributed RPC as well which uses a different type of id for the batches. `BatchBolt` can actually be parameterized with the type of the id, so if you only intend to use the batch bolt for transactional topologies, you can extend `BaseTransactionalBolt` which has this definition: ```java public abstract class BaseTransactionalBolt extends BaseBatchBolt<TransactionAttempt> { @@ -211,9 +211,9 @@ This section outlines the different piec There are three kinds of bolts possible in a transactional topology: -1. [BasicBolt](javadocs/backtype/storm/topology/base/BaseBasicBolt.html): This bolt doesn't deal with batches of tuples and just emits tuples based on a single tuple of input. -2. [BatchBolt](javadocs/backtype/storm/topology/base/BaseBatchBolt.html): This bolt processes batches of tuples. `execute` is called for each tuple, and `finishBatch` is called when the batch is complete. -3. BatchBolt's that are marked as committers: The only difference between this bolt and a regular batch bolt is when `finishBatch` is called. A committer bolt has `finishedBatch` called during the commit phase. The commit phase is guaranteed to occur only after all prior batches have successfully committed, and it will be retried until all bolts in the topology succeed the commit for the batch. There are two ways to make a `BatchBolt` a committer, by having the `BatchBolt` implement the [ICommitter](javadocs/backtype/storm/transactional/ICommitter.html) marker interface, or by using the `setCommiterBolt` method in `TransactionalTopologyBuilder`. +1. [BasicBolt](javadocs/org/apache/storm/topology/base/BaseBasicBolt.html): This bolt doesn't deal with batches of tuples and just emits tuples based on a single tuple of input. +2. [BatchBolt](javadocs/org/apache/storm/topology/base/BaseBatchBolt.html): This bolt processes batches of tuples. `execute` is called for each tuple, and `finishBatch` is called when the batch is complete. +3. BatchBolt's that are marked as committers: The only difference between this bolt and a regular batch bolt is when `finishBatch` is called. A committer bolt has `finishedBatch` called during the commit phase. The commit phase is guaranteed to occur only after all prior batches have successfully committed, and it will be retried until all bolts in the topology succeed the commit for the batch. There are two ways to make a `BatchBolt` a committer, by having the `BatchBolt` implement the [ICommitter](javadocs/org/apache/storm/transactional/ICommitter.html) marker interface, or by using the `setCommiterBolt` method in `TransactionalTopologyBuilder`. #### Processing phase vs. commit phase in bolts @@ -237,7 +237,7 @@ Notice that you don't have to do any ack #### Failing a transaction -When using regular bolts, you can call the `fail` method on `OutputCollector` to fail the tuple trees of which that tuple is a member. Since transactional topologies hide the acking framework from you, they provide a different mechanism to fail a batch (and cause the batch to be replayed). Just throw a [FailedException](javadocs/backtype/storm/topology/FailedException.html). Unlike regular exceptions, this will only cause that particular batch to replay and will not crash the process. +When using regular bolts, you can call the `fail` method on `OutputCollector` to fail the tuple trees of which that tuple is a member. Since transactional topologies hide the acking framework from you, they provide a different mechanism to fail a batch (and cause the batch to be replayed). Just throw a [FailedException](javadocs/org/apache/storm/topology/FailedException.html). Unlike regular exceptions, this will only cause that particular batch to replay and will not crash the process. ### Transactional spout @@ -251,11 +251,11 @@ The coordinator on the left is a regular The need to be idempotent with respect to the tuples it emits requires a `TransactionalSpout` to store a small amount of state. The state is stored in Zookeeper. -The details of implementing a `TransactionalSpout` are in [the Javadoc](javadocs/backtype/storm/transactional/ITransactionalSpout.html). +The details of implementing a `TransactionalSpout` are in [the Javadoc](javadocs/org/apache/storm/transactional/ITransactionalSpout.html). #### Partitioned Transactional Spout -A common kind of transactional spout is one that reads the batches from a set of partitions across many queue brokers. For example, this is how [TransactionalKafkaSpout]({{page.git-tree-base}}/external/storm-kafka/src/jvm/storm/kafka/TransactionalKafkaSpout.java) works. An `IPartitionedTransactionalSpout` automates the bookkeeping work of managing the state for each partition to ensure idempotent replayability. See [the Javadoc](javadocs/backtype/storm/transactional/partitioned/IPartitionedTransactionalSpout.html) for more details. +A common kind of transactional spout is one that reads the batches from a set of partitions across many queue brokers. For example, this is how [TransactionalKafkaSpout]({{page.git-tree-base}}/external/storm-kafka/src/jvm/storm/kafka/TransactionalKafkaSpout.java) works. An `IPartitionedTransactionalSpout` automates the bookkeeping work of managing the state for each partition to ensure idempotent replayability. See [the Javadoc](javadocs/org/apache/storm/transactional/partitioned/IPartitionedTransactionalSpout.html) for more details. ### Configuration @@ -325,7 +325,7 @@ In this scenario, tuples 41-50 are skipp By failing all subsequent transactions on failure, no tuples are skipped. This also shows that a requirement of transactional spouts is that they always emit where the last transaction left off. -A non-idempotent transactional spout is more concisely referred to as an "OpaqueTransactionalSpout" (opaque is the opposite of idempotent). [IOpaquePartitionedTransactionalSpout](javadocs/backtype/storm/transactional/partitioned/IOpaquePartitionedTransactionalSpout.html) is an interface for implementing opaque partitioned transactional spouts, of which [OpaqueTransactionalKafkaSpout]({{page.git-tree-base}}/external/storm-kafka/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java) is an example. `OpaqueTransactionalKafkaSpout` can withstand losing individual Kafka nodes without sacrificing accuracy as long as you use the update strategy as explained in this section. +A non-idempotent transactional spout is more concisely referred to as an "OpaqueTransactionalSpout" (opaque is the opposite of idempotent). [IOpaquePartitionedTransactionalSpout](javadocs/org/apache/storm/transactional/partitioned/IOpaquePartitionedTransactionalSpout.html) is an interface for implementing opaque partitioned transactional spouts, of which [OpaqueTransactionalKafkaSpout]({{page.git-tree-base}}/external/storm-kafka/src/jvm/storm/kafka/OpaqueTransactionalKafkaSpout.java) is an example. `OpaqueTransactionalKafkaSpout` can withstand losing individual Kafka nodes without sacrificing accuracy as long as you use the update strategy as explained in this section. ## Implementation Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-API-Overview.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-API-Overview.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-API-Overview.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-API-Overview.md Fri Mar 18 17:56:59 2016 @@ -467,7 +467,7 @@ Repartitioning operations run a function 3. partitionBy: partitionBy takes in a set of fields and does semantic partitioning based on that set of fields. The fields are hashed and modded by the number of target partitions to select the target partition. partitionBy guarantees that the same set of fields always goes to the same target partition. 4. global: All tuples are sent to the same partition. The same partition is chosen for all batches in the stream. 5. batchGlobal: All tuples in the batch are sent to the same partition. Different batches in the stream may go to different partitions. -6. partition: This method takes in a custom partitioning function that implements backtype.storm.grouping.CustomStreamGrouping +6. partition: This method takes in a custom partitioning function that implements org.apache.storm.grouping.CustomStreamGrouping ## Aggregation operations @@ -491,7 +491,7 @@ The groupBy operation repartitions the s  -If you run aggregators on a grouped stream, the aggregation will be run within each group instead of against the whole batch. persistentAggregate can also be run on a GroupedStream, in which case the results will be stored in a [MapState]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/state/map/MapState.java) with the key being the grouping fields. You can read more about persistentAggregate in the [Trident state doc](Trident-state.html). +If you run aggregators on a grouped stream, the aggregation will be run within each group instead of against the whole batch. persistentAggregate can also be run on a GroupedStream, in which case the results will be stored in a [MapState]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/state/map/MapState.java) with the key being the grouping fields. You can read more about persistentAggregate in the [Trident state doc](Trident-state.html). Like regular streams, aggregators on grouped streams can be chained. Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-spouts.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-spouts.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-spouts.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-spouts.md Fri Mar 18 17:56:59 2016 @@ -34,10 +34,10 @@ Even while processing multiple batches s Here are the following spout APIs available: -1. [ITridentSpout]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/spout/ITridentSpout.java): The most general API that can support transactional or opaque transactional semantics. Generally you'll use one of the partitioned flavors of this API rather than this one directly. -2. [IBatchSpout]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/spout/IBatchSpout.java): A non-transactional spout that emits batches of tuples at a time -3. [IPartitionedTridentSpout]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/spout/IPartitionedTridentSpout.java): A transactional spout that reads from a partitioned data source (like a cluster of Kafka servers) -4. [IOpaquePartitionedTridentSpout]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/spout/IOpaquePartitionedTridentSpout.java): An opaque transactional spout that reads from a partitioned data source +1. [ITridentSpout]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/spout/ITridentSpout.java): The most general API that can support transactional or opaque transactional semantics. Generally you'll use one of the partitioned flavors of this API rather than this one directly. +2. [IBatchSpout]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/spout/IBatchSpout.java): A non-transactional spout that emits batches of tuples at a time +3. [IPartitionedTridentSpout]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/spout/IPartitionedTridentSpout.java): A transactional spout that reads from a partitioned data source (like a cluster of Kafka servers) +4. [IOpaquePartitionedTridentSpout]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/spout/IOpaquePartitionedTridentSpout.java): An opaque transactional spout that reads from a partitioned data source And, like mentioned in the beginning of this tutorial, you can use regular IRichSpout's as well. Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-state.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-state.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-state.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Trident-state.md Fri Mar 18 17:56:59 2016 @@ -309,7 +309,7 @@ public interface Snapshottable<T> extend } ``` -[MemoryMapState]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/testing/MemoryMapState.java) and [MemcachedState](https://github.com/nathanmarz/trident-memcached/blob/{{page.version}}/src/jvm/trident/memcached/MemcachedState.java) each implement both of these interfaces. +[MemoryMapState]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/testing/MemoryMapState.java) and [MemcachedState](https://github.com/nathanmarz/trident-memcached/blob/{{page.version}}/src/jvm/trident/memcached/MemcachedState.java) each implement both of these interfaces. ## Implementing Map States @@ -322,10 +322,10 @@ public interface IBackingMap<T> { } ``` -OpaqueMap's will call multiPut with [OpaqueValue]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/state/OpaqueValue.java)'s for the vals, TransactionalMap's will give [TransactionalValue]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/state/TransactionalValue.java)'s for the vals, and NonTransactionalMaps will just pass the objects from the topology through. +OpaqueMap's will call multiPut with [OpaqueValue]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/state/OpaqueValue.java)'s for the vals, TransactionalMap's will give [TransactionalValue]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/state/TransactionalValue.java)'s for the vals, and NonTransactionalMaps will just pass the objects from the topology through. -Trident also provides the [CachedMap]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/state/map/CachedMap.java) class to do automatic LRU caching of map key/vals. +Trident also provides the [CachedMap]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/state/map/CachedMap.java) class to do automatic LRU caching of map key/vals. -Finally, Trident provides the [SnapshottableMap]({{page.git-blob-base}}/storm-core/src/jvm/storm/trident/state/map/SnapshottableMap.java) class that turns a MapState into a Snapshottable object, by storing global aggregations into a fixed key. +Finally, Trident provides the [SnapshottableMap]({{page.git-blob-base}}/storm-core/src/jvm/org/apache/storm/trident/state/map/SnapshottableMap.java) class that turns a MapState into a Snapshottable object, by storing global aggregations into a fixed key. Take a look at the implementation of [MemcachedState](https://github.com/nathanmarz/trident-memcached/blob/master/src/jvm/trident/memcached/MemcachedState.java) to see how all these utilities can be put together to make a high performance MapState implementation. MemcachedState allows you to choose between opaque transactional, transactional, and non-transactional semantics. Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Troubleshooting.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Troubleshooting.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Troubleshooting.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Troubleshooting.md Fri Mar 18 17:56:59 2016 @@ -81,11 +81,11 @@ Symptoms: ``` java.lang.RuntimeException: java.util.ConcurrentModificationException - at backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:84) - at backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:55) - at backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:56) - at backtype.storm.disruptor$consume_loop_STAR_$fn__1597.invoke(disruptor.clj:67) - at backtype.storm.util$async_loop$fn__465.invoke(util.clj:377) + at org.apache.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:84) + at org.apache.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:55) + at org.apache.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:56) + at org.apache.storm.disruptor$consume_loop_STAR_$fn__1597.invoke(disruptor.clj:67) + at org.apache.storm.util$async_loop$fn__465.invoke(util.clj:377) at clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:679) Caused by: java.util.ConcurrentModificationException @@ -101,12 +101,12 @@ Caused by: java.util.ConcurrentModificat at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1416) at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1174) at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:346) - at backtype.storm.serialization.SerializableSerializer.write(SerializableSerializer.java:21) + at org.apache.storm.serialization.SerializableSerializer.write(SerializableSerializer.java:21) at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:554) at com.esotericsoftware.kryo.serializers.CollectionSerializer.write(CollectionSerializer.java:77) at com.esotericsoftware.kryo.serializers.CollectionSerializer.write(CollectionSerializer.java:18) at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:472) - at backtype.storm.serialization.KryoValuesSerializer.serializeInto(KryoValuesSerializer.java:27) + at org.apache.storm.serialization.KryoValuesSerializer.serializeInto(KryoValuesSerializer.java:27) ``` Solution: @@ -122,21 +122,21 @@ Symptoms: ``` java.lang.RuntimeException: java.lang.NullPointerException - at backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:84) - at backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:55) - at backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:56) - at backtype.storm.disruptor$consume_loop_STAR_$fn__1596.invoke(disruptor.clj:67) - at backtype.storm.util$async_loop$fn__465.invoke(util.clj:377) + at org.apache.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:84) + at org.apache.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:55) + at org.apache.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:56) + at org.apache.storm.disruptor$consume_loop_STAR_$fn__1596.invoke(disruptor.clj:67) + at org.apache.storm.util$async_loop$fn__465.invoke(util.clj:377) at clojure.lang.AFn.run(AFn.java:24) at java.lang.Thread.run(Thread.java:662) Caused by: java.lang.NullPointerException - at backtype.storm.serialization.KryoTupleSerializer.serialize(KryoTupleSerializer.java:24) - at backtype.storm.daemon.worker$mk_transfer_fn$fn__4126$fn__4130.invoke(worker.clj:99) - at backtype.storm.util$fast_list_map.invoke(util.clj:771) - at backtype.storm.daemon.worker$mk_transfer_fn$fn__4126.invoke(worker.clj:99) - at backtype.storm.daemon.executor$start_batch_transfer__GT_worker_handler_BANG_$fn__3904.invoke(executor.clj:205) - at backtype.storm.disruptor$clojure_handler$reify__1584.onEvent(disruptor.clj:43) - at backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:81) + at org.apache.storm.serialization.KryoTupleSerializer.serialize(KryoTupleSerializer.java:24) + at org.apache.storm.daemon.worker$mk_transfer_fn$fn__4126$fn__4130.invoke(worker.clj:99) + at org.apache.storm.util$fast_list_map.invoke(util.clj:771) + at org.apache.storm.daemon.worker$mk_transfer_fn$fn__4126.invoke(worker.clj:99) + at org.apache.storm.daemon.executor$start_batch_transfer__GT_worker_handler_BANG_$fn__3904.invoke(executor.clj:205) + at org.apache.storm.disruptor$clojure_handler$reify__1584.onEvent(disruptor.clj:43) + at org.apache.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:81) ... 6 more ``` @@ -145,34 +145,34 @@ or ``` java.lang.RuntimeException: java.lang.NullPointerException at -backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:128) +org.apache.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:128) ~[storm-core-0.9.3.jar:0.9.3] at -backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:99) +org.apache.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:99) ~[storm-core-0.9.3.jar:0.9.3] at -backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:80) +org.apache.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:80) ~[storm-core-0.9.3.jar:0.9.3] at -backtype.storm.disruptor$consume_loop_STAR_$fn__759.invoke(disruptor.clj:94) +org.apache.storm.disruptor$consume_loop_STAR_$fn__759.invoke(disruptor.clj:94) ~[storm-core-0.9.3.jar:0.9.3] - at backtype.storm.util$async_loop$fn__458.invoke(util.clj:463) + at org.apache.storm.util$async_loop$fn__458.invoke(util.clj:463) ~[storm-core-0.9.3.jar:0.9.3] at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_65] Caused by: java.lang.NullPointerException: null at clojure.lang.RT.intCast(RT.java:1087) ~[clojure-1.5.1.jar:na] at -backtype.storm.daemon.worker$mk_transfer_fn$fn__3548.invoke(worker.clj:129) +org.apache.storm.daemon.worker$mk_transfer_fn$fn__3548.invoke(worker.clj:129) ~[storm-core-0.9.3.jar:0.9.3] at -backtype.storm.daemon.executor$start_batch_transfer__GT_worker_handler_BANG_$fn__3282.invoke(executor.clj:258) +org.apache.storm.daemon.executor$start_batch_transfer__GT_worker_handler_BANG_$fn__3282.invoke(executor.clj:258) ~[storm-core-0.9.3.jar:0.9.3] at -backtype.storm.disruptor$clojure_handler$reify__746.onEvent(disruptor.clj:58) +org.apache.storm.disruptor$clojure_handler$reify__746.onEvent(disruptor.clj:58) ~[storm-core-0.9.3.jar:0.9.3] at -backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:125) +org.apache.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:125) ~[storm-core-0.9.3.jar:0.9.3] ... 6 common frames omitted ``` Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Tutorial.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Tutorial.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Tutorial.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Tutorial.md Fri Mar 18 17:56:59 2016 @@ -28,10 +28,10 @@ To do realtime computation on Storm, you Running a topology is straightforward. First, you package all your code and dependencies into a single jar. Then, you run a command like the following: ``` -storm jar all-my-code.jar backtype.storm.MyTopology arg1 arg2 +storm jar all-my-code.jar org.apache.storm.MyTopology arg1 arg2 ``` -This runs the class `backtype.storm.MyTopology` with the arguments `arg1` and `arg2`. The main function of the class defines the topology and submits it to Nimbus. The `storm jar` part takes care of connecting to Nimbus and uploading the jar. +This runs the class `org.apache.storm.MyTopology` with the arguments `arg1` and `arg2`. The main function of the class defines the topology and submits it to Nimbus. The `storm jar` part takes care of connecting to Nimbus and uploading the jar. Since topology definitions are just Thrift structs, and Nimbus is a Thrift service, you can create and submit topologies using any programming language. The above example is the easiest way to do it from a JVM-based language. See [Running topologies on a production cluster](Running-topologies-on-a-production-cluster.html)] for more information on starting and stopping topologies. @@ -103,11 +103,11 @@ This topology contains a spout and two b This code defines the nodes using the `setSpout` and `setBolt` methods. These methods take as input a user-specified id, an object containing the processing logic, and the amount of parallelism you want for the node. In this example, the spout is given id "words" and the bolts are given ids "exclaim1" and "exclaim2". -The object containing the processing logic implements the [IRichSpout](javadocs/backtype/storm/topology/IRichSpout.html) interface for spouts and the [IRichBolt](javadocs/backtype/storm/topology/IRichBolt.html) interface for bolts. +The object containing the processing logic implements the [IRichSpout](javadocs/org/apache/storm/topology/IRichSpout.html) interface for spouts and the [IRichBolt](javadocs/org/apache/storm/topology/IRichBolt.html) interface for bolts. The last parameter, how much parallelism you want for the node, is optional. It indicates how many threads should execute that component across the cluster. If you omit it, Storm will only allocate one thread for that node. -`setBolt` returns an [InputDeclarer](javadocs/backtype/storm/topology/InputDeclarer.html) object that is used to define the inputs to the Bolt. Here, component "exclaim1" declares that it wants to read all the tuples emitted by component "words" using a shuffle grouping, and component "exclaim2" declares that it wants to read all the tuples emitted by component "exclaim1" using a shuffle grouping. "shuffle grouping" means that tuples should be randomly distributed from the input tasks to the bolt's tasks. There are many ways to group data between components. These will be explained in a few sections. +`setBolt` returns an [InputDeclarer](javadocs/org/apache/storm/topology/InputDeclarer.html) object that is used to define the inputs to the Bolt. Here, component "exclaim1" declares that it wants to read all the tuples emitted by component "words" using a shuffle grouping, and component "exclaim2" declares that it wants to read all the tuples emitted by component "exclaim1" using a shuffle grouping. "shuffle grouping" means that tuples should be randomly distributed from the input tasks to the bolt's tasks. There are many ways to group data between components. These will be explained in a few sections. If you wanted component "exclaim2" to read all the tuples emitted by both component "words" and component "exclaim1", you would write component "exclaim2"'s definition like this: @@ -168,7 +168,7 @@ public static class ExclamationBolt impl The `prepare` method provides the bolt with an `OutputCollector` that is used for emitting tuples from this bolt. Tuples can be emitted at anytime from the bolt -- in the `prepare`, `execute`, or `cleanup` methods, or even asynchronously in another thread. This `prepare` implementation simply saves the `OutputCollector` as an instance variable to be used later on in the `execute` method. -The `execute` method receives a tuple from one of the bolt's inputs. The `ExclamationBolt` grabs the first field from the tuple and emits a new tuple with the string "!!!" appended to it. If you implement a bolt that subscribes to multiple input sources, you can find out which component the [Tuple](/javadoc/apidocs/backtype/storm/tuple/Tuple.html) came from by using the `Tuple#getSourceComponent` method. +The `execute` method receives a tuple from one of the bolt's inputs. The `ExclamationBolt` grabs the first field from the tuple and emits a new tuple with the string "!!!" appended to it. If you implement a bolt that subscribes to multiple input sources, you can find out which component the [Tuple](/javadoc/apidocs/org/apache/storm/tuple/Tuple.html) came from by using the `Tuple#getSourceComponent` method. There's a few other things going on in the `execute` method, namely that the input tuple is passed as the first argument to `emit` and the input tuple is acked on the final line. These are part of Storm's reliability API for guaranteeing no data loss and will be explained later in this tutorial. @@ -233,7 +233,7 @@ The configuration is used to tune variou 1. **TOPOLOGY_WORKERS** (set with `setNumWorkers`) specifies how many _processes_ you want allocated around the cluster to execute the topology. Each component in the topology will execute as many _threads_. The number of threads allocated to a given component is configured through the `setBolt` and `setSpout` methods. Those _threads_ exist within worker _processes_. Each worker _process_ contains within it some number of _threads_ for some number of components. For instance, you may have 300 threads specified across all your components and 50 worker processes specified in your config. Each worker process will execute 6 threads, each of which of could belong to a different component. You tune the performance of Storm topologies by tweaking the parallelism for each component and the number of worker processes those threads should run within. 2. **TOPOLOGY_DEBUG** (set with `setDebug`), when set to true, tells Storm to log every message every emitted by a component. This is useful in local mode when testing topologies, but you probably want to keep this turned off when running topologies on the cluster. -There's many other configurations you can set for the topology. The various configurations are detailed on [the Javadoc for Config](javadocs/backtype/storm/Config.html). +There's many other configurations you can set for the topology. The various configurations are detailed on [the Javadoc for Config](javadocs/org/apache/storm/Config.html). To learn about how to set up your development environment so that you can run topologies in local mode (such as in Eclipse), see [Creating a new Storm project](Creating-a-new-Storm-project.html). Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Understanding-the-parallelism-of-a-Storm-topology.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Understanding-the-parallelism-of-a-Storm-topology.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Understanding-the-parallelism-of-a-Storm-topology.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/Understanding-the-parallelism-of-a-Storm-topology.md Fri Mar 18 17:56:59 2016 @@ -30,25 +30,25 @@ The following sections give an overview ### Number of worker processes * Description: How many worker processes to create _for the topology_ across machines in the cluster. -* Configuration option: [TOPOLOGY_WORKERS](javadocs/backtype/storm/Config.html#TOPOLOGY_WORKERS) +* Configuration option: [TOPOLOGY_WORKERS](javadocs/org/apache/storm/Config.html#TOPOLOGY_WORKERS) * How to set in your code (examples): - * [Config#setNumWorkers](javadocs/backtype/storm/Config.html) + * [Config#setNumWorkers](javadocs/org/apache/storm/Config.html) ### Number of executors (threads) * Description: How many executors to spawn _per component_. * Configuration option: None (pass ``parallelism_hint`` parameter to ``setSpout`` or ``setBolt``) * How to set in your code (examples): - * [TopologyBuilder#setSpout()](javadocs/backtype/storm/topology/TopologyBuilder.html) - * [TopologyBuilder#setBolt()](javadocs/backtype/storm/topology/TopologyBuilder.html) + * [TopologyBuilder#setSpout()](javadocs/org/apache/storm/topology/TopologyBuilder.html) + * [TopologyBuilder#setBolt()](javadocs/org/apache/storm/topology/TopologyBuilder.html) * Note that as of Storm 0.8 the ``parallelism_hint`` parameter now specifies the initial number of executors (not tasks!) for that bolt. ### Number of tasks * Description: How many tasks to create _per component_. -* Configuration option: [TOPOLOGY_TASKS](javadocs/backtype/storm/Config.html#TOPOLOGY_TASKS) +* Configuration option: [TOPOLOGY_TASKS](javadocs/org/apache/storm/Config.html#TOPOLOGY_TASKS) * How to set in your code (examples): - * [ComponentConfigurationDeclarer#setNumTasks()](javadocs/backtype/storm/topology/ComponentConfigurationDeclarer.html) + * [ComponentConfigurationDeclarer#setNumTasks()](javadocs/org/apache/storm/topology/ComponentConfigurationDeclarer.html) Here is an example code snippet to show these settings in practice: @@ -91,7 +91,7 @@ StormSubmitter.submitTopology( And of course Storm comes with additional configuration settings to control the parallelism of a topology, including: -* [TOPOLOGY_MAX_TASK_PARALLELISM](javadocs/backtype/storm/Config.html#TOPOLOGY_MAX_TASK_PARALLELISM): This setting puts a ceiling on the number of executors that can be spawned for a single component. It is typically used during testing to limit the number of threads spawned when running a topology in local mode. You can set this option via e.g. [Config#setMaxTaskParallelism()](javadocs/backtype/storm/Config.html#setMaxTaskParallelism(int)). +* [TOPOLOGY_MAX_TASK_PARALLELISM](javadocs/org/apache/storm/Config.html#TOPOLOGY_MAX_TASK_PARALLELISM): This setting puts a ceiling on the number of executors that can be spawned for a single component. It is typically used during testing to limit the number of threads spawned when running a topology in local mode. You can set this option via e.g. [Config#setMaxTaskParallelism()](javadocs/org/apache/storm/Config.html#setMaxTaskParallelism(int)). ## How to change the parallelism of a running topology Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/distcache-blobstore.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/distcache-blobstore.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/distcache-blobstore.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/distcache-blobstore.md Fri Mar 18 17:56:59 2016 @@ -68,7 +68,7 @@ key âkey1â having a local file ``` storm jar /home/y/lib/storm-starter/current/storm-starter-jar-with-dependencies.jar -storm.starter.clj.word_count test_topo -c topology.blobstore.map='{"key1":{"localname":"blob_file", "uncompress":"false"},"key2":{}}' +org.apache.storm.starter.clj.word_count test_topo -c topology.blobstore.map='{"key1":{"localname":"blob_file", "uncompress":"false"},"key2":{}}' ``` ### Blob Creation Process @@ -98,7 +98,7 @@ end, the supervisor and localizer talks ## Additional Features and Documentation ``` -storm jar /home/y/lib/storm-starter/current/storm-starter-jar-with-dependencies.jar storm.starter.clj.word_count test_topo +storm jar /home/y/lib/storm-starter/current/storm-starter-jar-with-dependencies.jar org.apache.storm.starter.clj.word_count test_topo -c topology.blobstore.map='{"key1":{"localname":"blob_file", "uncompress":"false"},"key2":{}}' ``` @@ -275,8 +275,8 @@ blobstore.dir: The directory where all b node and for HDFS file system it represents the hdfs file system path. supervisor.blobstore.class: This configuration is meant to set the client for the supervisor in order to talk to the blobstore. -For a local file system blobstore it is set to âbacktype.storm.blobstore.NimbusBlobStoreâ and for the HDFS blobstore it is set -to âbacktype.storm.blobstore.HdfsClientBlobStoreâ. +For a local file system blobstore it is set to âorg.apache.storm.blobstore.NimbusBlobStoreâ and for the HDFS blobstore it is set +to âorg.apache.storm.blobstore.HdfsClientBlobStoreâ. supervisor.blobstore.download.thread.count: This configuration spawns multiple threads for from the supervisor in order download blobs concurrently. The default is set to 5 @@ -292,7 +292,7 @@ of the distributed cache contents. It is supervisor.localizer.cleanup.interval.ms: The distributed cache cleanup interval. Controls how often it scans to attempt to cleanup anything over the cache target size. By default it is set to 600000 milliseconds. -nimbus.blobstore.class: Sets the blobstore implementation nimbus uses. It is set to "backtype.storm.blobstore.LocalFsBlobStore" +nimbus.blobstore.class: Sets the blobstore implementation nimbus uses. It is set to "org.apache.storm.blobstore.LocalFsBlobStore" nimbus.blobstore.expiration.secs: During operations with the blobstore, via master, how long a connection is idle before nimbus considers it dead and drops the session and any associated connections. The default is set to 600. @@ -300,7 +300,7 @@ considers it dead and drops the session storm.blobstore.inputstream.buffer.size.bytes: The buffer size it uses for blobstore upload. It is set to 65536 bytes. client.blobstore.class: The blobstore implementation the storm client uses. The current implementation uses the default -config "backtype.storm.blobstore.NimbusBlobStore". +config "org.apache.storm.blobstore.NimbusBlobStore". blobstore.replication.factor: It sets the replication for each blob within the blobstore. The âtopology.min.replication.countâ ensures the minimum replication the topology specific blobs are set before launching the topology. You might want to set the @@ -397,7 +397,7 @@ file-name-like format and extension, so ###### Example: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -storm jar /home/y/lib/storm-starter/current/storm-starter-jar-with-dependencies.jar storm.starter.clj.word_count test_topo -c topology.blobstore.map='{"key1":{"localname":"blob_file", "uncompress":"false"},"key2":{}}' +storm jar /home/y/lib/storm-starter/current/storm-starter-jar-with-dependencies.jar org.apache.storm.starter.clj.word_count test_topo -c topology.blobstore.map='{"key1":{"localname":"blob_file", "uncompress":"false"},"key2":{}}' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Note: Please take care of the quotes. @@ -503,17 +503,17 @@ ClientBlobStore clientBlobStore = Utils. The required Utils package can by imported by: ```java -import backtype.storm.utils.Utils; +import org.apache.storm.utils.Utils; ``` ClientBlobStore and other blob-related classes can be imported by: ```java -import backtype.storm.blobstore.ClientBlobStore; -import backtype.storm.blobstore.AtomicOutputStream; -import backtype.storm.blobstore.InputStreamWithMeta; -import backtype.storm.blobstore.BlobStoreAclHandler; -import backtype.storm.generated.*; +import org.apache.storm.blobstore.ClientBlobStore; +import org.apache.storm.blobstore.AtomicOutputStream; +import org.apache.storm.blobstore.InputStreamWithMeta; +import org.apache.storm.blobstore.BlobStoreAclHandler; +import org.apache.storm.generated.*; ``` ### Creating ACLs to be used for blobs Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/flux.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/flux.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/flux.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/flux.md Fri Mar 18 17:56:59 2016 @@ -241,7 +241,7 @@ sentence-spout[1](org.apache.storm.flux. ---------------- BOLTS --------------- splitsentence[1](org.apache.storm.flux.bolts.GenericShellBolt) log[1](org.apache.storm.flux.wrappers.bolts.LogInfoBolt) -count[1](backtype.storm.testing.TestWordCounter) +count[1](org.apache.storm.testing.TestWordCounter) --------------- STREAMS --------------- sentence-spout --SHUFFLE--> splitsentence splitsentence --FIELDS--> count @@ -260,7 +260,7 @@ definition consists of the following: * A list of spouts, each identified by a unique ID * A list of bolts, each identified by a unique ID * A list of "stream" objects representing a flow of tuples between spouts and bolts - 4. **OR** (A JVM class that can produce a `backtype.storm.generated.StormTopology` instance: + 4. **OR** (A JVM class that can produce a `org.apache.storm.generated.StormTopology` instance: * A `topologySource` definition. @@ -275,13 +275,13 @@ config: # spout definitions spouts: - id: "spout-1" - className: "backtype.storm.testing.TestWordSpout" + className: "org.apache.storm.testing.TestWordSpout" parallelism: 1 # bolt definitions bolts: - id: "bolt-1" - className: "backtype.storm.testing.TestWordCounter" + className: "org.apache.storm.testing.TestWordCounter" parallelism: 1 - id: "bolt-2" className: "org.apache.storm.flux.wrappers.bolts.LogInfoBolt" @@ -329,7 +329,7 @@ You would then be able to reference thos ```yaml - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "${kafka.zookeeper.hosts}" ``` @@ -349,13 +349,13 @@ Components are essentially named object bolts. If you are familiar with the Spring framework, components are roughly analagous to Spring beans. Every component is identified, at a minimum, by a unique identifier (String) and a class name (String). For example, -the following will make an instance of the `storm.kafka.StringScheme` class available as a reference under the key -`"stringScheme"` . This assumes the `storm.kafka.StringScheme` has a default constructor. +the following will make an instance of the `org.apache.storm.kafka.StringScheme` class available as a reference under the key +`"stringScheme"` . This assumes the `org.apache.storm.kafka.StringScheme` has a default constructor. ```yaml components: - id: "stringScheme" - className: "storm.kafka.StringScheme" + className: "org.apache.storm.kafka.StringScheme" ``` ### Contructor Arguments, References, Properties and Configuration Methods @@ -367,7 +367,7 @@ object by calling the constructor that t ```yaml - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "localhost:2181" ``` @@ -382,10 +382,10 @@ to another component's constructor: ```yaml components: - id: "stringScheme" - className: "storm.kafka.StringScheme" + className: "org.apache.storm.kafka.StringScheme" - id: "stringMultiScheme" - className: "backtype.storm.spout.SchemeAsMultiScheme" + className: "org.apache.storm.spout.SchemeAsMultiScheme" constructorArgs: - ref: "stringScheme" # component with id "stringScheme" must be declared above. ``` @@ -397,7 +397,7 @@ JavaBean-like setter methods and fields ```yaml - id: "spoutConfig" - className: "storm.kafka.SpoutConfig" + className: "org.apache.storm.kafka.SpoutConfig" constructorArgs: # brokerHosts - ref: "zkHosts" @@ -493,7 +493,7 @@ FileRotationPolicy rotationPolicy = new ## Topology Config The `config` section is simply a map of Storm topology configuration parameters that will be passed to the -`backtype.storm.StormSubmitter` as an instance of the `backtype.storm.Config` class: +`org.apache.storm.StormSubmitter` as an instance of the `org.apache.storm.Config` class: ```yaml config: @@ -538,7 +538,7 @@ topologySource: ``` __N.B.:__ The specified method must accept a single argument of type `java.util.Map<String, Object>` or -`backtype.storm.Config`, and return a `backtype.storm.generated.StormTopology` object. +`org.apache.storm.Config`, and return a `org.apache.storm.generated.StormTopology` object. # YAML DSL ## Spouts and Bolts @@ -569,21 +569,21 @@ Kafka spout example: ```yaml components: - id: "stringScheme" - className: "storm.kafka.StringScheme" + className: "org.apache.storm.kafka.StringScheme" - id: "stringMultiScheme" - className: "backtype.storm.spout.SchemeAsMultiScheme" + className: "org.apache.storm.spout.SchemeAsMultiScheme" constructorArgs: - ref: "stringScheme" - id: "zkHosts" - className: "storm.kafka.ZkHosts" + className: "org.apache.storm.kafka.ZkHosts" constructorArgs: - "localhost:2181" # Alternative kafka config # - id: "kafkaConfig" -# className: "storm.kafka.KafkaConfig" +# className: "org.apache.storm.kafka.KafkaConfig" # constructorArgs: # # brokerHosts # - ref: "zkHosts" @@ -593,7 +593,7 @@ components: # - "myKafkaClientId" - id: "spoutConfig" - className: "storm.kafka.SpoutConfig" + className: "org.apache.storm.kafka.SpoutConfig" constructorArgs: # brokerHosts - ref: "zkHosts" @@ -615,7 +615,7 @@ config: # spout definitions spouts: - id: "kafka-spout" - className: "storm.kafka.KafkaSpout" + className: "org.apache.storm.kafka.KafkaSpout" constructorArgs: - ref: "spoutConfig" @@ -642,7 +642,7 @@ bolts: # ... - id: "count" - className: "backtype.storm.testing.TestWordCounter" + className: "org.apache.storm.testing.TestWordCounter" parallelism: 1 # ... ``` @@ -709,7 +709,7 @@ Custom stream groupings are defined by s that tells Flux how to instantiate the custom class. The `customClass` definition extends `component`, so it supports constructor arguments, references, and properties as well. -The example below creates a Stream with an instance of the `backtype.storm.testing.NGrouping` custom stream grouping +The example below creates a Stream with an instance of the `org.apache.storm.testing.NGrouping` custom stream grouping class. ```yaml @@ -719,7 +719,7 @@ class. grouping: type: CUSTOM customClass: - className: "backtype.storm.testing.NGrouping" + className: "org.apache.storm.testing.NGrouping" constructorArgs: - 1 ``` @@ -787,7 +787,7 @@ bolts: parallelism: 1 - id: "count" - className: "backtype.storm.testing.TestWordCounter" + className: "org.apache.storm.testing.TestWordCounter" parallelism: 1 #stream definitions Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/nimbus-ha-design.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/nimbus-ha-design.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/nimbus-ha-design.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/nimbus-ha-design.md Fri Mar 18 17:56:59 2016 @@ -204,7 +204,7 @@ be rare in general case. You can use nimbus ha with default configuration , however the default configuration assumes a single nimbus host so it trades off replication for lower topology submission latency. Depending on your use case you can adjust following configurations: * storm.codedistributor.class : This is a string representing fully qualified class name of a class that implements -backtype.storm.codedistributor.ICodeDistributor. The default is set to "backtype.storm.codedistributor.LocalFileSystemCodeDistributor". +org.apache.storm.codedistributor.ICodeDistributor. The default is set to "org.apache.storm.codedistributor.LocalFileSystemCodeDistributor". This class leverages local file system to store both meta files and code/configs. This class adds extra load on zookeeper as even after downloading the code-distrbutor meta file it contacts zookeeper in order to figure out hosts from where it can download actual code/config and to get the current replication count. An alternative is to use @@ -219,4 +219,4 @@ The default is 60 seconds, a value of -1 Note: Even though all nimbus hosts have watchers on zookeeper to be notified immediately as soon as a new topology is available for code download, the callback pretty much never results in code download. In practice we have observed that the desired replication is only achieved once the background-thread runs. -So you should expect your topology submission time to be somewhere between 0 to (2 * nimbus.code.sync.freq.secs) for any nimbus.min.replication.count > 1. \ No newline at end of file +So you should expect your topology submission time to be somewhere between 0 to (2 * nimbus.code.sync.freq.secs) for any nimbus.min.replication.count > 1. Modified: storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/storm-kafka.md URL: http://svn.apache.org/viewvc/storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/storm-kafka.md?rev=1735653&r1=1735652&r2=1735653&view=diff ============================================================================== --- storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/storm-kafka.md (original) +++ storm/branches/bobby-versioned-site/releases/1.0.0-SNAPSHOT/storm-kafka.md Fri Mar 18 17:56:59 2016 @@ -68,7 +68,7 @@ In addition to these parameters, SpoutCo // Exponential back-off retry settings. These are used when retrying messages after a bolt // calls OutputCollector.fail(). - // Note: be sure to set backtype.storm.Config.MESSAGE_TIMEOUT_SECS appropriately to prevent + // Note: be sure to set org.apache.storm.Config.MESSAGE_TIMEOUT_SECS appropriately to prevent // resubmitting the message while still retrying. public long retryInitialDelayMs = 0; public double retryDelayMultiplier = 1.0; @@ -187,9 +187,9 @@ use Kafka 0.8.1.1 built against Scala 2. Note that the ZooKeeper and log4j dependencies are excluded to prevent version conflicts with Storm's dependencies. ##Writing to Kafka as part of your topology -You can create an instance of storm.kafka.bolt.KafkaBolt and attach it as a component to your topology or if you -are using trident you can use storm.kafka.trident.TridentState, storm.kafka.trident.TridentStateFactory and -storm.kafka.trident.TridentKafkaUpdater. +You can create an instance of org.apache.storm.kafka.bolt.KafkaBolt and attach it as a component to your topology or if you +are using trident you can use org.apache.storm.kafka.trident.TridentState, org.apache.storm.kafka.trident.TridentStateFactory and +org.apache.storm.kafka.trident.TridentKafkaUpdater. You need to provide implementation of following 2 interfaces
