http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_plugins/releases.rb ---------------------------------------------------------------------- diff --git a/_plugins/releases.rb b/_plugins/releases.rb new file mode 100644 index 0000000..f28ccd2 --- /dev/null +++ b/_plugins/releases.rb @@ -0,0 +1,84 @@ +module Releases + class Generator < Jekyll::Generator + def dir_to_releasename(dir) + ret = nil + splitdir = dir.split("/").select{ |a| a != ""}; + if (splitdir[0] == 'releases') + ret = splitdir[1] + if (ret == 'current') + ret = File.readlink(splitdir.join("/")).split("/")[-1] + end + end + return ret + end + + def set_if_unset(hash, key, value) + hash[key] = hash[key] || value; + end + + def parse_version(version_string) + return version_string.split('.').map{|e| e.to_i} + end + + def release_from_pom() + text= `mvn -f ../pom.xml help:evaluate -Dexpression=project.version` + return text.split("\n").select{|a| !a.start_with?('[')}[0] + end + + def branch_from_git() + return `git rev-parse --abbrev-ref HEAD` + end + + def generate(site) + if site.config['storm_release_only'] + release_name = release_from_pom() + puts "release: #{release_name}" + git_branch = branch_from_git() + puts "branch: #{git_branch}" + for page in site.pages do + page.data['version'] = release_name; + page.data['git-tree-base'] = "http://github.com/apache/storm/tree/#{git_branch}"; + page.data['git-blob-base'] = "http://github.com/apache/storm/blob/#{git_branch}"; + end + return + end + + releases = Hash.new + if (site.data['releases']) + for rel_data in site.data['releases'] do + releases[rel_data['name']] = rel_data + end + end + + for page in site.pages do + release_name = dir_to_releasename(page.dir) + if (release_name != nil) + if !releases.has_key?(release_name) + releases[release_name] = {'name' => release_name}; + end + releases[release_name]['documented'] = true + end + end + + releases.each { |release_name, release_data| + set_if_unset(release_data, 'git-tag-or-branch', "v#{release_data['name']}") + set_if_unset(release_data, 'git-tree-base', "http://github.com/apache/storm/tree/#{release_data['git-tag-or-branch']}") + set_if_unset(release_data, 'git-blob-base', "http://github.com/apache/storm/blob/#{release_data['git-tag-or-branch']}") + set_if_unset(release_data, 'base-name', "apache-storm-#{release_data['name']}") + set_if_unset(release_data, 'has-download', !release_name.end_with?('-SNAPSHOT')) + } + + for page in site.pages do + release_name = dir_to_releasename(page.dir) + if (release_name != nil) + release_data = releases[release_name] + page.data['version'] = release_name; + page.data['git-tree-base'] = release_data['git-tree-base']; + page.data['git-blob-base'] = release_data['git-blob-base']; + end + end + site.data['releases'] = releases.values.sort{|x,y| parse_version(y['name']) <=> + parse_version(x['name'])}; + end + end +end
http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2012-08-02-storm080-released.md ---------------------------------------------------------------------- diff --git a/_posts/2012-08-02-storm080-released.md b/_posts/2012-08-02-storm080-released.md new file mode 100644 index 0000000..d5636e4 --- /dev/null +++ b/_posts/2012-08-02-storm080-released.md @@ -0,0 +1,120 @@ +--- +layout: post +title: Storm 0.8.0 and Trident released +author: Nathan Marz +--- + +I'm happy to announce the release of Storm 0.8.0. This is a *huge* release. Thanks to everyone who tested out the dev release and helped find and fix issues. And thanks to everyone who contributed pull requests. There's one big new thing available in the release: Trident. You may have heard me hint about Trident before, and now it's finally public. + +Trident is a higher level abstraction for Storm. It allows you to seamlessly mix high throughput (millions of messages per second), stateful stream processing with low latency distributed querying. If you're familiar with high level batch processing tools like Pig or Cascading, the concepts of Trident will be very familiar - Trident has joins, aggregations, grouping, functions, and filters. In addition to these, Trident adds primitives for doing stateful, incremental processing on top of any database or persistence store. Trident has consistent, exactly-once semantics, so it is easy to reason about Trident topologies. Trident is bundled with Storm, and you can read documentation about it on [this page](https://github.com/apache/incubator-storm/wiki/Documentation) (start with "Trident tutorial"). + +Trident supersedes both LinearDRPCTopologyBuilder and transactional topologies, both of which will be deprecated soon and eventually removed from the codebase. + +Here are the other highlights of this release: + + +Executors +--------- + +Prior to Storm 0.8.0, a running topology consisted of some number of workers and some number of tasks that ran on those workers. In the old model, worker = process and task = thread. Storm 0.8.0 changes this model by introducing executors. In this model, a worker = process, an executor = thread, and one executor runs many tasks from the same spout/bolt. + +The reason for the change is that the old model complected the semantics of the topology with its physical execution. For example, if you had a bolt with 4 tasks doing a fields grouping on some stream, in order to maintain the semantics of the fields grouping (that the same value always goes to the same task id for that bolt), the number of tasks for that bolt needs to be fixed for the lifetime of the topology, and since task = thread, the number of threads for that bolt is fixed for the lifetime of the topology. In the new model, the number of threads for a bolt is disassociated from the number of tasks, meaning you can change the number of threads for a spout/bolt dynamically without affecting semantics. Similarly, if you're keeping large amounts of state in your bolts, and you want to increase the parallelism of the bolt without having to repartition the state, you can do this with the new executors. + +At the API level, the "parallelism_hint" now specifies the initial number of executors for that bolt. You can specify the number of tasks using the TOPOLOGY_TASKS component config. For example: + +<code> +builder.setBolt(new MyBolt(), 3).setNumTasks(128).shuffleGrouping("spout"); +</code> + +This sets the initial number of executors to 3 and the number of tasks to 128. If you don't specify the number of tasks for a component, it will be fixed to the initial number of executors for the lifetime of the topology. + +Finally, you can change the number of workers and/or number of executors for components using the "storm rebalance" command. The following command changes the number of workers for the "demo" topology to 3, the number of executors for the "myspout" component to 5, and the number of executors for the "mybolt" component to 1: + +<code> +storm rebalance demo -n 3 -e myspout=5 -e mybolt=1 +</code> + +Pluggable scheduler +------------------- + +You can now implement your own scheduler to replace the default scheduler to assign executors to workers. You configure the class to use using the "storm.scheduler" config in your storm.yaml, and your scheduler must implement [this interface](https://github.com/apache/incubator-storm/blob/master/src/jvm/backtype/storm/scheduler/IScheduler.java). + +Throughput improvements +----------------------- + +The internals of Storm have been rearchitected for extremely significant performance gains. I'm seeing throughput increases of anywhere from 5-10x of what it was before. I've benchmarked Storm at 1M tuples / sec / node on an internal Twitter cluster. + +The key changes made were: + +a) Replacing all the internal in-memory queuing with the LMAX Disruptor +b) Doing intelligent auto-batching of processing so that the consumers can keep up with the producers + +Here are the configs which affect how buffering/batching is done: + +topology.executor.receive.buffer.size +topology.executor.send.buffer.size +topology.receiver.buffer.size +topology.transfer.buffer.size + +These may require some tweaking to optimize your topologies, but most likely the default values will work fine for you out of the box. + +Decreased Zookeeper load / increased Storm UI performance +----------------------- +Storm sends significantly less traffic to Zookeeper now (on the order of 10x less). Since it also uses so many fewer znodes to store state, the UI is significantly faster as well. + +Abstractions for shared resources +----------------------- + +The TopologyContext has methods "getTaskData", "getExecutorData", and "getResource" for sharing resources at the task level, executor level, or worker level respectively. Currently you can't set any worker resources, but this is in development. Storm currently provides a shared ExecutorService worker resource (via "getSharedExecutor" method) that can be used for launching background tasks on a shared thread pool. + +Tick tuples +----------------------- + +It's common to require a bolt to "do something" at a fixed interval, like flush writes to a database. Many people have been using variants of a ClockSpout to send these ticks. The problem with a ClockSpout is that you can't internalize the need for ticks within your bolt, so if you forget to set up your bolt correctly within your topology it won't work correctly. 0.8.0 introduces a new "tick tuple" config that lets you specify the frequency at which you want to receive tick tuples via the "topology.tick.tuple.freq.secs" component- specific config, and then your bolt will receive a tuple from the __system component and __tick stream at that frequency. + +Improved Storm UI +----------------------- + +The Storm UI now has a button for showing/hiding the "system stats" (tuples sent on streams other than ones you've defined, like acker streams), making it easier to digest what your topology is doing. + +Here's the full changelog for Storm 0.8.0: + + * Added Trident, the new high-level abstraction for intermixing high throughput, stateful stream processing with low-latency distributed querying + * Added executor abstraction between workers and tasks. Workers = processes, executors = threads that run many tasks from the same spout or bolt. + * Pluggable scheduler (thanks xumingming) + * Eliminate explicit storage of task->component in Zookeeper + * Number of workers can be dynamically changed at runtime through rebalance command and -n switch + * Number of executors for a component can be dynamically changed at runtime through rebalance command and -e switch (multiple -e switches allowed) + * Use worker heartbeats instead of task heartbeats (thanks xumingming) + * UI performance for topologies with many executors/tasks much faster due to optimized usage of Zookeeper (10x improvement) + * Added button to show/hide system stats (e.g., acker component and stream stats) from the Storm UI (thanks xumingming) + * Stats are tracked on a per-executor basis instead of per-task basis + * Major optimization for unreliable spouts and unanchored tuples (will use far less CPU) + * Revamped internals of Storm to use LMAX disruptor for internal queuing. Dramatic reductions in contention and CPU usage. + * Numerous micro-optimizations all throughout the codebase to reduce CPU usage. + * Optimized internals of Storm to use much fewer threads - two fewer threads per spout and one fewer thread per acker. + * Removed error method from task hooks (to be re-added at a later time) + * Validate that subscriptions come from valid components and streams, and if it's a field grouping that the schema is correct (thanks xumingming) + * MemoryTransactionalSpout now works in cluster mode + * Only track errors on a component by component basis to reduce the amount stored in zookeeper (to speed up UI). A side effect of this change is the removal of the task page in the UI. + * Add TOPOLOGY-TICK-TUPLE-FREQ-SECS config to have Storm automatically send "tick" tuples to a bolt's execute method coming from the __system component and __tick stream at the configured frequency. Meant to be used as a component-specific configuration. + * Upgrade Kryo to v2.17 + * Tuple is now an interface and is much cleaner. The Clojure DSL helpers have been moved to TupleImpl + * Added shared worker resources. Storm provides a shared ExecutorService thread pool by default. The number of threads in the pool can be configured with topology.worker.shared.thread.pool.size + * Improve CustomStreamGrouping interface to make it more flexible by providing more information + * Enhanced INimbus interface to allow for forced schedulers and better integration with global scheduler + * Added assigned method to ISupervisor so it knows exactly what's running and not running + * Custom serializers can now have one of four constructors: (), (Kryo), (Class), or (Kryo, Class) + * Disallow ":", ".", and "\" from topology names + * Errors in multilang subprocesses that go to stderr will be captured and logged to the worker logs (thanks vinodc) + * Workers detect and warn for missing outbound connections from assignment, drop messages for which there's no outbound connection + * Zookeeper connection timeout is now configurable (via storm.zookeeper.connection.timeout config) + * Storm is now less aggressive about halting process when there are Zookeeper errors, preferring to wait until client calls return exceptions. + * Can configure Zookeeper authentication for Storm's Zookeeper clients via "storm.zookeeper.auth.scheme" and "storm.zookeeper.auth.payload" configs + * Supervisors only download code for topologies assigned to them + * Include task id information in task hooks (thanks velvia) + * Use execvp to spawn daemons (replaces the python launcher process) (thanks ept) + * Expanded INimbus/ISupervisor interfaces to provide more information (used in Storm/Mesos integration) + * Bug fix: Realize task ids when worker heartbeats to supervisor. Some users were hitting deserialization problems here in very rare cases (thanks herberteuler) + * Bug fix: Fix bug where a topology's status would get corrupted to true if nimbus is restarted while status is rebalancing + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2012-09-06-storm081-released.md ---------------------------------------------------------------------- diff --git a/_posts/2012-09-06-storm081-released.md b/_posts/2012-09-06-storm081-released.md new file mode 100644 index 0000000..da24349 --- /dev/null +++ b/_posts/2012-09-06-storm081-released.md @@ -0,0 +1,47 @@ +--- +layout: post +title: Storm 0.8.1 released +author: Nathan Marz +--- + +Storm 0.8.1 is now available on the downloads page and in Maven. This release contains many bug fixes as well as a few important new features. These include: + +Storm's unit testing facilities have been exposed via Java +----------------------- +This is an extremely powerful API that lets you do things like: + a) Easily bring up and tear down local clusters + b) Run a fixed set of tuples through a topology and see all the tuples emitted by all components + c) Feed some tuples through the topology, wait until they've been processed, and then run your assertions + d) Use time simulation and step through time via an API. This is useful for testing topologies that do things based on time advancing. You can see examples of the unit testing API [here](https://github.com/xumingming/storm-lib/blob/master/src/jvm/storm/TestingApiDemo.java). + +Spout wait strategies +--------------------- + +There's two situations in which a spout needs to wait. The first is when the max spout pending limit is reached. The second is when nothing is emitted from nextTuple. Previously, Storm would just have that spout sit in a busy loop in those cases. What Storm does in those situations is now pluggable, and the default is now for the spout to sleep for 1 ms. This will cause the spout to use dramatically less CPU when it hits those cases, and it also obviates the need for spouts to do any sleeping in their implementation to be "polite". The wait strategy can be configured with TOPOLOGY_SPOUT_WAIT_STRATEGY and can be configured on a spout by spout basis. The interface to implement for a wait strategy is backtype.storm.spout.ISpoutWaitStrategy + +The full changelog is below: + + * Exposed Storm's unit testing facilities via the backtype.storm.Testing class. Notable functions are Testing/withLocalCluster and Testing/completeTopology + * Implemented pluggable spout wait strategy that is invoked when a spout emits nothing from nextTuple or when a spout hits the MAX_SPOUT_PENDING limit + * Spouts now have a default wait strategy of a 1 millisecond sleep + * Changed log level of "Failed message" logging to DEBUG + * Deprecated LinearDRPCTopologyBuilder, TimeCacheMap, and transactional topologies + * During "storm jar", whether topology is already running or not is checked before submitting jar to save time (thanks jasonjckn) + * Added BaseMultiReducer class to Trident that provides empty implementations of prepare and cleanup + * Added Negate builtin operation to reverse a Filter + * Added topology.kryo.decorators config that allows functions to be plugged in to customize Kryo (thanks jasonjckn) + * Enable message timeouts when using LocalCluster + * Multilang subprocesses can set "need_task_ids" to false when emitting tuples to tell Storm not to send task ids back (performance optimization) (thanks barrywhart) + * Add contains method on Tuple (thanks okapies) + * Added ISchemableSpout interface + * Bug fix: When an item is consumed off an internal buffer, the entry on the buffer is nulled to allow GC to happen on that data + * Bug fix: Helper class for Trident MapStates now clear their read cache when a new commit happens, preventing updates from spilling over from a failed batch attempt to the next attempt + * Bug fix: Fix NonTransactionalMap to take in an IBackingMap for regular values rather than TransactionalValue (thanks sjoerdmulder) + * Bug fix: Fix NPE when no input fields given for regular Aggregator + * Bug fix: Fix IndexOutOfBoundsExceptions when a bolt for global aggregation had a parallelism greater than 1 (possible with splitting, stateQuerying, and multiReduce) + * Bug fix: Fix "fields size" error that would sometimes occur when splitting a stream with multiple eaches + * Bug fix: Fix bug where a committer spout (including opaque spouts) could cause Trident batches to fail + * Bug fix: Fix Trident bug where multiple groupings on same stream would cause tuples to be duplicated to all consumers + * Bug fix: Fixed error when repartitioning stream twice in a row without any operations in between + * Bug fix: Fix rare bug in supervisor where it would continuously fail to clean up workers because the worker was already partially cleaned up + * Bug fix: Fix emitDirect in storm.py http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2013-01-11-storm082-released.md ---------------------------------------------------------------------- diff --git a/_posts/2013-01-11-storm082-released.md b/_posts/2013-01-11-storm082-released.md new file mode 100644 index 0000000..e657d09 --- /dev/null +++ b/_posts/2013-01-11-storm082-released.md @@ -0,0 +1,82 @@ +--- +layout: post +title: Storm 0.8.2 released +author: Nathan Marz +--- + +Storm 0.8.2 has been released and is available from [the downloads page](/downloads.html). This release contains a ton of improvements and fixes and is a highly recommended upgrade for everyone. + +Isolation Scheduler +------------------- + +The highlight of this release is the new "Isolation scheduler" that makes it easy and safe to share a cluster among many topologies. The isolation scheduler lets you specify which topologies should be "isolated", meaning that they run on a dedicated set of machines within the cluster where no other topologies will be running. These isolated topologies are given priority on the cluster, so resources will be allocated to isolated topologies if there's competition with non-isolated topologies, and resources will be taken away from non-isolated topologies if necessary to get resources for an isolated topology. Once all isolated topologies are allocated, the remaining machines on the cluster are shared among all non-isolated topologies. + +You configure the isolation scheduler in the Nimbus configuration. Set "storm.scheduler" to "backtype.storm.scheduler.IsolationScheduler". Then, use the "isolation.scheduler.machines" config to specify how many machines each topology should get. This config is a map from topology name to number of machines. For example: + +<script src="https://gist.github.com/4514691.js";></script> + +Any topologies submitted to the cluster not listed there will not be isolated. Note that there is no way for a user of Storm to affect their isolation settings â this is only allowed by the administrator of the cluster (this is very much intentional). + +The isolation scheduler solves the multi-tenancy problem â avoiding resource contention between topologies â by providing full isolation between topologies. The intention is that "productionized" topologies should be listed in the isolation config, and test or in-development topologies should not. The remaining machines on the cluster serve the dual role of failover for isolated topologies and for running the non-isolated topologies. + +Storm UI improvements +------------------- + +The Storm UI has also been made significantly more useful. There are new stats "#executed", "execute latency", and "capacity" tracked for all bolts. The "capacity" metric is very useful and tells you what % of the time in the last 10 minutes the bolt spent executing tuples. If this value is close to 1, then the bolt is "at capacity" and is a bottleneck in your topology. The solution to at-capacity bolts is to increase the parallelism of that bolt. + +Another useful improvement is the ability to kill, activate, deactivate, and rebalance topologies from the Storm UI. + + +Important bug fixes +------------------- + +There are also a few important bug fixes in this release. We fixed two bugs that could cause a topology to freeze up and stop processing. One of these bugs was extremely unlikely to hit, but the other one was a regression in 0.8.1 and there was a small chance of hitting it anytime a worker was restarted. + + +Changelog +--------- + + * Added backtype.storm.scheduler.IsolationScheduler. This lets you run topologies that are completely isolated at the machine level. Configure Nimbus to isolate certain topologies, and how many machines to give to each of those topologies, with the isolation.scheduler.machines config in Nimbus's storm.yaml. Topologies run on the cluster that are not listed there will share whatever remaining machines there are on the cluster after machines are allocated to the listed topologies. + * Storm UI now uses nimbus.host to find Nimbus rather than always using localhost (thanks Frostman) + * Added report-error! to Clojure DSL + * Automatically throttle errors sent to Zookeeper/Storm UI when too many are reported in a time interval (all errors are still logged) Configured with TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL and TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS + * Kryo instance used for serialization can now be controlled via IKryoFactory interface and TOPOLOGY_KRYO_FACTORY config + * Add ability to plug in custom code into Nimbus to allow/disallow topologies to be submitted via NIMBUS_TOPOLOGY_VALIDATOR config + * Added TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS config to control how often a batch can be emitted in a Trident topology. Defaults to 500 milliseconds. This is used to prevent too much load from being placed on Zookeeper in the case that batches are being processed super quickly. + * Log any topology submissions errors in nimbus.log + * Add static helpers in Config when using regular maps + * Make Trident much more memory efficient during failures by immediately removing state for failed attempts when a more recent attempt is seen + * Add ability to name portions of a Trident computation and have those names appear in the Storm UI + * Show Nimbus and topology configurations through Storm UI (thanks rnfein) + * Added ITupleCollection interface for TridentState's and TupleCollectionGet QueryFunction for getting the full contents of a state. MemoryMapState and LRUMemoryMapState implement this + * Can now submit a topology in inactive state. Storm will wait to call open/prepare on the spouts/bolts until it is first activated. + * Can now activate, deactive, rebalance, and kill topologies from the Storm UI (thanks Frostman) + * Can now use --config option to override which yaml file from ~/.storm to use for the config (thanks tjun) + * Redesigned the pluggable resource scheduler (INimbus, ISupervisor) interfaces to allow for much simpler integrations + * Added prepare method to IScheduler + * Added "throws Exception" to TestJob interface + * Added reportError to multilang protocol and updated Python and Ruby adapters to use it (thanks Lazyshot) + * Number tuples executed now tracked and shown in Storm UI + * Added ReportedFailedException which causes a batch to fail without killing worker and reports the error to the UI + * Execute latency now tracked and shown in Storm UI + * Adding testTuple methods for easily creating Tuple instances to Testing API (thanks xumingming) + * Trident now throws an error during construction of a topology when try to select fields that don't exist in a stream (thanks xumingming) + * Compute the capacity of a bolt based on execute latency and #executed over last 10 minutes and display in UI + * Storm UI displays exception instead of blank page when there's an error rendering the page (thanks Frostman) + * Added MultiScheme interface (thanks sritchie) + * Added MockTridentTuple for testing (thanks emblem) + * Add whitelist methods to Cluster to allow only a subset of hosts to be revealed as available slots + * Updated Trident Debug filter to take in an identifier to use when logging (thanks emblem) + * Number of DRPC server worker threads now customizable (thanks xiaokang) + * DRPC server now uses a bounded queue for requests to prevent being overloaded with requests (thanks xiaokang) + * Add __hash__ method to all generated Python Thrift objects so that Python code can read Nimbus stats which use Thrift objects as dict keys + * Bug fix: Fix for bug that could cause topology to hang when ZMQ blocks sending to a worker that got reassigned + * Bug fix: Fix deadlock bug due to variant of dining philosophers problem. Spouts now use an overflow buffer to prevent blocking and guarantee that it can consume the incoming queue of acks/fails. + * Bug fix: Fix race condition in supervisor that would lead to supervisor continuously crashing due to not finding "stormconf.ser" file for an already killed topology + * Bug fix: bin/storm script now displays a helpful error message when an invalid command is specified + * Bug fix: fixed NPE when emitting during emit method of Aggregator + * Bug fix: URLs with periods in them in Storm UI now route correctly + * Bug fix: Fix occasional cascading worker crashes due when a worker dies due to not removing connections from connection cache appropriately + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2013-12-08-storm090-released.md ---------------------------------------------------------------------- diff --git a/_posts/2013-12-08-storm090-released.md b/_posts/2013-12-08-storm090-released.md new file mode 100644 index 0000000..8e1e948 --- /dev/null +++ b/_posts/2013-12-08-storm090-released.md @@ -0,0 +1,127 @@ +--- +layout: post +title: Storm 0.9.0 Released +author: P. Taylor Goetz +--- + +We are pleased to announce that Storm 0.9.0 has been released and is available from [the downloads page](/downloads.html). This release represents an important milestone in the evolution of Storm. + +While a number of new features have been added, a key focus area for this release has been stability-related fixes. Though many users are successfully running work-in-progress builds for Storm 0.9.x in production, this release represents the most stable version to-date, and is highly recommended for everyone, especially users of 0.8.x versions. + + +Netty Transport +--------------- +The first hightlight of this release is the new [Netty](http://netty.io/index.html) Transport contributed by [Yahoo! Engineering](http://yahooeng.tumblr.com/). Storm's core network transport mechanism is now plugable, and Storm now comes with two implementations: The original 0MQ transport, and a new Netty-based implementation. + +In earlier versions, Storm relied solely on 0MQ for transport. Since 0MQ is a native library, it was highly platform-dependent and, at times, challenging to install properly. In addition, stability between versions varied widely between versions and only a relatively old 0MQ version (2.1.7) was certified to work with Storm. + +The Netty transport offers a pure Java alternative that eliminates Storm's dependency on native libraries. The Netty transport's performance is up to twice as fast as 0MQ, and it will open the door for authorization and authentication between worker processes. For an in-depth performance comparison of the 0MQ and Netty transports, see [this blog post](http://yahooeng.tumblr.com/post/64758709722/making-storm-fly-with-netty) by Storm contributor [Bobby Evans](https://github.com/revans2). + +To configure Storm to use the Netty transport simply add the following to your `storm.yaml` configuration and adjust the values to best suit your use-case: + +``` +storm.messaging.transport: "backtype.storm.messaging.netty.Context" +storm.messaging.netty.server_worker_threads: 1 +storm.messaging.netty.client_worker_threads: 1 +storm.messaging.netty.buffer_size: 5242880 +storm.messaging.netty.max_retries: 100 +storm.messaging.netty.max_wait_ms: 1000 +storm.messaging.netty.min_wait_ms: 100 +``` +You can also write your own transport implementation by implementing the [`backtype.storm.messaging.IContext`](https://github.com/apache/incubator-storm/blob/master/storm-core/src/jvm/backtype/storm/messaging/IContext.java) interface. + + +Log Viewer UI +------------- +Storm now includes a helpful new feature for debugging and monitoring topologies: The `logviewer` daemon. + +In earlier versions of Storm, viewing worker logs involved determining a worker's location (host/port), typically through Storm UI, then `ssh`ing to that host and `tail`ing the corresponding worker log file. With the new log viewer. You can now easily access a specific worker's log in a web browser by clicking on a worker's port number right from Storm UI. + +The `logviewer` daemon runs as a separate process on Storm supervisor nodes. To enable the `logviewer` run the following command (under supervision) on your cluster's supervisor nodes: + +``` +$ storm logviewer +``` + + +Improved Windows Support +------------------------ +In previous versions, running Storm on Microsoft Windows required installing third-party binaries (0MQ), modifying Storm's source, and adding Windows-specific scripts. With the addition of the platform-independent Netty transport, as well as numerous enhancements to make Storm more platform-independent, running Storm on Windows is easier than ever. + + +Security Improvements +--------------------- +Security, Authentication, and Authorization have been and will continue to be important focus areas for upcoming features. Storm 0.9.0 introduces an API for pluggable tuple serialization and a blowfish encryption based implementation for encrypting tuple data for sensitive use cases. + + +API Compatibility and Upgrading +------------------------------- +For most Storm topology developers, upgrading to 0.9.0 is simply a matter of updating the [dependency](https://clojars.org/storm). Storm's core API has changed very little since the 0.8.2 release. + +On the devops side, when upgrading to a new Storm release, it is safest to clear any existing state (Zookeeper, `storm.local.dir`), prior to upgrading. + +Logging Changes +--------------- +Another important change in 0.9.0 has to do with logging. Storm has largely switched over to the [slf4j API](http://www.slf4j.org) (backed by a [logback](http://logback.qos.ch) logger implementation). Some Storm dependencies rely on the log4j API, so Storm currently depends on [log4j-over-slf4j](http://www.slf4j.org/legacy.html#log4j-over-slf4j). + +These changes have implications for existing topologies and topology components that use the log4j API. + +In general, and when possible, Storm topologies and topology components should use the [slf4j API](http://www.slf4j.org) for logging. + + +Thanks +------ +Special thanks are due to all those who have contributed to Storm -- whether through direct code contributions, documentation, bug reports, or helping other users on the mailing lists. Your efforts are much appreciated. + + +Changelog +--------- + +* Update build configuration to force compatibility with Java 1.6 +* Fixed a netty client issue where sleep times for reconnection could be negative (thanks brndnmtthws) +* Fixed an issue that would cause storm-netty unit tests to fail +* Added configuration to limit ShellBolt internal _pendingWrites queue length (thanks xiaokang) +* Fixed a a netty client issue where sleep times for reconnection could be negative (thanks brndnmtthws) +* Fixed a display issue with system stats in Storm UI (thanks d2r) +* Nimbus now does worker heartbeat timeout checks as soon as heartbeats are updated (thanks d2r) +* The logviewer now determines log file location by examining the logback configuration (thanks strongh) +* Allow tick tuples to work with the system bolt (thanks xumingming) +* Add default configuration values for the netty transport and the ability to configure the number of worker threads (thanks revans2) +* Added timeout to unit tests to prevent a situation where tests would hang indefinitely (thanks d2r) +* Fixed an issue in the system bolt where local mode would not be detected accurately (thanks miofthena) +* Fixed `storm jar` command to work properly when STORM_JAR_JVM_OPTS is not specified (thanks roadkill001) +* All logging now done with slf4j +* Replaced log4j logging system with logback +* Logs are now limited to 1GB per worker (configurable via logging configuration file) +* Build upgraded to leiningen 2.0 +* Revamped Trident spout interfaces to support more dynamic spouts, such as a spout who reads from a changing set of brokers +* How tuples are serialized is now pluggable (thanks anfeng) +* Added blowfish encryption based tuple serialization (thanks anfeng) +* Have storm fall back to installed storm.yaml (thanks revans2) +* Improve error message when Storm detects bundled storm.yaml to show the URL's for offending resources (thanks revans2) +* Nimbus throws NotAliveException instead of FileNotFoundException from various query methods when topology is no longer alive (thanks revans2) +* Escape HTML and Javascript appropriately in Storm UI (thanks d2r) +* Storm's Zookeeper client now uses bounded exponential backoff strategy on failures +* Automatically drain and log error stream of multilang subprocesses +* Append component name to thread name of running executors so that logs are easier to read +* Messaging system used for passing messages between workers is now pluggable (thanks anfeng) +* Netty implementation of messaging (thanks anfeng) +* Include topology id, worker port, and worker id in properties for worker processes, useful for logging (thanks d2r) +* Tick tuples can now be scheduled using floating point seconds (thanks tscurtu) +* Added log viewer daemon and links from UI to logviewers (thanks xiaokang) +* DRPC server childopts now configurable (thanks strongh) +* Default number of ackers to number of workers, instead of just one (thanks lyogavin) +* Validate that Storm configs are of proper types/format/structure (thanks d2r) +* FixedBatchSpout will now replay batches appropriately on batch failure (thanks ptgoetz) +* Can set JAR_JVM_OPTS env variable to add jvm options when calling 'storm jar' (thanks srmelody) +* Throw error if batch id for transaction is behind the batch id in the opaque value (thanks mrflip) +* Sort topologies by name in UI (thanks jaked) +* Added LoggingMetricsConsumer to log all metrics to a file, by default not enabled (thanks mrflip) +* Add prepare(Map conf) method to TopologyValidator (thanks ankitoshniwal) +* Bug fix: Supervisor provides full path to workers to logging config rather than relative path (thanks revans2) +* Bug fix: Call ReducerAggregator#init properly when used within persistentAggregate (thanks lorcan) +* Bug fix: Set component-specific configs correctly for Trident spouts + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-10-storm-logo-contest.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-10-storm-logo-contest.md b/_posts/2014-04-10-storm-logo-contest.md new file mode 100644 index 0000000..59f2a5d --- /dev/null +++ b/_posts/2014-04-10-storm-logo-contest.md @@ -0,0 +1,66 @@ +--- +layout: post +title: Apache Storm Logo Contest +author: P. Taylor Goetz +--- + +Apache Storm is a platform for distributed, real-time computation that has been undergoing incubation at Apache since September 18, 2013. You can contribute by helping establish the Storm brand by submitting a proposal for the Apache Storm logo. + +Your proposal will be published on the [Storm project website](http://storm.incubator.apache.org) next to the other proposals. On **Wednesday, April 30, 2014** the community will choose the new logo through a vote on the [Apache Storm Mailing List](http://mail-archives.apache.org/mod_mbox/incubator-storm-dev/). If more than 10 proposals are received, voting will take place in two rounds, with the first round selecting 10 proposals to move to round two. + +The winning artist will be credited on the Apache Storm website as having designed the logo, and may include a link to her/his website or blog. Prizes for the winner(s) are currently TBD. + +##GUIDELINES + +The logo should meet the following requirements: + + * Include 2 versions of the logo: a rectangular layout (full brand) and a square layout (icon, stand-alone logo) + * Be scalable up or down (hi-res raster, or vector) + * Translate to a 2-color representation (black/white, 2 greys, etc.) + * Able to be inverted if running on a dark background + * Comply with the [Apache branding requirements](http://www.apache.org/foundation/marks/) + +Optional, but nice to have: + + * Icons representing storm components (suitable for use in diagrams, presentations, etc.): + * Spouts + * Bolts + * Streams + * Trident + * Functions/Filters + +Note: The winning design as accepted by popular vote will be treated as a proposal that can be modified and iterated upon to reach its final form. + +There is no limit on the number of entries per individual. + +##INSPIRATION + +While designing the logo, keep in mind what describes Storm: + + * Distributed Computation + * High Performance + * Fault Tolerance + * Community Driven + + +##DEADLINE + +The contest will be open until **Wednesday, April 30, 2014**. + +##HOW TO ENTER + +Send the [Apache Storm Mailing List](mailto:[email protected]) links to: + + * A PNG version of the logo (and it's scalable variations) + * A ZIP file with all assets + * (Optional) A link to a blog post, etc. describing the proposal + +Alternatively, contributers can mail the above to the [contest coordinator](mailto:[email protected]) to have them forwarded to the mailing list. + +Contributors may also publicise their proposal by tweeting links to a proposal with the hashtag #stormlogocontest + + +##LICENSE + +The authors of the selected Apache Storm logo(s) will be required to donate them to the Apache Storm project and complete and [Apache Individual Contributor License Agreement(ICLA)](http://www.apache.org/licenses/icla.txt) + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-17-logo-pforrest.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-17-logo-pforrest.md b/_posts/2014-04-17-logo-pforrest.md new file mode 100644 index 0000000..edde18a --- /dev/null +++ b/_posts/2014-04-17-logo-pforrest.md @@ -0,0 +1,12 @@ +--- +layout: post +title: Logo Entry No. 1 - Patricia Forrest +--- + +{{ page.title }} +================ + + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-17-logo-squinones.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-17-logo-squinones.md b/_posts/2014-04-17-logo-squinones.md new file mode 100644 index 0000000..89bf877 --- /dev/null +++ b/_posts/2014-04-17-logo-squinones.md @@ -0,0 +1,11 @@ +--- +layout: post +title: Logo Entry No. 2 - Samuel Quiñones +--- + +{{ page.title }} +================ + + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-19-logo-ssuleman.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-19-logo-ssuleman.md b/_posts/2014-04-19-logo-ssuleman.md new file mode 100644 index 0000000..071f950 --- /dev/null +++ b/_posts/2014-04-19-logo-ssuleman.md @@ -0,0 +1,10 @@ +--- +layout: post +title: Logo Entry No. 3- Shaan Shiv Suleman +--- + +{{ page.title }} +================ + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-21-logo-rmarshall.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-21-logo-rmarshall.md b/_posts/2014-04-21-logo-rmarshall.md new file mode 100644 index 0000000..3f910b3 --- /dev/null +++ b/_posts/2014-04-21-logo-rmarshall.md @@ -0,0 +1,14 @@ +--- +layout: post +title: Logo Entry No. 4 - Richard Brownlie-Marshall +--- + +{{ page.title }} +================ + + + + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-22-logo-zsayari.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-22-logo-zsayari.md b/_posts/2014-04-22-logo-zsayari.md new file mode 100644 index 0000000..d7175a1 --- /dev/null +++ b/_posts/2014-04-22-logo-zsayari.md @@ -0,0 +1,11 @@ +--- +layout: post +title: Logo Entry No. 5 - Ziba Sayari +--- + +{{ page.title }} +================ + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-23-logo-abartos.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-23-logo-abartos.md b/_posts/2014-04-23-logo-abartos.md new file mode 100644 index 0000000..7e99126 --- /dev/null +++ b/_posts/2014-04-23-logo-abartos.md @@ -0,0 +1,17 @@ +--- +layout: post +title: Logo Entry No. 6 - Alec Bartos +--- + +{{ page.title }} +================ + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-27-logo-cboustead.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-27-logo-cboustead.md b/_posts/2014-04-27-logo-cboustead.md new file mode 100644 index 0000000..1069413 --- /dev/null +++ b/_posts/2014-04-27-logo-cboustead.md @@ -0,0 +1,14 @@ +--- +layout: post +title: Logo Entry No. 7 - Calum Boustead +--- + +{{ page.title }} +================ + + + + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-27-logo-sasili.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-27-logo-sasili.md b/_posts/2014-04-27-logo-sasili.md new file mode 100644 index 0000000..bf475bc --- /dev/null +++ b/_posts/2014-04-27-logo-sasili.md @@ -0,0 +1,12 @@ +--- +layout: post +title: Logo Entry No. 8 - Stefano Asili +--- + +{{ page.title }} +================ + + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-29-logo-jlee1.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-29-logo-jlee1.md b/_posts/2014-04-29-logo-jlee1.md new file mode 100644 index 0000000..2f03f51 --- /dev/null +++ b/_posts/2014-04-29-logo-jlee1.md @@ -0,0 +1,12 @@ +--- +layout: post +title: Logo Entry No. 9 - Jennifer Lee +--- + +{{ page.title }} +================ + + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-29-logo-jlee2.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-29-logo-jlee2.md b/_posts/2014-04-29-logo-jlee2.md new file mode 100644 index 0000000..c8b732b --- /dev/null +++ b/_posts/2014-04-29-logo-jlee2.md @@ -0,0 +1,12 @@ +--- +layout: post +title: Logo Entry No. 10 - Jennifer Lee +--- + +{{ page.title }} +================ + + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-04-29-logo-jlee3.md ---------------------------------------------------------------------- diff --git a/_posts/2014-04-29-logo-jlee3.md b/_posts/2014-04-29-logo-jlee3.md new file mode 100644 index 0000000..a9fa18f --- /dev/null +++ b/_posts/2014-04-29-logo-jlee3.md @@ -0,0 +1,12 @@ +--- +layout: post +title: Logo Entry No. 11 - Jennifer Lee +--- + +{{ page.title }} +================ + + + + + http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-05-27-round1-results.md ---------------------------------------------------------------------- diff --git a/_posts/2014-05-27-round1-results.md b/_posts/2014-05-27-round1-results.md new file mode 100644 index 0000000..49d46b0 --- /dev/null +++ b/_posts/2014-05-27-round1-results.md @@ -0,0 +1,40 @@ +--- +layout: post +title: Logo Contest - Round 1 Results +--- + +{{ page.title }} +================ + +Round one of the Apache Storm logo contest is now complete and was a great success. We received votes from 7 PPMC members as well as 46 votes from the greater Storm community. + +We would like to extend a very special thanks to all those who took the time and effort to create and submit a logo proposal. We would also like to thank everyone who voted. + +## Results ## +The Storm PPMC and community votes were closely aligned, with the community vote resolving a PPMC tie for the 3rd finalist as shown in the result table below. + +The three finalists entering the final round are: + + * [No. 6 - Alec Bartos](/2014/04/23/logo-abartos.html) + * [No. 9 - Jennifer Lee](/2014/04/29/logo-jlee1.html) + * [No. 10 - Jennifer Lee](/2014/04/29/logo-jlee2.html) + +Congratulations Alec and Jennifer! + +Stay tuned for the final vote, which will be announced shortly. + +------ + +| Entry | PPMC | Community | +|:-----------------------------|-----:|----------:| +|1 - Patricia Forrest | 2 | 23 | +|2 - Samuel Quiñones | 3 | 6 | +|3- Shaan Shiv Suleman | 0 | 7 | +|4 - Richard Brownlie-Marshall | 0 | 6 | +|5 - Ziba Sayari | 0 | 9 | +|6 - Alec Bartos | 3 | 32 | +|7 - Calum Boustead | 0 | 0 | +|8 - Stefano Asili | 0 | 2 | +|9 - Jennifer Lee | 9 | 63 | +|10 - Jennifer Lee | 18 | 64 | +|11 - Jennifer Lee | 0 | 18 | \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-06-17-contest-results.md ---------------------------------------------------------------------- diff --git a/_posts/2014-06-17-contest-results.md b/_posts/2014-06-17-contest-results.md new file mode 100644 index 0000000..4b6b7de --- /dev/null +++ b/_posts/2014-06-17-contest-results.md @@ -0,0 +1,24 @@ +--- +layout: post +title: Storm Logo Contest Results +author: P. Taylor Goetz +--- + +The Apache Storm logo contest is now complete and was a great success. We received votes from 7 PPMC members as well as 55 votes from the greater Storm community. Thank you to everyone who participated by voting. + +## The Winner ## +Congratulations to Jennifer Lee, whose [3rd entry](/2014/04/29/logo-jlee2.html) received the most points from both the PPMC as well as the Storm community. Final vote tallies are listed below. + + + +The Apache Storm project management team will now begin the process of finalizing the logo and making it official. + +Congratulations Jennifer, and thanks again to everyone who participated! + +------ + +| Entry | PPMC | Community | +|:-----------------------------|-----:|----------:| +|[6 - Alec Bartos](/2014/04/23/logo-abartos.html) | 2 | 41 | +|[9 - Jennifer Lee](/2014/04/29/logo-jlee1.html) | 7 | 111 | +|[10 - Jennifer Lee](/2014/04/29/logo-jlee2.html) | 26 | 123 | http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-06-25-storm092-released.md ---------------------------------------------------------------------- diff --git a/_posts/2014-06-25-storm092-released.md b/_posts/2014-06-25-storm092-released.md new file mode 100644 index 0000000..7f8c438 --- /dev/null +++ b/_posts/2014-06-25-storm092-released.md @@ -0,0 +1,137 @@ +--- +layout: post +title: Storm 0.9.2 released +author: P. Taylor Goetz +--- + +We are pleased to announce that Storm 0.9.2-incubating has been released and is available from [the downloads page](/downloads.html). This release includes many important fixes and improvements. + +Netty Transport Improvements +---------------------------- +Storm's Netty-based transport has been overhauled to significantly improve performance through better utilization of thread, CPU, and network resources, particularly in cases where message sizes are small. Storm contributor Sean Zhong ([@clockfly](https://github.com/clockfly)) deserves a great deal of credit not only for discovering, analyzing, documenting and fixing the root cause, but also for persevering through an extended review process and promptly addressing all concerns. + +Those interested in the technical details and evolution of this patch can find out more in the [JIRA ticket for STORM-297](https://issues.apache.org/jira/browse/STORM-297). + +Sean also discovered and fixed an [elusive bug](https://issues.apache.org/jira/browse/STORM-342) in Storm's usage of the Disruptor queue that could lead to out-of-order or lost messages. + +Many thanks to Sean for contributing these important fixes. + +Storm UI Improvements +--------------------- +This release also includes a number of improvements to the Storm UI service. Contributor Sriharsha Chintalapani([@harshach](https://github.com/harshach)) added a REST API to the Storm UI service to expose metrics and operations in JSON format, and updated the UI to use that API. + +The new REST API will make it considerably easier for other services to consume availabe cluster and topology metrics for monitoring and visualization applications. Kyle Nusbaum ([@knusbaum](https://github.com/knusbaum)) has already leveraged the REST API to create a topology visualization tool now included in Storm UI and illustrated in the screenshot below. + + + + + + + +In the visualization, spout components are represented as blue, while bolts are colored between green and red depending on their associated capacity metric. The width of the lines between the components represent the flow of tuples relative to the other visible streams. + +Kafka Spout +----------- +This is the first Storm release to include official support for consuming data from Kafka 0.8.x. In the past, development of Kafka spouts for Storm had become somewhat fragmented and finding an implementation that worked with certain versions of Storm and Kafka proved burdensome for some developers. This is no longer the case, as the `storm-kafka` module is now part of the Storm project and associated artifacts are released to official channels (Maven Central) along with Storm's other components. + +Thanks are due to GitHub user [@wurstmeister]() for picking up Nathan Marz' original Kafka 0.7.x implementation, updating it to work with Kafka 0.8.x, and contributing that work back to the Storm community. + +The `storm-kafka` module can be found in the `/external/` directory of the source tree and binary distributions. The `external` area has been set up to contain projects that while not required by Storm, are often used in conjunction with Storm to integrate with some other technology. Such projects also come with a maintenance committment from at least one Storm committer to ensure compatibility with Storm's main codebase as it evolves. + +The `storm-kafka` dependency is available now from Maven Central at the following coordinates: + + + groupId: org.apache.storm + artifactId: storm-kafka + version: 0.9.2-incubating + +Users, and Scala developers in particular, should note that the Kafka dependency is listed as `provided`. This allows users to choose a specific Scala version as described in the [project README](https://github.com/apache/incubator-storm/tree/v0.9.2-incubating/external/storm-kafka). + + +Storm Starter and Examples +-------------------------- + +Similar to the `external` section of the codebase, we have also added an `examples` directory and pulled in the `storm-starter` project to ensure it will be maintained in lock-step with Storm's main codebase. + +Thank you to Storm committer Michael G. Noll for his continued work in maintaining and improving the `storm-starter` project. + + +Plugable Serialization for Multilang +------------------------------------ +In previous versions of Storm, serialization of data to and from multilang components was limited to JSON, imposing somewhat of performance penalty. Thanks to a contribution from John Gilmore ([@jsgilmore](https://github.com/jsgilmore)) the serialization mechanism is now plugable and enables the use of more performant serialization frameworks like protocol buffers in addition to JSON. + + +Thanks +------ +Special thanks are due to all those who have contributed to Storm -- whether through direct code contributions, documentation, bug reports, or helping other users on the mailing lists. Your efforts are much appreciated. + + +Changelog +--------- + + * STORM-352: [storm-kafka] PartitionManager does not save offsets to ZooKeeper + * STORM-66: send taskid on initial handshake + * STORM-342: Contention in Disruptor Queue which may cause out of order or lost messages + * STORM-338: Move towards idiomatic Clojure style + * STORM-335: add drpc test for removing timed out requests from queue + * STORM-69: Storm UI Visualizations for Topologies + * STORM-297: Performance scaling with CPU + * STORM-244: DRPC timeout can return null instead of throwing an exception + * STORM-63: remove timeout drpc request from its function's request queue + * STORM-313: Remove log-level-page from logviewer + * STORM-205: Add REST API To Storm UI + * STORM-326: tasks send duplicate metrics + * STORM-331: Update the Kafka dependency of storm-kafka to 0.8.1.1 + * STORM-308: Add support for config_value to {supervisor,nimbus,ui,drpc,logviewer} childopts + * STORM-309: storm-starter Readme: windows documentation update + * STORM-318: update storm-kafka to use apache curator-2.4.0 + * STORM-303: storm-kafka reliability improvements + * STORM-233: Removed inline heartbeat to nimbus to avoid workers being killed when under heavy ZK load + * STORM-267: fix package name of LoggingMetricsConsumer in storm.yaml.example + * STORM-265: upgrade to clojure 1.5.1 + * STORM-232: ship JNI dependencies with the topology jar + * STORM-295: Add storm configuration to define JAVA_HOME + * STORM-138: Pluggable serialization for multilang + * STORM-264: Removes references to the deprecated topology.optimize + * STORM-245: implement Stream.localOrShuffle() for trident + * STORM-317: Add SECURITY.md to release binaries + * STORM-310: Change Twitter authentication + * STORM-305: Create developer documentation + * STORM-280: storm unit tests are failing on windows + * STORM-298: Logback file does not include full path for metrics appender fileNamePattern + * STORM-316: added validation to registermetrics to have timebucketSizeInSecs >= 1 + * STORM-315: Added progress bar when submitting topology + * STORM-214: Windows: storm.cmd does not properly handle multiple -c arguments + * STORM-306: Add security documentation + * STORM-302: Fix Indentation for pom.xml in storm-dist + * STORM-235: Registering a null metric should blow up early + * STORM-113: making thrift usage thread safe for local cluster + * STORM-223: use safe parsing for reading YAML + * STORM-238: LICENSE and NOTICE files are duplicated in storm-core jar + * STORM-276: Add support for logviewer in storm.cmd. + * STORM-286: Use URLEncoder#encode with the encoding specified. + * STORM-296: Storm kafka unit tests are failing on windows + * STORM-291: upgrade http-client to 4.3.3 + * STORM-252: Upgrade curator to latest version + * STORM-294: Commas not escaped in command line + * STORM-287: Fix the positioning of documentation strings in clojure code + * STORM-290: Fix a log binding conflict caused by curator dependencies + * STORM-289: Fix Trident DRPC memory leak + * STORM-173: Treat command line "-c" option number config values as such + * STORM-194: Support list of strings in *.worker.childopts, handle spaces + * STORM-288: Fixes version spelling in pom.xml + * STORM-208: Add storm-kafka as an external module + * STORM-285: Fix storm-core shade plugin config + * STORM-12: reduce thread usage of netty transport + * STORM-281: fix and issue with config parsing that could lead to leaking file descriptors + * STORM-196: When JVM_OPTS are set, storm jar fails to detect storm.jar from environment + * STORM-260: Fix a potential race condition with simulated time in Storm's unit tests + * STORM-258: Update commons-io version to 2.4 + * STORM-270: don't package .clj files in release jars. + * STORM-273: Error while running storm topologies on Windows using "storm jar" + * STROM-247: Replace links to github resources in storm script + * STORM-263: Update Kryo version to 2.21+ + * STORM-187: Fix Netty error "java.lang.IllegalArgumentException: timeout value is negative" + * STORM-186: fix float secs to millis long convertion + * STORM-70: Upgrade to ZK-3.4.5 and curator-1.3.3 + * STORM-146: Unit test regression when storm is compiled with 3.4.5 zookeeper http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-10-20-storm093-release-candidate.md ---------------------------------------------------------------------- diff --git a/_posts/2014-10-20-storm093-release-candidate.md b/_posts/2014-10-20-storm093-release-candidate.md new file mode 100644 index 0000000..6270283 --- /dev/null +++ b/_posts/2014-10-20-storm093-release-candidate.md @@ -0,0 +1,11 @@ +--- +layout: post +title: Storm 0.9.3 release candidate 1 available +author: Michael G. Noll +--- + +We are pleased to announce that Apache Storm version 0.9.3 has reached the release candidate stage. The latest release is 0.9.3-rc1. We are moving closer to the GA release, so this is a good time to give this new 0.9.3 version a try! + +Storm 0.9.3 is a maintenance release with bug fixes and better stability as summarized in the [0.9.3 changelog](https://github.com/apache/storm/blob/master/CHANGELOG.md). On top of that version 0.9.3 continues to improve the integration with [Apache Kafka](http://kafka.apache.org/) as well as the [documentation of the core spout and Trident spout for Kafka](https://github.com/apache/storm/blob/master/external/storm-kafka/README.md), which will provide our users with the best Kafka connectivity for Storm to date. + +We heartily encourage you to [test the 0.9.3 release candidate](http://storm.apache.org/downloads.html) and provide your feedback regarding any issues via [our mailing lists](http://storm.apache.org/community.html), which is an easy and valuable way to contribute back to the Storm community and to help us moving to an official 0.9.3 release. You can find the [0.9.3 release candidate in our Downloads section](http://storm.apache.org/downloads.html). http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2014-11-25-storm093-released.md ---------------------------------------------------------------------- diff --git a/_posts/2014-11-25-storm093-released.md b/_posts/2014-11-25-storm093-released.md new file mode 100644 index 0000000..df3da75 --- /dev/null +++ b/_posts/2014-11-25-storm093-released.md @@ -0,0 +1,164 @@ +--- +layout: post +title: Storm 0.9.3 released +author: P. Taylor Goetz +--- + +We are pleased to announce that Apache Storm 0.9.3 has been released and is available from [the downloads page](/downloads.html). This release includes 100 new fixes and improvements from 62 individual contributors. + +Improved Kafka Integration +---------------------------- + +Apache Storm has supported [Apache Kafka](http://kafka.apache.org/) as a streaming data source since version 0.9.2-incubating. Version 0.9.3 brings a number of improvements to the Kafka integration including the ability to write data to one or more Kafka clusters and topics. + +The ability to both read from and write to Kafka further unlocks potential of the already powerful combination of Storm and Kafka. Storm users can now use Kafka as a source of and destination for streaming data. This allows for inter-topology communication, topology chaining, combining spout/bolt-based topologies with Trident-based data flows, and integration with any external system that supports data ingest from Kafka. + +More information about Storm's Kafka integration can be found in the [storm-kafka project documentation](https://github.com/apache/storm/blob/v0.9.3/external/storm-kafka/README.md). + + +HDFS Integration +---------------- + +Many stream processing use cases involve storing data in HDFS for further analysis and offline (batch) processing. Apache Stormâs HDFS integration consists of several bolt and Trident state implementations that allow topology developers to easily write data to HDFS from any Storm topology. + +More information about Storm's HDFS integration can be found in the [storm-hdfs project documentation](https://github.com/apache/storm/tree/v0.9.3/external/storm-hdfs). + + +HBase Integration +----------------- + +Apache Stormâs HBase integration includes a number of components that allow Storm topologies to both write to and query HBase in real-time. Many organizations use Apache HBase as part of their big data strategy for batch, interactive, and real-time workflows. Stormâs HBase integration allows users to leverage data assets in HBase as streaming queries, as well as using HBase as a destination for streaming computation results. + +More information about Storm's HBase integration can be found in the [storm-hbase project documentation](https://github.com/apache/storm/tree/v0.9.3/external/storm-hbase). + +Reduced Dependency Conflicts +---------------------------- +In previous Storm releases, it was not uncommon for users' topology dependencies to conflict with the libraries used by Storm. In Storm 0.9.3 several dependency packages that were common sources of conflicts have been package-relocated (shaded) to avoid this situation. Developers are free to use the Storm-packaged versions, or supply their own version. + +The following table lists the dependency package relocations: + +| Dependency | Original Package | Storm Package | +|:---|:---|:---| +| Apache Thrift | `org.apache.thrift` | `org.apache.thrift7` | +| Netty | `org.jboss.netty` | `org.apache.storm.netty` | +| Google Guava | `com.google.common` | `org.apache.storm.guava` | +| | `com.google.thirdparty` | `org.apache.storm.guava.thirdparty` | +| Apache HTTPClient | `org.apache.http` | `org.apache.storm.http` | +| Apache ZooKeeper | `org.apache.zookeeper` | `org.apache.storm.zookeeper` | +| Apache Curator | `org.apache.curator` | `org.apache.storm.curator` | + +Multi-Lang Improvements +----------------------- +Apache Storm 0.9.3 includes a new [Node.js](http://nodejs.org) multi-lang implementation that allows developers to write spouts and bolts in JavaScript. + +In addition to the Node.js implementation, the multi-lang protocol has been substantially improved in terms of robustness and error handling capabilities. As a result, **the multi-lang API has changed in a non-backward-compatible way**. Users with existing multi-lang topologies should consult the Python, Ruby, and JavaScript multi-lang examples to determine the impact prior to upgrading. + + +Thanks +------ +Special thanks are due to all those who have contributed to Storm -- whether through direct code contributions, documentation, bug reports, or helping other users on the mailing lists. Your efforts are much appreciated. + + +Full Changelog +--------- + + * STORM-558: change "swap!" to "reset!" to fix assignment-versions in supervisor + * STORM-555: Storm json response should set charset to UTF-8 + * STORM-513: check heartbeat from multilang subprocess + * STORM-549: "topology.enable.message.timeouts" does nothing + * STORM-546: Local hostname configuration ignored by executor + * STORM-492: Test timeout should be configurable + * STORM-540: Change default time format in logs to ISO8601 in order to include timezone + * STORM-511: Storm-Kafka spout keeps sending fetch requests with invalid offset + * STORM-538: Guava com.google.thirdparty.publicsuffix is not shaded + * STORM-497: don't modify the netty server taskToQueueId mapping while the someone could be reading it. + * STORM-537: A worker reconnects infinitely to another dead worker (Sergey Tryuber) + * STORM-519: add tuple as an input param to HBaseValueMapper + * STORM-488: Exit with 254 error code if storm CLI is run with unknown command + * STORM-506: Do not count bolt acks & fails in total stats + * STORM-490: fix build under Windows + * STORM-439: Replace purl.js qith jquery URL plugin + * STORM-499: Document and clean up shaded dependncy resolution with maven + * STORM-210: Add storm-hbase module + * STORM-507: Topology visualization should not block ui + * STORM-504: Class used by `repl` command is deprecated. + * STORM-330: Implement storm exponential backoff stategy for netty client and curator + * STORM-461: exit-process! does not always exit the process, but throws an exception + * STORM-341: fix assignment sorting + * STORM-476: external/storm-kafka: avoid NPE on null message payloads + * STORM-424: fix error message related to kafka offsets + * STORM-454: correct documentation in STORM-UI-REST-API.md + * STORM-474: Reformat UI HTML code + * STORM-447: shade/relocate packages of dependencies that are common causes of dependency conflicts + * STORM-279: cluster.xml doesn't take in STORM_LOG_DIR values. + * STORM-380: Kafka spout: throw RuntimeException if a leader cannot be found for a partition + * STORM-435: Improve storm-kafka documentation + * STORM-405: Add kafka trident state so messages can be sent to kafka topic + * STORM-211: Add module for HDFS integration + * STORM-337: Expose managed spout ids publicly + * STORM-320: Support STORM_CONF_DIR environment variable to support + * STORM-360: Add node details for Error Topology and Component pages + * STORM-54: Per-Topology Classpath and Environment for Workers + * STORM-355: excluding outdated netty transitively included via curator + * STORM-183: Replacing RunTime.halt() with RunTime.exit() + * STORM-213: Decouple In-Process ZooKeeper from LocalCluster. + * STORM-365: Add support for Python 3 to storm command. + * STORM-332: Enable Kryo serialization in storm-kafka + * STORM-370: Add check for empty table before sorting dom in UI + * STORM-359: add logviewer paging and download + * STORM-372: Typo in storm_env.ini + * STORM-266: Adding shell process pid and name in the log message + * STORM-367: Storm UI REST api documentation. + * STORM-200: Proposal for Multilang's Metrics feature + * STORM-351: multilang python process fall into endless loop + * STORM-375: Smarter downloading of assignments by supervisors and workers + * STORM-328: More restrictive Config checks, strict range check within Utils.getInt() + * STORM-381: Replace broken jquery.tablesorter.min.js to latest + * STORM-312: add storm monitor tools to monitor throughtput interactively + * STORM-354: Testing: allow users to pass TEST-TIMEOUT-MS as param for complete-topology + * STORM-254: one Spout/Bolt can register metric twice with same name in different timeBucket + * STORM-403: heartbeats-to-nimbus in supervisor-test failed due to uninten... + * STORM-402: FileNotFoundException when using storm with apache tika + * STORM-364: The exception time display as default timezone. + * STORM-420: Missing quotes in storm-starter python code + * STORM-399: Kafka Spout defaulting to latest offset when current offset is older then 100k + * STORM-421: Memoize local hostname lookup in executor + * STORM-414: support logging level to multilang protocol spout and bolt + * STORM-321: Added a tool to see the current status of STORM JIRA and github pulls. + * STORM-415: validate-launched-once in supervisor-test can not handle multiple topologies + * STORM-155: Storm rebalancing code causes multiple topologies assigned to a single port + * STORM-419: Updated test so sort ordering is very explicit. + * STORM-406: Fix for reconnect logic in netty client. + * STORM-366: Add color span to most recent error and fix ui templates. + * STORM-369: topology summary page displays wrong order. + * STORM-239: Allow supervisor to operate in paths with spaces in them + * STORM-87: fail fast on ShellBolt exception + * STORM-417: Storm UI lost table sort style when tablesorter was updated + * STORM-396: Replace NullPointerException with IllegalArgumentExeption + * STORM-451: Latest storm does not build due to a pom.xml error in storm-hdfs pom.xml + * STORM-453: migrated to curator 2.5.0 + * STORM-458: sample spout uses incorrect name when connecting bolt + * STORM-455: Report error-level messages from ShellBolt children + * STORM-443: multilang log's loglevel protocol can cause hang + * STORM-449: Updated ShellBolt to not exit when shutting down. + * STORM-464: Simulated time advanced after test cluster exits causes intermittent test failures + * STORM-463: added static version of metrics helpers for Config + * STORM-376: Add compression to serialization + * STORM-437: Enforce utf-8 when multilang reads from stdin + * STORM-361: Add JSON-P support to Storm UI API + * STORM-373: Provide Additional String substitutions for *.worker.childopts + * STORM-274: Add support for command remoteconfvalue in storm.cmd + * STORM-132: sort supervisor by free slot in desending order + * STORM-472: Improve error message for non-completeable testing spouts + * STORM-401: handle InterruptedIOException properly. + * STORM-461: exit-process! does not always exit the process, but throws an exception instead + * STORM-475: Storm UI pages do not use UTF-8 + * STORM-336: Logback version should be upgraded + * STORM-386: nodejs multilang protocol implementation and examples + * STORM-500: Add Spinner when UI is loading stats from nimbus + * STORM-501: Missing StormSubmitter API + * STORM-493: Workers inherit storm.conf.file/storm.options properties of their supervisor + * STORM-498: make ZK connection timeout configurable in Kafka spout + * STORM-428: extracted ITuple interface + * STORM-508: Update DEVELOPER.md now that Storm has graduated from Incubator + * STORM-514: Update storm-starter README now that Storm has graduated from Incubator http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2015-03-25-storm094-released.md ---------------------------------------------------------------------- diff --git a/_posts/2015-03-25-storm094-released.md b/_posts/2015-03-25-storm094-released.md new file mode 100644 index 0000000..0119e38 --- /dev/null +++ b/_posts/2015-03-25-storm094-released.md @@ -0,0 +1,24 @@ +--- +layout: post +title: Storm 0.9.4 released +author: P. Taylor Goetz +--- + +The Apache Storm community is pleased to announce that version 0.9.4 has been released and is available from [the downloads page](/downloads.html). + +This is a maintenance release that includes a number of important bug fixes that improve Storm's stability and fault tolerance. We encourage users of previous versions to upgrade to this latest release. + + +Thanks +------ +Special thanks are due to all those who have contributed to Apache Storm -- whether through direct code contributions, documentation, bug reports, or helping other users on the mailing lists. Your efforts are much appreciated. + + +Full Changelog +--------- + + * STORM-559: ZkHosts in README should use 2181 as port. + * STORM-682: supervisor should handle worker state corruption gracefully. + * STORM-693: when kafka bolt fails to write tuple, it should report error instead of silently acking. + * STORM-329: fix cascading Storm failure by improving reconnection strategy and buffering messages + * STORM-130: Supervisor getting killed due to java.io.FileNotFoundException: File '../stormconf.ser' does not exist. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/storm-site/blob/d070189e/_posts/2015-06-04-storm095-released.md ---------------------------------------------------------------------- diff --git a/_posts/2015-06-04-storm095-released.md b/_posts/2015-06-04-storm095-released.md new file mode 100644 index 0000000..1d018c0 --- /dev/null +++ b/_posts/2015-06-04-storm095-released.md @@ -0,0 +1,23 @@ +--- +layout: post +title: Storm 0.9.5 released +author: P. Taylor Goetz +--- + +The Apache Storm community is pleased to announce that version 0.9.5 has been released and is available from [the downloads page](/downloads.html). + +This is a maintenance release that includes a number of important bug fixes that improve Storm's stability and fault tolerance. We encourage users of previous versions to upgrade to this latest release. + + +Thanks +------ +Special thanks are due to all those who have contributed to Apache Storm -- whether through direct code contributions, documentation, bug reports, or helping other users on the mailing lists. Your efforts are much appreciated. + + +Full Changelog +--------- + + * STORM-790: Log "task is null" instead of letting worker die when task is null in transfer-fn + * STORM-796: Add support for "error" command in ShellSpout + * STORM-745: fix storm.cmd to evaluate 'shift' correctly with 'storm jar' + * STORM-130: Supervisor getting killed due to java.io.FileNotFoundException: File '../stormconf.ser' does not exist.
