link examples to source
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a791604a Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a791604a Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a791604a Branch: refs/heads/master Commit: a791604ac5d274cededae8a01362dac4dfd82d84 Parents: 0c1e0aa Author: P. Taylor Goetz <ptgo...@gmail.com> Authored: Tue Apr 7 00:07:49 2015 -0400 Committer: P. Taylor Goetz <ptgo...@gmail.com> Committed: Tue Apr 7 00:07:49 2015 -0400 ---------------------------------------------------------------------- .../src/test/resources/configs/hdfs_test.yaml | 97 ++++++++++++++++++++ flux-examples/README.md | 8 +- 2 files changed, 101 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/a791604a/flux-core/src/test/resources/configs/hdfs_test.yaml ---------------------------------------------------------------------- diff --git a/flux-core/src/test/resources/configs/hdfs_test.yaml b/flux-core/src/test/resources/configs/hdfs_test.yaml new file mode 100644 index 0000000..8fe0a9a --- /dev/null +++ b/flux-core/src/test/resources/configs/hdfs_test.yaml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test ability to wire together shell spouts/bolts +--- + +# topology definition +# name to be used when submitting +name: "hdfs-topology" + +# Components +# Components are analagous to Spring beans. They are meant to be used as constructor, +# property(setter), and builder arguments. +# +# for the time being, components must be declared in the order they are referenced +components: + - id: "syncPolicy" + className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy" + constructorArgs: + - 1000 + - id: "rotationPolicy" + className: "org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy" + constructorArgs: + - 5.0 + - MB + + - id: "fileNameFormat" + className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat" + configMethods: + - name: "withPath" + args: ["/tmp/foo/"] + - name: "withExtension" + args: [".txt"] + + - id: "recordFormat" + className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat" + configMethods: + - name: "withFieldDelimiter" + args: ["|"] + + - id: "rotationAction" + className: "org.apache.storm.hdfs.common.rotation.MoveFileAction" + configMethods: + - name: "toDestination" + args: ["/tmp/dest2"] + +# spout definitions +spouts: + - id: "spout-1" + className: "backtype.storm.testing.TestWordSpout" + parallelism: 1 + # ... + +# bolt definitions + +# HdfsBolt bolt = new HdfsBolt() +# .withConfigKey("hdfs.config") +# .withFsUrl(args[0]) +# .withFileNameFormat(fileNameFormat) +# .withRecordFormat(format) +# .withRotationPolicy(rotationPolicy) +# .withSyncPolicy(syncPolicy) +# .addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/")); +bolts: + - id: "bolt-1" + className: "org.apache.storm.hdfs.bolt.HdfsBolt" + configMethods: + - name: "withConfigKey" + args: ["hdfs.config"] + - name: "withFsUrl" + args: ["hdfs://hadoop:54310"] + - name: "withFileNameFormat" + args: [ref: "fileNameFormat"] + - name: "withRecordFormat" + args: [ref: "recordFormat"] + - name: "withRotationPolicy" + args: [ref: "rotationPolicy"] + - name: "withSyncPolicy" + args: [ref: "syncPolicy"] + - name: "addRotationAction" + args: [ref: "rotationAction"] + parallelism: 1 + # ... + http://git-wip-us.apache.org/repos/asf/storm/blob/a791604a/flux-examples/README.md ---------------------------------------------------------------------- diff --git a/flux-examples/README.md b/flux-examples/README.md index 9f5682e..425ad98 100644 --- a/flux-examples/README.md +++ b/flux-examples/README.md @@ -28,21 +28,21 @@ storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux - ## Available Examples -### simple_wordcount.yaml +### [simple_wordcount.yaml](src/main/resources/simple_wordcount.yaml) This is a very basic wordcount example using Java spouts and bolts. It simply logs the running count of each word received. -### multilang.yaml +### [multilang.yaml](src/main/resources/multilang.yaml) Another wordcount example that uses a spout written in JavaScript (node.js), a bolt written in Python, and two bolts written in java. -### kafka_spout.yaml +### [kafka_spout.yaml](src/main/resources/kafka_spout.yaml) This example illustrates how to configure Storm's `storm-kafka` spout using Flux YAML DSL `components`, `references`, and `constructor arguments` constructs. -### simple_hdfs.yaml +### [simple_hdfs.yaml](src/main/resources/simple_hdfs.yaml) This example demonstrates using Flux to setup a storm-hdfs bolt to write to an HDFS cluster. It also demonstrates Flux's variable substitution/filtering feature.