That's the problem. I am not well versed with this script to fix on WIN but
i guess as a workaround you might want to add $FLUME_INSTALL_DIR/lib/. this
lib has all the required jars and should get you moving forward.
Mind, filing a JIRA for the same.
- inder
On Mon, Jul 30, 2012 at 7:23 PM, mardan Khan <mardan8...@gmail.com> wrote:
> Thanks Inder,
>
> when I am running the $echo $CLASSPATH, I am just getting . (dot) and
> nothing more.
>
> Please see below the flume-ng files. Can you just pointout where I should
> put echo$CLASSPATH.
>
> Thanks
>
>
> CLEAN_FLAG=1
> ################################
> # functions
> ################################
>
> info() {
> if [ ${CLEAN_FLAG} -ne 0 ]; then
> local msg=$1
> echo "Info: $msg" >&2
> fi
> }
>
> warn() {
> if [ ${CLEAN_FLAG} -ne 0 ]; then
> local msg=$1
> echo "Warning: $msg" >&2
> fi
> }
>
> error() {
> local msg=$1
> local exit_code=$2
>
> echo "Error: $msg" >&2
>
> if [ -n "$exit_code" ] ; then
> exit $exit_code
> fi
> }
>
> # If avail, add Hadoop paths to the FLUME_CLASSPATH and to the
> # FLUME_JAVA_LIBRARY_PATH env vars.
> # Requires Flume jars to already be on FLUME_CLASSPATH.
> add_hadoop_paths() {
> local HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH"
> \
> which hadoop 2>/dev/null)
>
> if [ -f "${HADOOP_IN_PATH}" ]; then
> info "Including Hadoop libraries found via ($HADOOP_IN_PATH) for HDFS
> access"
>
> # determine hadoop java.library.path and use that for flume
> local HADOOP_CLASSPATH=""
> local HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$FLUME_CLASSPATH" \
> ${HADOOP_IN_PATH} org.apache.flume.tools.GetJavaProperty \
> java.library.path 2>/dev/null)
>
> # look for the line that has the desired property value
> # (considering extraneous output from some GC options that write to
> stdout)
> # IFS = InternalFieldSeparator (set to recognize only newline char as
> delimiter)
> IFS=$'\n'
> for line in $HADOOP_JAVA_LIBRARY_PATH; do
> if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
> HADOOP_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
> break
> fi
> done
> unset IFS
>
> if [ -n "${HADOOP_JAVA_LIBRARY_PATH}" ]; then
>
> FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HADOOP_JAVA_LIBRARY_PATH"
> fi
>
> # determine hadoop classpath
> HADOOP_CLASSPATH=$($HADOOP_IN_PATH classpath 2>/dev/null)
>
> # hack up and filter hadoop classpath
> local ELEMENTS=$(sed -e 's/:/ /g' <<<${HADOOP_CLASSPATH})
> local ELEMENT
> for ELEMENT in $ELEMENTS; do
> local PIECE
> for PIECE in $(echo $ELEMENT); do
> if [[ $PIECE =~ slf4j-(api|log4j12).*\.jar ]]; then
> info "Excluding $PIECE from classpath"
> continue
> else
> FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
> fi
> done
> done
>
> fi
> }
> add_HBASE_paths() {
> local HBASE_IN_PATH=$(PATH="${HBASE_HOME}/bin:$PATH" \
> which hbase 2>/dev/null)
>
> if [ -f "${HBASE_IN_PATH}" ]; then
> info "Including HBASE libraries found via ($HBASE_IN_PATH) for HBASE
> access"
>
> # determine HBASE java.library.path and use that for flume
> local HBASE_CLASSPATH=""
> local HBASE_JAVA_LIBRARY_PATH=$(HBASE_CLASSPATH="$FLUME_CLASSPATH" \
> ${HBASE_IN_PATH} org.apache.flume.tools.GetJavaProperty \
> java.library.path 2>/dev/null)
>
> # look for the line that has the desired property value
> # (considering extraneous output from some GC options that write to
> stdout)
> # IFS = InternalFieldSeparator (set to recognize only newline char as
> delimiter)
> IFS=$'\n'
> for line in $HBASE_JAVA_LIBRARY_PATH; do
> if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
> HBASE_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
> break
> fi
> done
> unset IFS
>
> if [ -n "${HBASE_JAVA_LIBRARY_PATH}" ]; then
>
> FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HBASE_JAVA_LIBRARY_PATH"
> fi
>
> # determine HBASE classpath
> HBASE_CLASSPATH=$($HBASE_IN_PATH classpath 2>/dev/null)
>
> # hack up and filter HBASE classpath
> local ELEMENTS=$(sed -e 's/:/ /g' <<<${HBASE_CLASSPATH})
> local ELEMENT
> for ELEMENT in $ELEMENTS; do
> local PIECE
> for PIECE in $(echo $ELEMENT); do
> if [[ $PIECE =~ slf4j-(api|log4j12).*\.jar ]]; then
> info "Excluding $PIECE from classpath"
> continue
> else
> FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
> fi
> done
> done
> FLUME_CLASSPATH="$FLUME_CLASSPATH:$HBASE_HOME/conf"
>
> fi
> }
>
> display_help() {
> cat <<EOF
> Usage: $0 <command> [options]...
>
> commands:
> help display this help text
> agent run a Flume agent
> avro-client run an avro Flume client
> version show Flume version info
>
> global options:
> --conf,-c <conf> use configs in <conf> directory
> --classpath,-C <cp> append to the classpath
> --dryrun,-d do not actually start Flume, just print the command
> -Dproperty=value sets a JDK system property value
>
> agent options:
> --conf-file,-f <file> specify a config file (required)
> --name,-n <name> the name of this agent (required)
> --help,-h display help text
>
> avro-client options:
> --host,-H <host> hostname to which events will be sent (required)
> --port,-p <port> port of the avro source (required)
> --filename,-F <file> text file to stream to avro source [default: std
> input]
> --headerFile,-R <file> headerFile containing headers as key/value pairs
> on each new line
> --help,-h display help text
>
> Note that if <conf> directory is specified, then it is always included
> first
> in the classpath.
>
> EOF
> }
>
> run_flume() {
> local FLUME_APPLICATION_CLASS
>
> if [ "$#" -gt 0 ]; then
> FLUME_APPLICATION_CLASS=$1
> shift
> else
> error "Must specify flume application class" 1
> fi
>
> if [ ${CLEAN_FLAG} -ne 0 ]; then
> set -x
> fi
> $EXEC $JAVA_HOME/bin/java $JAVA_OPTS -cp "$FLUME_CLASSPATH" \
> -Djava.library.path=$FLUME_JAVA_LIBRARY_PATH
> "$FLUME_APPLICATION_CLASS" $*
> }
>
> ################################
> # main
> ################################
>
> # set default params
> FLUME_CLASSPATH=""
> FLUME_JAVA_LIBRARY_PATH=""
> JAVA_OPTS="-Xmx20m"
>
> opt_conf=""
> opt_classpath=""
> opt_java_props=""
> opt_dryrun=""
>
> mode=$1
> shift
>
> case "$mode" in
> help)
> display_help
> exit 0
> ;;
> agent)
> opt_agent=1
> ;;
> node)
> opt_agent=1
> warn "The \"node\" command is deprecated. Please use \"agent\"
> instead."
> ;;
> avro-client)
> opt_avro_client=1
> ;;
> version)
> opt_version=1
> CLEAN_FLAG=0
> ;;
> *)
> error "Unknown or unspecified command '$mode'"
> echo
> display_help
> exit 1
> ;;
> esac
>
> while [ -n "$*" ] ; do
> arg=$1
> shift
>
> case "$arg" in
> --conf|-c)
> [ -n "$1" ] || error "Option --conf requires an argument" 1
> opt_conf=$1
> shift
> ;;
> --classpath|-C)
> [ -n "$1" ] || error "Option --classpath requires an argument" 1
> opt_classpath=$1
> shift
> ;;
> --dryrun|-d)
> opt_dryrun="1"
> ;;
> -D*)
> opt_java_props="$opt_java_props $arg"
> ;;
> *)
> args="$args $arg"
> ;;
> esac
> done
>
> # make opt_conf absolute
> if [[ -n "$opt_conf" && -d "$opt_conf" ]]; then
> opt_conf=$(cd $opt_conf; pwd)
> fi
>
> # allow users to override the default env vars via conf/flume-env.sh
> if [ -z "$opt_conf" ]; then
> warn "No configuration directory set! Use --conf <dir> to override."
> elif [ -f "$opt_conf/flume-env.sh" ]; then
> info "Sourcing environment configuration script $opt_conf/flume-env.sh"
> source "$opt_conf/flume-env.sh"
> fi
>
> # append command-line java options to stock or env script JAVA_OPTS
> if [ -n "${opt_java_props}" ]; then
> JAVA_OPTS="${JAVA_OPTS} ${opt_java_props}"
> fi
>
> # prepend command-line classpath to env script classpath
> if [ -n "${opt_classpath}" ]; then
> if [ -n "${FLUME_CLASSPATH}" ]; then
> FLUME_CLASSPATH="${opt_classpath}:${FLUME_CLASSPATH}"
> else
> FLUME_CLASSPATH="${opt_classpath}"
> fi
> fi
>
> if [ -z "${FLUME_HOME}" ]; then
> FLUME_HOME=$(cd $(dirname $0)/..; pwd)
> fi
>
> # prepend $FLUME_HOME/lib jars to the specified classpath (if any)
> if [ -n "${FLUME_CLASSPATH}" ] ; then
> FLUME_CLASSPATH="${FLUME_HOME}/lib/*:$FLUME_CLASSPATH"
> else
> FLUME_CLASSPATH="${FLUME_HOME}/lib/*"
> fi
>
> # find java
> if [ -z "${JAVA_HOME}" ] ; then
> warn "JAVA_HOME is not set!"
> # Try to use Bigtop to autodetect JAVA_HOME if it's available
> if [ -e /usr/libexec/bigtop-detect-javahome ] ; then
> . /usr/libexec/bigtop-detect-javahome
> elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ] ; then
> . /usr/lib/bigtop-utils/bigtop-detect-javahome
> fi
>
> # Using java from path if bigtop is not installed or couldn't find it
> if [ -z "${JAVA_HOME}" ] ; then
> JAVA_DEFAULT=$(type -p java)
> [ -n "$JAVA_DEFAULT" ] || error "Unable to find java executable. Is it
> in your PATH?" 1
> JAVA_HOME=$(cd $(dirname $JAVA_DEFAULT)/..; pwd)
> fi
> fi
>
> # look for hadoop libs
> add_hadoop_paths
> add_HBASE_paths
>
> # prepend conf dir to classpath
> if [ -n "$opt_conf" ]; then
> FLUME_CLASSPATH="$opt_conf:$FLUME_CLASSPATH"
> fi
>
> # allow dryrun
> EXEC="exec"
> if [ -n "${opt_dryrun}" ]; then
> warn "Dryrun mode enabled (will not actually initiate startup)"
> EXEC="echo"
> fi
>
> # finally, invoke the appropriate command
> if [ -n "$opt_agent" ] ; then
> run_flume $FLUME_AGENT_CLASS $args
> elif [ -n "$opt_avro_client" ] ; then
> run_flume $FLUME_AVRO_CLIENT_CLASS $args
> elif [ -n "${opt_version}" ] ; then
> run_flume $FLUME_VERSION_CLASS $args
> else
> error "This message should never appear" 1
> fi
>
> exit 0
>
>
>
>
>
>
>
>
> On Mon, Jul 30, 2012 at 2:40 PM, Inder Pall <inder.p...@gmail.com> wrote:
>
>> Hey if you edit flume-ng script and just before java is invoked put an
>>
>> echo $CLASSPATH that should help in figuring what all dir's are there in
>> CLASSPATH.
>> I am sorry but i am a big fan of windows hence don't run that O.S. to try
>> it out.
>>
>> Thanks,
>> - inder
>>
>> On Mon, Jul 30, 2012 at 7:00 PM, mardan Khan <mardan8...@gmail.com>wrote:
>>
>>> HI Inder,
>>>
>>> Sorry for the previous email. Actually I past the snapshot directly into
>>> editor which convert into some noisy text. Please see the snapshot as
>>> attachment .
>>>
>>> Please let me know for any solution
>>>
>>> Thanks
>>>
>>>
>>>
>>>
>>> On Mon, Jul 30, 2012 at 1:20 PM, Inder Pall <inder.p...@gmail.com>wrote:
>>>
>>>> can you echo your CLASSPATH in the flume startup script.
>>>>
>>>> - inder
>>>>
>>>>
>>>> On Mon, Jul 30, 2012 at 4:25 PM, mardan Khan <mardan8...@gmail.com>wrote:
>>>>
>>>>> Hi Brock
>>>>>
>>>>> Thanks for reply. I have solved the problem of cygwin. Now i am
>>>>> getting another error message: Could not find or load main class of
>>>>> org.apache.flume.node.Application.
>>>>>
>>>>> I am running the following command:
>>>>>
>>>>> bin/flume-ng agent -n agent -c conf -f
>>>>> conf/flume-conf.properties.template
>>>>>
>>>>> Thanks
>>>>>
>>>>>
>>>>>
>>>>> On Sun, Jul 29, 2012 at 6:03 PM, Brock Noland <br...@cloudera.com>wrote:
>>>>>
>>>>>> Hi,
>>>>>>
>>>>>> Unfortunately I am not familiar with cygwin and don't have a windows
>>>>>> computer to test on. Hopefully someone else on the list has an insight.
>>>>>>
>>>>>> Brock
>>>>>>
>>>>>>
>>>>>> On Sun, Jul 29, 2012 at 11:27 AM, mardan Khan
>>>>>> <mardan8...@gmail.com>wrote:
>>>>>>
>>>>>>> Thanks Brock.
>>>>>>>
>>>>>>>
>>>>>>> Now when i am run the command: flume$ bin/flume-ng agent -n agent
>>>>>>> -f conf/flume-conf.properties.
>>>>>>> template.
>>>>>>>
>>>>>>>
>>>>>>> Cygwin Warning: MS-DOS style path detected
>>>>>>> /usr/local/bin/c:\program......................................................................
>>>>>>> bin/flume-ng line:210 exec c:\program not found.
>>>>>>>
>>>>>>> How to solve this problem.
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>> On Sun, Jul 29, 2012 at 4:41 PM, Brock Noland <br...@cloudera.com>wrote:
>>>>>>>
>>>>>>>> Hi,
>>>>>>>>
>>>>>>>> Inline
>>>>>>>>
>>>>>>>> On Sat, Jul 28, 2012 at 11:54 PM, mardan Khan <mardan8...@gmail.com
>>>>>>>> > wrote:
>>>>>>>>
>>>>>>>>> Hi Brock.
>>>>>>>>>
>>>>>>>>> Thanks for reply. I want to use the full flume as i have to upload
>>>>>>>>> the data into hdfs. I have installed the cygwin and hadoop.
>>>>>>>>>
>>>>>>>>
>>>>>>>> OK this is a great start.
>>>>>>>>
>>>>>>>>
>>>>>>>>> I have download the flume binary from the mentioned link. Now you
>>>>>>>>> please let me explain the follow:
>>>>>>>>>
>>>>>>>>> 1). I need just to decopress the downloaded folder?
>>>>>>>>>
>>>>>>>>
>>>>>>>> Yes you download a tar.gz which is like a zip. In cygwin you can
>>>>>>>> use the command tar -zxvf to untar it.
>>>>>>>>
>>>>>>>>
>>>>>>>>> 2). where i should decompress the flume binary. Should i
>>>>>>>>> decompress c:\cygwin\home\user-name
>>>>>>>>>
>>>>>>>>
>>>>>>>> Whereever you want, however a common place for "tarball" software
>>>>>>>> is /usr/local or /opt
>>>>>>>>
>>>>>>>>
>>>>>>>>> 3). How I will run the flume-ng.
>>>>>>>>>
>>>>>>>>
>>>>>>>> You configure flume and then run the flume-ng command in the bin
>>>>>>>> directory
>>>>>>>>
>>>>>>>>
>>>>>>>>> 4) The flume.conf file will configure same as configuring using
>>>>>>>>> Linux?
>>>>>>>>>
>>>>>>>>>
>>>>>>>> Yes configure flume will be the same as on Linux in cygwin.
>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> Many thanks
>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>> On Sun, Jul 29, 2012 at 4:20 AM, Brock Noland
>>>>>>>>> <br...@cloudera.com>wrote:
>>>>>>>>>
>>>>>>>>>> Hi,
>>>>>>>>>>
>>>>>>>>>> If you just want run a flume agent on Windows you just download
>>>>>>>>>> the binary here: http://flume.apache.org/download.html
>>>>>>>>>>
>>>>>>>>>> Note that the flume startup script is written in bash so unless
>>>>>>>>>> you use cygwin you'll need to start the JVM and setup the classpath
>>>>>>>>>> yourself.
>>>>>>>>>>
>>>>>>>>>> Brock
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> On Sat, Jul 28, 2012 at 8:01 PM, mardan Khan <
>>>>>>>>>> mardan8...@gmail.com> wrote:
>>>>>>>>>>
>>>>>>>>>>> Hi,
>>>>>>>>>>>
>>>>>>>>>>> I am struggling from couple of weeks to install and configure
>>>>>>>>>>> the flume1.x on windows but could not success. Could you please
>>>>>>>>>>> someone
>>>>>>>>>>> show me step-by-step how to install flume on window. I have read a
>>>>>>>>>>> couple
>>>>>>>>>>> of posts which mentioned the git and Maven but i dont understand
>>>>>>>>>>> how can
>>>>>>>>>>> use it to install the flume.
>>>>>>>>>>>
>>>>>>>>>>> Please help me as i am total stuck and running out of time.
>>>>>>>>>>>
>>>>>>>>>>> Thanks
>>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> --
>>>>>>>>>> Apache MRUnit - Unit testing MapReduce -
>>>>>>>>>> http://incubator.apache.org/mrunit/
>>>>>>>>>>
>>>>>>>>>
>>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> --
>>>>>>>> Apache MRUnit - Unit testing MapReduce -
>>>>>>>> http://incubator.apache.org/mrunit/
>>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>
>>>>>>
>>>>>> --
>>>>>> Apache MRUnit - Unit testing MapReduce -
>>>>>> http://incubator.apache.org/mrunit/
>>>>>>
>>>>>
>>>>>
>>>>
>>>>
>>>> --
>>>> Thanks,
>>>> - Inder
>>>> Tech Platforms @Inmobi
>>>> Linkedin - http://goo.gl/eR4Ub
>>>>
>>>
>>>
>>
>>
>> --
>> Thanks,
>> - Inder
>> Tech Platforms @Inmobi
>> Linkedin - http://goo.gl/eR4Ub
>>
>
>
--
Thanks,
- Inder
Tech Platforms @Inmobi
Linkedin - http://goo.gl/eR4Ub
