Re: Spark job tracker.

[abhiguruvayya]

I know this is a very trivial question to ask but I'm a complete new bee to
this stuff so i don't have ne clue on this. Any help is much appreciated. 

For example if i have a class like below, and when i run this through
command line i want to see progress status. some thing like,

10% completed...
30% completed...
100% completed...Job done!

I am using spark 1.0 on yarn and using Java API.

public class MyJavaWordCount {
  public static void main(String[] args) throws Exception {
    if (args.length < 2) {
      System.err.println("Usage: MyJavaWordCount <master> <file>");
      System.exit(1);
    }

    System.out.println("args[0]: <master>="+args[0]);
    System.out.println("args[1]: <file>="+args[1]);

    JavaSparkContext ctx = new JavaSparkContext(
                                        args[0], 
                                        "MyJavaWordCount",
                                        System.getenv("SPARK_HOME"), 
                                        System.getenv("SPARK_EXAMPLES_JAR"));
    JavaRDD<String> lines = ctx.textFile(args[1], 1);

    //      output                                            input   output    
    
    JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String,
String>() {
      //              output       input
      public Iterable<String> call(String s) {
        return Arrays.asList(s.split(" "));
      }
    });
    
    //          K       V                                               
input   K       V
    JavaPairRDD<String, Integer> ones = words.mapToPair(new
PairFunction<String, String, Integer>() {
      //            K       V             input
      public Tuple2<String, Integer> call(String s) {
        //                K       V
        return new Tuple2<String, Integer>(s, 1);
      }
    });
    
    JavaPairRDD<String, Integer> counts = ones.reduceByKey(new
Function2<Integer, Integer, Integer>() {
      public Integer call(Integer i1, Integer i2) {
        return i1 + i2;
      }
    });

    List<Tuple2&lt;String, Integer>> output = counts.collect();
    for (Tuple2 tuple : output) {
      System.out.println(tuple._1 + ": " + tuple._2);
    }
    System.exit(0);
  }
}



--
View this message in context: 
http://apache-spark-user-list.1001560.n3.nabble.com/Spark-job-tracker-tp8367p8472.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.