I have disabled it because of it started generating ERROR's when upgrading from Spark 1.4 to 1.5.1
2015-10-27T20:50:11.574+0100 ERROR TungstenSort.newOrdering() - Failed to generate ordering, fallback to interpreted java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: Line 15, Column 9: Invalid character input "@" (character code 64) public SpecificOrdering generate(org.apache.spark.sql.catalyst.expressions.Expression[] expr) { return new SpecificOrdering(expr); } class SpecificOrdering extends org.apache.spark.sql.catalyst.expressions.codegen.BaseOrdering { private org.apache.spark.sql.catalyst.expressions.Expression[] expressions; public SpecificOrdering(org.apache.spark.sql.catalyst.expressions.Expression[] expr) { expressions = expr; } @Override public int compare(InternalRow a, InternalRow b) { InternalRow i = null; // Holds current row being evaluated. i = a; boolean isNullA2; long primitiveA3; { /* input[2, LongType] */ boolean isNull0 = i.isNullAt(2); long primitive1 = isNull0 ? -1L : (i.getLong(2)); isNullA2 = isNull0; primitiveA3 = primitive1; } i = b; boolean isNullB4; long primitiveB5; { /* input[2, LongType] */ boolean isNull0 = i.isNullAt(2); long primitive1 = isNull0 ? -1L : (i.getLong(2)); isNullB4 = isNull0; primitiveB5 = primitive1; } if (isNullA2 && isNullB4) { // Nothing } else if (isNullA2) { return 1; } else if (isNullB4) { return -1; } else { int comp = (primitiveA3 > primitiveB5 ? 1 : primitiveA3 < primitiveB5 ? -1 : 0); if (comp != 0) { return -comp; } } return 0; } } at org.spark-project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306) at org.spark-project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293) at org.spark-project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) at org.spark-project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135) at org.spark-project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410) at org.spark-project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380) at org.spark-project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) at org.spark-project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257) at org.spark-project.guava.cache.LocalCache.get(LocalCache.java:4000) at org.spark-project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004) at org.spark-project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.compile(CodeGenerator.scala:362) at org.apache.spark.sql.catalyst.expressions.codegen.GenerateOrdering$.create(GenerateOrdering.scala:139) at org.apache.spark.sql.catalyst.expressions.codegen.GenerateOrdering$.create(GenerateOrdering.scala:37) at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:425) at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:422) at org.apache.spark.sql.execution.SparkPlan.newOrdering(SparkPlan.scala:294) at org.apache.spark.sql.execution.TungstenSort.org $apache$spark$sql$execution$TungstenSort$$preparePartition$1(sort.scala:131) at org.apache.spark.sql.execution.TungstenSort$$anonfun$doExecute$3.apply(sort.scala:169) at org.apache.spark.sql.execution.TungstenSort$$anonfun$doExecute$3.apply(sort.scala:169) at org.apache.spark.rdd.MapPartitionsWithPreparationRDD.compute(MapPartitionsWithPreparationRDD.scala:59) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297) at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297) at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297) at org.apache.spark.rdd.RDD.iterator(RDD.scala:264) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:88) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) 2015-10-14 21:00 GMT+02:00 Reynold Xin <r...@databricks.com>: > Can you reply to this email and provide us with reasons why you disable it? > > Thanks. > >