svn commit: r1802675 - in /pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/spark/ src/org/apache/pig/backend/hadoop/executionengine/spark/converter/

[szita]

Author: szita
Date: Sat Jul 22 11:28:02 2017
New Revision: 1802675

URL: http://svn.apache.org/viewvc?rev=1802675&view=rev
Log:
PIG-5274: TestEvalPipelineLocal#testSetLocationCalledInFE is failing in spark 
mode after PIG-5157 (nkollar via szita)

Modified:
    pig/trunk/CHANGES.txt
    
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
    
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java
    
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java

Modified: pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1802675&r1=1802674&r2=1802675&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Jul 22 11:28:02 2017
@@ -38,6 +38,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-5274: TestEvalPipelineLocal#testSetLocationCalledInFE is failing in spark 
mode after PIG-5157 (nkollar via szita)
+
 PIG-4767: Partition filter not pushed down when filter clause references 
variable from another load path (knoguchi)
 
 PIG-5270: Typo in Pig Logging (FromAlaska49 via daijy)

Modified: 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: 
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1802675&r1=1802674&r2=1802675&view=diff
==============================================================================
--- 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
 (original)
+++ 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
 Sat Jul 22 11:28:02 2017
@@ -175,7 +175,6 @@ public class SparkLauncher extends Launc
         SparkPigStats sparkStats = (SparkPigStats) pigContext
                 .getExecutionEngine().instantiatePigStats();
         sparkStats.initialize(pigContext, sparkplan, jobConf);
-        UDFContext.getUDFContext().addJobConf(jobConf);
         PigStats.start(sparkStats);
 
         startSparkIfNeeded(jobConf, pigContext);

Modified: 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java
URL: 
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java?rev=1802675&r1=1802674&r2=1802675&view=diff
==============================================================================
--- 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java
 (original)
+++ 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java
 Sat Jul 22 11:28:02 2017
@@ -60,7 +60,12 @@ public abstract class SparkShims impleme
 
     public static SparkShims getInstance() {
         if (sparkShims == null) {
-            String sparkVersion = 
UDFContext.getUDFContext().getJobConf().get(SPARK_VERSION, "");
+            String sparkVersion;
+            if (UDFContext.getUDFContext().isFrontend()) {
+                sparkVersion = SparkContext.getOrCreate().version();
+            } else {
+                sparkVersion = 
UDFContext.getUDFContext().getJobConf().get(SPARK_VERSION, "");
+            }
             LOG.info("Initializing SparkShims for Spark version: " + 
sparkVersion);
             String sparkMajorVersion = getSparkMajorVersion(sparkVersion);
             try {

Modified: 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
URL: 
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java?rev=1802675&r1=1802674&r2=1802675&view=diff
==============================================================================
--- 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
 (original)
+++ 
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
 Sat Jul 22 11:28:02 2017
@@ -232,8 +232,8 @@ public class SkewedJoinConverter impleme
                             }
 
                             return result;
-                        } catch (Exception e) {
-                            log.warn(e);
+                        } catch (ExecException e) {
+                            log.error(e);
                             return null;
                         }
                     }