svn commit: r1911037 - in /pig/branches/branch-0.18: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java

[rohini]

Author: rohini
Date: Sat Jul 15 20:15:34 2023
New Revision: 1911037

URL: http://svn.apache.org/viewvc?rev=1911037&view=rev
Log:
PIG-5441: Pig skew join tez grace reducer fails to find shuffle data (yigress 
via rohini)

Modified:
    pig/branches/branch-0.18/CHANGES.txt
    
pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java

Modified: pig/branches/branch-0.18/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/pig/branches/branch-0.18/CHANGES.txt?rev=1911037&r1=1911036&r2=1911037&view=diff
==============================================================================
--- pig/branches/branch-0.18/CHANGES.txt (original)
+++ pig/branches/branch-0.18/CHANGES.txt Sat Jul 15 20:15:34 2023
@@ -122,6 +122,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-5441: Pig skew join tez grace reducer fails to find shuffle data (yigress 
via rohini)
+
 PIG-5432: OrcStorage fails to detect schema in some cases (jtolar via rohini)
 
 PIG-5439: pig.exec.reducers.max does not take effect for skewed join 
(vnarayanan7 via rohini)

Modified: 
pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java
URL: 
http://svn.apache.org/viewvc/pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java?rev=1911037&r1=1911036&r2=1911037&view=diff
==============================================================================
--- 
pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java
 (original)
+++ 
pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java
 Sat Jul 15 20:15:34 2023
@@ -165,7 +165,8 @@ public class PigGraceShuffleVertexManage
             Map<String, EdgeProperty> edgeManagers = new HashMap<String, 
EdgeProperty>();
             for(Map.Entry<String,EdgeProperty> entry : 
getContext().getInputVertexEdgeProperties().entrySet()) {
                 EdgeProperty edge = entry.getValue();
-                edge = EdgeProperty.create(DataMovementType.SCATTER_GATHER, 
edge.getDataSourceType(), edge.getSchedulingType(),
+                DataMovementType movementType = edge.getDataMovementType() == 
DataMovementType.BROADCAST? DataMovementType.BROADCAST: 
DataMovementType.SCATTER_GATHER;
+                edge = EdgeProperty.create(movementType, 
edge.getDataSourceType(), edge.getSchedulingType(),
                         edge.getEdgeSource(), edge.getEdgeDestination());
                 edgeManagers.put(entry.getKey(), edge);
             }