Author: rohini Date: Sat Jul 15 20:15:34 2023 New Revision: 1911037 URL: http://svn.apache.org/viewvc?rev=1911037&view=rev Log: PIG-5441: Pig skew join tez grace reducer fails to find shuffle data (yigress via rohini)
Modified: pig/branches/branch-0.18/CHANGES.txt pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java Modified: pig/branches/branch-0.18/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/branches/branch-0.18/CHANGES.txt?rev=1911037&r1=1911036&r2=1911037&view=diff ============================================================================== --- pig/branches/branch-0.18/CHANGES.txt (original) +++ pig/branches/branch-0.18/CHANGES.txt Sat Jul 15 20:15:34 2023 @@ -122,6 +122,8 @@ OPTIMIZATIONS BUG FIXES +PIG-5441: Pig skew join tez grace reducer fails to find shuffle data (yigress via rohini) + PIG-5432: OrcStorage fails to detect schema in some cases (jtolar via rohini) PIG-5439: pig.exec.reducers.max does not take effect for skewed join (vnarayanan7 via rohini) Modified: pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java URL: http://svn.apache.org/viewvc/pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java?rev=1911037&r1=1911036&r2=1911037&view=diff ============================================================================== --- pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java (original) +++ pig/branches/branch-0.18/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigGraceShuffleVertexManager.java Sat Jul 15 20:15:34 2023 @@ -165,7 +165,8 @@ public class PigGraceShuffleVertexManage Map<String, EdgeProperty> edgeManagers = new HashMap<String, EdgeProperty>(); for(Map.Entry<String,EdgeProperty> entry : getContext().getInputVertexEdgeProperties().entrySet()) { EdgeProperty edge = entry.getValue(); - edge = EdgeProperty.create(DataMovementType.SCATTER_GATHER, edge.getDataSourceType(), edge.getSchedulingType(), + DataMovementType movementType = edge.getDataMovementType() == DataMovementType.BROADCAST? DataMovementType.BROADCAST: DataMovementType.SCATTER_GATHER; + edge = EdgeProperty.create(movementType, edge.getDataSourceType(), edge.getSchedulingType(), edge.getEdgeSource(), edge.getEdgeDestination()); edgeManagers.put(entry.getKey(), edge); }