Author: hashutosh Date: Thu Feb 4 21:29:16 2010 New Revision: 906657 URL: http://svn.apache.org/viewvc?rev=906657&view=rev Log: PIG-1209: Port POJoinPackage to proactively spill
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=906657&r1=906656&r2=906657&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Thu Feb 4 21:29:16 2010 @@ -24,6 +24,8 @@ IMPROVEMENTS +PIG-1209: Port POJoinPackage to proactively spill (ashutoshc) + PIG-1190: Handling of quoted strings in pig-latin/grunt commands (ashutoshc) PIG-1214: Pig 0.6 Docs fixes (chandec via olgan) Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java?rev=906657&r1=906656&r2=906657&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java Thu Feb 4 21:29:16 2010 @@ -20,12 +20,14 @@ import java.util.List; import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan; import org.apache.pig.data.DataBag; import org.apache.pig.data.DataType; +import org.apache.pig.data.InternalCachedBag; import org.apache.pig.data.Tuple; import org.apache.pig.impl.io.NullableTuple; import org.apache.pig.impl.plan.NodeIdGenerator; @@ -124,9 +126,17 @@ { lastInputTuple = false; //Put n-1 inputs into bags + String bagType = null; + if (PigMapReduce.sJobConf != null) { + bagType = PigMapReduce.sJobConf.get("pig.cachedbag.type"); + } dbs = new DataBag[numInputs]; for (int i = 0; i < numInputs; i++) { - dbs[i] = mBagFactory.newDefaultBag(); + if (bagType != null && bagType.equalsIgnoreCase("default")) { + dbs[i] = mBagFactory.newDefaultBag(); + } else { + dbs[i] = new InternalCachedBag(numInputs); + } } //For each Nullable tuple in the input, put it