Author: daijy Date: Mon May 17 17:21:13 2010 New Revision: 945250 URL: http://svn.apache.org/viewvc?rev=945250&view=rev Log: PIG-1415: LoadFunc signature is not correct in LoadFunc.getSchema sometimes
Added: hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=945250&r1=945249&r2=945250&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Mon May 17 17:21:13 2010 @@ -276,6 +276,8 @@ OPTIMIZATIONS BUG FIXES +PIG-1415: LoadFunc signature is not correct in LoadFunc.getSchema sometimes (daijy) + PIG-1403: Make Pig work with remote HDFS in secure mode (daijy) PIG-1394: POCombinerPackage hold too much memory for InternalCachedBag (daijy) Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java?rev=945250&r1=945249&r2=945250&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java Mon May 17 17:21:13 2010 @@ -59,6 +59,7 @@ public class LOLoad extends RelationalOp transient private Configuration conf; private static Log log = LogFactory.getLog(LOLoad.class); private Schema mDeterminedSchema = null; + private Schema scriptSchema = null; private RequiredFieldList requiredFieldList; private boolean mDeterminedSchemaCached = false; @@ -149,6 +150,20 @@ public class LOLoad extends RelationalOp if(null == mDeterminedSchema) { mSchema = determineSchema(); } + if (mSchema == null) { + log.debug("Operator schema is null; Setting it to new schema"); + mSchema = scriptSchema; + } else { + log.debug("Reconciling schema"); + log.debug("mSchema: " + mSchema + " schema: " + scriptSchema); + try { + mSchema = mSchema.mergePrefixSchema(scriptSchema, true, true); + } catch (SchemaMergeException e) { + int errCode = 1019; + String msg = "Unable to merge schemas"; + throw new FrontendException(msg, errCode, PigException.INPUT, false, null, e); + } + } mIsSchemaComputed = true; } catch (IOException ioe) { int errCode = 1018; @@ -182,27 +197,7 @@ public class LOLoad extends RelationalOp */ @Override public void setSchema(Schema schema) throws FrontendException { - // In general, operators don't generate their schema until they're - // asked, so ask them to do it. - try { - getSchema(); - } catch (FrontendException ioe) { - // It's fine, it just means we don't have a schema yet. - } - if (mSchema == null) { - log.debug("Operator schema is null; Setting it to new schema"); - mSchema = schema; - } else { - log.debug("Reconciling schema"); - log.debug("mSchema: " + mSchema + " schema: " + schema); - try { - mSchema = mSchema.mergePrefixSchema(schema, true, true); - } catch (SchemaMergeException e) { - int errCode = 1019; - String msg = "Unable to merge schemas"; - throw new FrontendException(msg, errCode, PigException.INPUT, false, null, e); - } - } + scriptSchema = schema; } Added: hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java?rev=945250&view=auto ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java (added) +++ hadoop/pig/trunk/test/org/apache/pig/test/PigStorageWithSchema.java Mon May 17 17:21:13 2010 @@ -0,0 +1,46 @@ +package org.apache.pig.test; + +import java.io.IOException; + +import org.apache.hadoop.mapreduce.Job; +import org.apache.pig.Expression; +import org.apache.pig.LoadMetadata; +import org.apache.pig.ResourceSchema; +import org.apache.pig.ResourceStatistics; +import org.apache.pig.builtin.PigStorage; + +public class PigStorageWithSchema extends PigStorage implements LoadMetadata { + private String signature; + + @Override + public String[] getPartitionKeys(String location, Job job) + throws IOException { + return null; + } + + @Override + public ResourceSchema getSchema(String location, Job job) + throws IOException { + return null; + } + + @Override + public ResourceStatistics getStatistics(String location, Job job) + throws IOException { + return null; + } + + @Override + public void setPartitionFilter(Expression partitionFilter) + throws IOException { + } + + @Override + public void setUDFContextSignature(String signature) { + this.signature = signature; + } + + public String getUDFContextSignature() { + return signature; + } +} Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=945250&r1=945249&r2=945250&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java Mon May 17 17:21:13 2010 @@ -2145,6 +2145,14 @@ public class TestLogicalPlanBuilder exte "not occur", true, exceptionThrown); } + @Test + public void testLoaderSignature() { + LogicalPlan plan = buildPlan(" a = load '1.txt' using org.apache.pig.test.PigStorageWithSchema() as (a0:int, a1:int);"); + assertTrue(((PigStorageWithSchema)((LOLoad)plan.getLeaves().get(0)).getLoadFunc()).getUDFContextSignature().equals("a")); + plan = buildPlan(" b = load '1.txt' using org.apache.pig.test.PigStorageWithSchema();"); + assertTrue(((PigStorageWithSchema)((LOLoad)plan.getLeaves().get(0)).getLoadFunc()).getUDFContextSignature().equals("b")); + } + private void printPlan(LogicalPlan lp) { LOPrinter graphPrinter = new LOPrinter(System.err, lp); System.err.println("Printing the logical plan");