Author: szita Date: Tue Jan 9 10:03:07 2018 New Revision: 1820631 URL: http://svn.apache.org/viewvc?rev=1820631&view=rev Log: PIG-5325: Schema disambiguation can't be turned off for nested schemas (szita)
Modified: pig/trunk/CHANGES.txt pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java pig/trunk/test/org/apache/pig/test/TestSchema.java Modified: pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1820631&r1=1820630&r2=1820631&view=diff ============================================================================== --- pig/trunk/CHANGES.txt (original) +++ pig/trunk/CHANGES.txt Tue Jan 9 10:03:07 2018 @@ -64,6 +64,8 @@ OPTIMIZATIONS BUG FIXES +PIG-5325: Schema disambiguation can't be turned off for nested schemas (szita) + PIG-5311: POReservoirSample fails for more than Integer.MAX_VALUE records (rohini) PIG-3864: ToDate(userstring, format, timezone) computes DateTime with strange handling of Daylight Saving Time with location based timezones (daijy via rohini) Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java?rev=1820631&r1=1820630&r2=1820631&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java (original) +++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java Tue Jan 9 10:03:07 2018 @@ -63,16 +63,31 @@ public class LOStore extends LogicalRela public LogicalSchema getSchema() throws FrontendException { schema = ((LogicalRelationalOperator)plan.getPredecessors(this).get(0)).getSchema(); - if (!disambiguationEnabled && schema != null && schema.getFields() != null) { + if (!disambiguationEnabled) { //If requested try and remove parent alias substring including colon(s) + removeDisambiguation(schema); + } + + return schema; + } + + /** + * Removes schema disambiguation parts (parent alias and :) from field aliases + * @param schema + * @return + */ + private static LogicalSchema removeDisambiguation(LogicalSchema schema) { + if (schema != null && schema.getFields() != null) { for (LogicalSchema.LogicalFieldSchema field : schema.getFields()) { + if (field.schema != null) { + removeDisambiguation(field.schema); + } if (field.alias == null || !field.alias.contains(":")) { continue; } field.alias = field.alias.substring(field.alias.lastIndexOf(":") + 1); } } - return schema; } Modified: pig/trunk/test/org/apache/pig/test/TestSchema.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSchema.java?rev=1820631&r1=1820630&r2=1820631&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/TestSchema.java (original) +++ pig/trunk/test/org/apache/pig/test/TestSchema.java Tue Jan 9 10:03:07 2018 @@ -990,6 +990,33 @@ public class TestSchema { } @Test + public void testDisabledDisambiguationContainsNoColonsForNestedSchema() throws IOException { + resetDisambiguationTestPropertyOverride(); + + String inputFileName = "testPrepend-nested-input.txt"; + String[] inputData = new String[]{"apple\t1\tred", "orange\t2\torange", "kiwi\t3\tgreen", "orange\t4\torange"}; + Util.createInputFile(cluster, inputFileName, inputData); + + String script = "A = LOAD '" + inputFileName + "' AS (fruit:chararray, foo:int, color: chararray);" + + "B = LOAD '" + inputFileName + "' AS (id:chararray, bar:int);" + + "C = JOIN A by fruit, B by id;" + + "D = GROUP C by fruit;" + + "E = LOAD '" + inputFileName + "' AS (name:chararray, qwe:int);" + + "F = JOIN E by name, D by group;"; + + Util.registerMultiLineQuery(pigServer, script); + + //Prepending should happen with default settings + assertEquals("{E::name: chararray,E::qwe: int,D::group: chararray,D::C: {(A::fruit: chararray,A::foo: int,A::color: chararray,B::id: chararray,B::bar: int)}}", pigServer.dumpSchema("F").toString()); + + //Override prepend property setting (check for flatten, join) + pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_STORE_SCHEMA_DISAMBIGUATE, "false"); + assertEquals("{name: chararray,qwe: int,group: chararray,C: {(fruit: chararray,foo: int,color: chararray,id:" + + " chararray,bar: int)}}", pigServer.dumpSchema("F").toString()); + assertTrue(pigServer.openIterator("F").hasNext()); + } + + @Test public void testEnabledDisambiguationPassesForDupeAliases() throws IOException { resetDisambiguationTestPropertyOverride();