Author: knoguchi
Date: Fri May 17 16:59:25 2024
New Revision: 1917791
URL: http://svn.apache.org/viewvc?rev=1917791&view=rev
Log:
Revert PIG-5453 which caused other test failures (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
pig/trunk/test/org/apache/pig/test/TestFlatten.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1917791&r1=1917790&r2=1917791&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 17 16:59:25 2024
@@ -42,8 +42,6 @@ PIG-5450: Pig-on-Spark3 E2E ORC test fai
PIG-5452: Null handling of FLATTEN with user defined schema (as clause)
(knoguchi)
-PIG-5453: FLATTEN shifting fields incorrectly (knoguchi)
-
Release 0.18.0 - Unreleased
INCOMPATIBLE CHANGES
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java?rev=1917791&r1=1917790&r2=1917791&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
Fri May 17 16:59:25 2024
@@ -549,8 +549,7 @@ public class POForEach extends PhysicalO
}
} else if(in instanceof Tuple) {
Tuple t = (Tuple)in;
- int size = flattenNumFields == null || flattenNumFields[i] ==
0 ?
- t.size() : Math.min(flattenNumFields[i], t.size());
+ int size = t.size();
for(int j = 0; j < size; ++j) {
if (knownSize) {
out.set(idx++, t.get(j));
@@ -558,14 +557,6 @@ public class POForEach extends PhysicalO
out.append(t.get(j));
}
}
- // if size >= flattenNumFields[i], then fill the rest with null
- for(int j = size; j < flattenNumFields[i]; ++j) {
- if (knownSize) {
- out.set(idx++, null);
- } else {
- out.append(null);
- }
- }
} else if (in instanceof Map.Entry) {
Map.Entry entry = (Map.Entry)in;
if (knownSize) {
@@ -735,12 +726,7 @@ public class POForEach extends PhysicalO
clone.addOriginalLocation(alias, getOriginalLocations());
clone.endOfAllInputProcessing = endOfAllInputProcessing;
clone.mapSideOnly = mapSideOnly;
- if( flattenNumFields != null ) {
- clone.flattenNumFields = new int[flattenNumFields.length];
- for(int i=0; i < flattenNumFields.length; i++) {
- clone.flattenNumFields[i] = flattenNumFields[i];
- }
- }
+ clone.flattenNumFields = flattenNumFields;
return clone;
}
Modified: pig/trunk/test/org/apache/pig/test/TestFlatten.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestFlatten.java?rev=1917791&r1=1917790&r2=1917791&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestFlatten.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestFlatten.java Fri May 17 16:59:25 2024
@@ -386,43 +386,4 @@ public class TestFlatten {
Util.checkQueryOutputs(actualResults.iterator(), expectedResults);
}
-
- @Test
- public void testFlattenWithInconsistentNumberOfFields() throws Exception {
- Storage.Data data = Storage.resetData(pig);
- data.set("input",
- Storage.tuple(
- 1,
- Storage.tuple("a","b","c")
- ),
- Storage.tuple(
- 2,
- Storage.tuple("a","b","c")
- ),
- Storage.tuple(
- 3,
- Storage.tuple("a","b")
- ),
- Storage.tuple(
- 4,
- Storage.tuple("a","b","c","d")
- )
- );
- pig.setBatchOn();
- pig.registerQuery("A = load 'input' using mock.Storage() as (a0:int,
a1:tuple());");
- pig.registerQuery("B = foreach A GENERATE a0, " +
- "FLATTEN(a1) as b1:chararray, b2:chararray,
b3:chararray, a0 as a4");
- pig.registerQuery("store B into 'output' using mock.Storage();");
- List<ExecJob> execJobs = pig.executeBatch();
- for( ExecJob execJob : execJobs ) {
- assertTrue(execJob.getStatus() == ExecJob.JOB_STATUS.COMPLETED );
- }
- List<Tuple> actualResults = data.get("output");
- List<Tuple> expectedResults = Util.getTuplesFromConstantTupleStrings(
- new String[] {
- "(1, 'a', 'b', 'c', 1)", "(2, 'a', 'b', 'c', 2)",
- "(3, 'a', 'b', null, 3)", "(4, 'a', 'b', 'c', 4)" });
-
- Util.checkQueryOutputs(actualResults.iterator(), expectedResults);
- }
}
