Author: daijy
Date: Tue Feb 24 22:59:45 2015
New Revision: 1662135
URL: http://svn.apache.org/r1662135
Log:
PIG-4433: Loading bigdecimal in nested tuple does not work (rollback the code
which belongs to PIG-4426)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/test/org/apache/pig/test/TestRank1.java
pig/trunk/test/org/apache/pig/test/TestRank2.java
pig/trunk/test/org/apache/pig/test/TestRank3.java
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1662135&r1=1662134&r2=1662135&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Tue Feb 24 22:59:45 2015
@@ -360,7 +360,7 @@ public class JobControlCompiler{
{
MapReduceOper mro = jobMroMap.get(job);
if (!pigContext.inIllustrator && mro.isCounterOperation())
- saveCounters(job,mro.getOperationID(), mro.isRowNumber());
+ saveCounters(job,mro.getOperationID());
plan.remove(mro);
}
}
@@ -378,7 +378,7 @@ public class JobControlCompiler{
* these values are passed via configuration file to PORank, by using the
unique
* operation identifier
*/
- private void saveCounters(Job job, String operationID, boolean isRowNumber
) {
+ private void saveCounters(Job job, String operationID) {
Counters counters;
Group groupCounters;
@@ -409,8 +409,7 @@ public class JobControlCompiler{
HashMap<Integer,Long> counterList = new HashMap<Integer, Long>();
- int numTasks = isRowNumber ? job.getJobConf().getNumMapTasks() :
job.getJobConf().getNumReduceTasks();
- for ( int i=0; i < numTasks; i++ ) {
+ for (int i=0;i<job.getJob().getNumReduceTasks();i++) {
Long value = groupCounters.getCounter(Integer.toString(i));
counterList.put(i, value);
}
Modified: pig/trunk/test/org/apache/pig/test/TestRank1.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRank1.java?rev=1662135&r1=1662134&r2=1662135&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRank1.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestRank1.java Tue Feb 24 22:59:45 2015
@@ -33,10 +33,7 @@ import org.junit.Before;
import org.junit.Test;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMultiset;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.TreeMultiset;
-import com.google.common.collect.Multiset;
+import com.google.common.collect.ImmutableSet;
public class TestRank1 {
private static TupleFactory tf = TupleFactory.getInstance();
@@ -79,7 +76,7 @@ public class TestRank1 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "A", 1, "N")),
tf.newTuple(ImmutableList.of((long) 2, "B", 2, "N")),
tf.newTuple(ImmutableList.of((long) 3, "C", 3, "M")),
@@ -103,7 +100,7 @@ public class TestRank1 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "Michael", "Blythe",
1,1, 1, 1, 4557045.046, 98027)),
tf.newTuple(ImmutableList.of((long) 2, "Linda","Mitchell", 2,
1, 1, 1, 5200475.231, 98027)),
tf.newTuple(ImmutableList.of((long) 3, "Jillian", "Carson",
3,1, 1, 1, 3857163.633, 98027)),
@@ -129,7 +126,7 @@ public class TestRank1 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "C", 3, "M")),
tf.newTuple(ImmutableList.of((long) 2, "A", 1, "N")),
tf.newTuple(ImmutableList.of((long) 2, "B", 2, "N")),
@@ -153,7 +150,7 @@ public class TestRank1 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "A", 1, "N")),
tf.newTuple(ImmutableList.of((long) 2, "B", 2, "N")),
tf.newTuple(ImmutableList.of((long) 3, "C", 3, "M")),
@@ -177,7 +174,7 @@ public class TestRank1 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "G", 10, "V")),
tf.newTuple(ImmutableList.of((long) 2, "F", 8, "T")),
tf.newTuple(ImmutableList.of((long) 2, "F", 8, "Q")),
@@ -201,7 +198,7 @@ public class TestRank1 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "Michael", "Blythe",
1,1, 1, 1, 4557045.046, 98027)),
tf.newTuple(ImmutableList.of((long) 1, "Linda","Mitchell", 2,
1, 1, 1, 5200475.231, 98027)),
tf.newTuple(ImmutableList.of((long) 1, "Jillian", "Carson",
3,1, 1, 1, 3857163.633, 98027)),
@@ -227,7 +224,7 @@ public class TestRank1 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "David", "Campbell",
8,6, 2, 3, 3587378.426, 98055)),
tf.newTuple(ImmutableList.of((long) 2, "Garrett","Vargas", 4,
1, 1, 1, 1764938.986, 98027)),
tf.newTuple(ImmutableList.of((long) 3, "Jae", "Pak", 12,6, 2,
4, 5015682.375, 98055)),
@@ -253,7 +250,7 @@ public class TestRank1 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "David", "Campbell", 8,
6, 2, 3, 3587378.426, 98055)),
tf.newTuple(ImmutableList.of((long) 2, "Garrett","Vargas", 4,
1, 1, 1, 1764938.986, 98027)),
tf.newTuple(ImmutableList.of((long) 3, "Jae", "Pak", 12,6, 2,
4, 5015682.375, 98055)),
@@ -279,7 +276,7 @@ public class TestRank1 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "A", 1, "N")),
tf.newTuple(ImmutableList.of((long) 2, "B", 2, "N")),
tf.newTuple(ImmutableList.of((long) 3, "C", 3, "M")),
@@ -295,21 +292,10 @@ public class TestRank1 {
verifyExpected(data.get("result"), expected);
}
- public void verifyExpected(List<Tuple> out, Multiset<Tuple> expected) {
- Multiset<Tuple> resultMultiset = TreeMultiset.create();
+ public void verifyExpected(List<Tuple> out, Set<Tuple> expected) {
for (Tuple tup : out) {
- resultMultiset.add(tup);
+ assertTrue(expected + " contains " + tup, expected.contains(tup));
}
-
- StringBuilder error = new StringBuilder("Result does not
match.\nActual result:\n");
- for (Tuple tup : resultMultiset.elementSet() ) {
- error.append(tup).append(" x
").append(resultMultiset.count(tup)).append("\n");
- }
- error.append("Expceted result:\n");
- for (Tuple tup : ImmutableSortedSet.copyOf(expected) ) {
- error.append(tup).append(" x
").append(expected.count(tup)).append("\n");
- }
-
- assertTrue(error.toString(), resultMultiset.equals(expected));
}
+
}
Modified: pig/trunk/test/org/apache/pig/test/TestRank2.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRank2.java?rev=1662135&r1=1662134&r2=1662135&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRank2.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestRank2.java Tue Feb 24 22:59:45 2015
@@ -33,10 +33,7 @@ import org.junit.Before;
import org.junit.Test;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMultiset;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.TreeMultiset;
-import com.google.common.collect.Multiset;
+import com.google.common.collect.ImmutableSet;
public class TestRank2 {
private static PigServer pigServer;
@@ -79,7 +76,7 @@ public class TestRank2 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "C", 3, "M")),
tf.newTuple(ImmutableList.of((long) 2, "A", 1, "N")),
tf.newTuple(ImmutableList.of((long) 2, "B", 2, "N")),
@@ -103,7 +100,7 @@ public class TestRank2 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "A", 1, "N")),
tf.newTuple(ImmutableList.of((long) 2, "B", 2, "N")),
tf.newTuple(ImmutableList.of((long) 3, "C", 3, "M")),
@@ -127,7 +124,7 @@ public class TestRank2 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "G", 10, "V")),
tf.newTuple(ImmutableList.of((long) 2, "F", 8, "T")),
tf.newTuple(ImmutableList.of((long) 2, "F", 8, "Q")),
@@ -151,7 +148,7 @@ public class TestRank2 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of((long) 1, "A", 1, "N")),
tf.newTuple(ImmutableList.of((long) 2, "B", 2, "N")),
tf.newTuple(ImmutableList.of((long) 3, "C", 3, "M")),
@@ -167,23 +164,9 @@ public class TestRank2 {
verifyExpected(data.get("result"), expected);
}
- public void verifyExpected(List<Tuple> out, Multiset<Tuple> expected) {
- Multiset<Tuple> resultMultiset = TreeMultiset.create();
+ public void verifyExpected(List<Tuple> out, Set<Tuple> expected) {
for (Tuple tup : out) {
- resultMultiset.add(tup);
+ assertTrue(expected + " contains " + tup, expected.contains(tup));
}
-
- StringBuilder error = new StringBuilder("Result does not
match.\nActual result:\n");
- for (Tuple tup : resultMultiset.elementSet() ) {
- error.append(tup).append(" x
").append(resultMultiset.count(tup)).append("\n");
- }
- error.append("Expceted result:\n");
- for (Tuple tup : ImmutableSortedSet.copyOf(expected) ) {
- error.append(tup).append(" x
").append(expected.count(tup)).append("\n");
- }
-
- //This one line test should be sufficient but adding the above
- //for-loop for better error messages
- assertTrue(error.toString(), resultMultiset.equals(expected));
}
-}
+}
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/TestRank3.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRank3.java?rev=1662135&r1=1662134&r2=1662135&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRank3.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestRank3.java Tue Feb 24 22:59:45 2015
@@ -35,10 +35,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMultiset;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.TreeMultiset;
-import com.google.common.collect.Multiset;
+import com.google.common.collect.ImmutableSet;
public class TestRank3 {
private static PigServer pigServer;
@@ -116,7 +113,7 @@ public class TestRank3 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of(
+ Set<Tuple> expected = ImmutableSet.of(
tf.newTuple(ImmutableList.of(1L,21L,5L,7L,1L,1L,0L,8L,8L)),
tf.newTuple(ImmutableList.of(2L,26L,2L,3L,2L,5L,1L,9L,10L)),
tf.newTuple(ImmutableList.of(3L,30L,24L,21L,2L,3L,1L,3L,10L)),
@@ -161,7 +158,7 @@ public class TestRank3 {
Util.registerMultiLineQuery(pigServer, query);
- Multiset<Tuple> expected = ImmutableMultiset.of();
+ Set<Tuple> expected = ImmutableSet.of();
verifyExpected(data.get("empty_result"), expected);
}
@@ -198,24 +195,10 @@ public class TestRank3 {
Util.checkQueryOutputsAfterSort(data.get("R4"), expectedResults);
}
- public void verifyExpected(List<Tuple> out, Multiset<Tuple> expected) {
- Multiset<Tuple> resultMultiset = TreeMultiset.create();
+ public void verifyExpected(List<Tuple> out, Set<Tuple> expected) {
for (Tuple tup : out) {
- resultMultiset.add(tup);
+ assertTrue(expected + " contains " + tup, expected.contains(tup));
}
-
- StringBuilder error = new StringBuilder("Result does not
match.\nActual result:\n");
- for (Tuple tup : resultMultiset.elementSet() ) {
- error.append(tup).append(" x
").append(resultMultiset.count(tup)).append("\n");
- }
- error.append("Expceted result:\n");
- for (Tuple tup : ImmutableSortedSet.copyOf(expected) ) {
- error.append(tup).append(" x
").append(expected.count(tup)).append("\n");
- }
-
- //This one line test should be sufficient but adding the above
- //for-loop for better error messages
- assertTrue(error.toString(), resultMultiset.equals(expected));
}
}
