Author: dvryaboy
Date: Thu May 20 17:33:06 2010
New Revision: 946722
URL: http://svn.apache.org/viewvc?rev=946722&view=rev
Log:
PIG-1420: Make CONCAT act on all fields of a tuple, instead of just the first
two fields of a tuple (rjurney via dvryaboy)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/builtin/CONCAT.java
hadoop/pig/trunk/src/org/apache/pig/builtin/StringConcat.java
hadoop/pig/trunk/src/org/apache/pig/data/DataByteArray.java
hadoop/pig/trunk/test/org/apache/pig/test/TestBuiltin.java
hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=946722&r1=946721&r2=946722&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu May 20 17:33:06 2010
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1420: Make CONCAT act on all fields of a tuple, instead of just the first
two fields of a tuple (rjurney via dvryaboy)
+
PIG-1408: Annotate explain plans with aliases (rding)
PIG-1410: Make PigServer can handle files with parameters (zjffdu)
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/CONCAT.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/CONCAT.java?rev=946722&r1=946721&r2=946722&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/CONCAT.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/CONCAT.java Thu May 20 17:33:06
2010
@@ -40,30 +40,32 @@ public class CONCAT extends EvalFunc<Dat
@Override
public DataByteArray exec(Tuple input) throws IOException {
try {
- DataByteArray dba1 = (DataByteArray)input.get(0);
- DataByteArray dba2 = (DataByteArray)input.get(1);
- if(dba1 != null && dba2 != null) {
- return new DataByteArray(dba1, dba2);
- } else {
+ if (input == null || input.size() == 0)
return null;
+
+ DataByteArray db = new DataByteArray();
+ for (int i = 0; i < input.size(); i++) {
+ db.append((DataByteArray)(input.get(i)));
}
+ return db;
} catch (ExecException exp) {
throw exp;
} catch (Exception e) {
int errCode = 2106;
String msg = "Error while computing concat in " +
this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
+ throw new ExecException(msg, errCode, PigException.BUG, e);
}
}
@Override
public Schema outputSchema(Schema input) {
- return new Schema(new Schema.FieldSchema(null, DataType.BYTEARRAY));
+ return new Schema(new Schema.FieldSchema(null, DataType.BYTEARRAY));
}
/* (non-Javadoc)
* @see org.apache.pig.EvalFunc#getArgToFuncMapping()
*/
+
@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
List<FuncSpec> funcList = new ArrayList<FuncSpec>();
@@ -76,5 +78,5 @@ public class CONCAT extends EvalFunc<Dat
s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
funcList.add(new FuncSpec(StringConcat.class.getName(), s));
return funcList;
- }
+ }
}
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/StringConcat.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/StringConcat.java?rev=946722&r1=946721&r2=946722&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/StringConcat.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/StringConcat.java Thu May 20
17:33:06 2010
@@ -27,30 +27,27 @@ import org.apache.pig.impl.logicalLayer.
/**
- * Generates the concatenation of the first two fields of a tuple.
+ * Generates the concatenation of all fields of a tuple.
*/
public class StringConcat extends EvalFunc<String> {
@Override
public String exec(Tuple input) throws IOException {
- try {
- String s1 = (String)(input.get(0));
- String s2 = (String)(input.get(1));
- if(s1 != null && s2 != null)
- {
- StringBuilder sb = new StringBuilder();
- sb.append(s1);
- sb.append(s2);
- return sb.toString();
- } else {
- return null;
- }
+ try{
+ if (input == null || input.size() == 0)
+ return null;
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < input.size(); i++){
+ sb.append(String.valueOf(input.get(i)));
+ }
+ return sb.toString();
} catch (ExecException exp) {
throw exp;
} catch (Exception e) {
int errCode = 2106;
String msg = "Error while computing concat in " +
this.getClass().getSimpleName();
- throw new ExecException(msg, errCode, PigException.BUG, e);
+ throw new ExecException(msg, errCode, PigException.BUG, e);
}
}
Modified: hadoop/pig/trunk/src/org/apache/pig/data/DataByteArray.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DataByteArray.java?rev=946722&r1=946721&r2=946722&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/data/DataByteArray.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/data/DataByteArray.java Thu May 20
17:33:06 2010
@@ -23,6 +23,7 @@ import java.io.UnsupportedEncodingExcept
import java.lang.StringBuilder;
import java.util.ArrayList;
import java.util.Collection;
+import java.nio.MappedByteBuffer;
import org.apache.pig.classification.InterfaceAudience;
import org.apache.pig.classification.InterfaceStability;
@@ -149,6 +150,33 @@ public class DataByteArray implements Co
}
/**
+ * Append given byte array to the internal byte array.
+ * @param b byte array who's contents to append. The contents of the byte
array are
+ * copied.
+ */
+ public void append(DataByteArray b) {
+
+ byte[] ba = (b == null) ? null : b.get();
+ int mDataLength = (mData == null) ? 0 : mData.length;
+ int baLength = (ba == null) ? 0 : ba.length;
+
+ int totalSize = mDataLength + baLength;
+ if(totalSize == 0) {
+ return;
+ }
+ byte[] oldData = mData == null ? new byte[0] : mData.clone();
+ mData = new byte[totalSize];
+ int i = 0;
+ for ( ;i < mDataLength; i++) {
+ mData[i] = oldData[i];
+ }
+
+ for (int j = 0; j < baLength; j++, i++) {
+ mData[i] = ba[j];
+ }
+ }
+
+ /**
* Convert the byte array to a string. UTF8 encoding will be assumed.
*/
@Override
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=946722&r1=946721&r2=946722&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestBuiltin.java Thu May 20
17:33:06 2010
@@ -1202,6 +1202,57 @@ public class TestBuiltin {
}
@Test
+ public void testMultiCONCAT() throws Exception {
+
+ // DataByteArray concat
+ byte[] a = {1,2,3};
+ byte[] b = {4,5,6};
+ byte[] c = {7,8,9};
+ byte[] d = {10,11,12};
+ byte[] e = {13,14,15};
+ byte[] expected = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
+ DataByteArray dbaExpected = new DataByteArray(expected);
+
+ DataByteArray dbaA = new DataByteArray(a);
+ DataByteArray dbaB = new DataByteArray(b);
+ DataByteArray dbaC = new DataByteArray(c);
+ DataByteArray dbaD = new DataByteArray(d);
+ DataByteArray dbaE = new DataByteArray(e);
+
+ EvalFunc<DataByteArray> concat = new CONCAT();
+
+ Tuple t = TupleFactory.getInstance().newTuple(5);
+ t.set(0, dbaA);
+ t.set(1, dbaB);
+ t.set(2, dbaC);
+ t.set(3, dbaD);
+ t.set(4, dbaE);
+
+ DataByteArray result = concat.exec(t);
+ String msg = "[Testing CONCAT on >2 tuples for input type: bytearray]";
+ assertTrue(msg, result.equals(dbaExpected));
+
+ // String concat
+ String s1 = "high ";
+ String s2 = "fives ";
+ String s3 = "kick ";
+ String s4 = "ass ";
+ String s5 = "yo";
+ String exp = "high fives kick ass yo";
+ EvalFunc<String> sConcat = new StringConcat();
+ Tuple ts = TupleFactory.getInstance().newTuple(5);
+ ts.set(0, s1);
+ ts.set(1, s2);
+ ts.set(2, s3);
+ ts.set(3, s4);
+ ts.set(4, s5);
+ String res = sConcat.exec(ts);
+ msg = "[Testing StringConcat on >2 tuples input type: String]";
+ assertTrue(msg, res.equals(exp));
+
+ }
+
+ @Test
public void testSIZE() throws Exception {
// DataByteArray size
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java?rev=946722&r1=946721&r2=946722&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java Thu May 20
17:33:06 2010
@@ -514,6 +514,15 @@ public class TestDataModel extends junit
}
@Test
+ public void testByteArrayAppend() throws Exception {
+ DataByteArray expected = new DataByteArray("hello world");
+ DataByteArray db1 = new DataByteArray("hello ");
+ DataByteArray db2 = new DataByteArray("world");
+ db1.append(db2);
+ assertTrue("appends as expected", db1.equals(expected));
+ }
+
+ @Test
public void testMapConversionErr() throws Exception {
List list = new ArrayList();
try {
