http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java b/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java new file mode 100644 index 0000000..d9a2735 --- /dev/null +++ b/crunch/src/test/java/org/apache/crunch/types/avro/AvrosTest.java @@ -0,0 +1,224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.types.avro; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.util.Collection; + +import org.apache.avro.Schema; +import org.apache.avro.Schema.Type; +import org.apache.avro.generic.GenericData; +import org.apache.avro.util.Utf8; +import org.apache.hadoop.io.LongWritable; +import org.junit.Test; + +import org.apache.crunch.Pair; +import org.apache.crunch.Tuple3; +import org.apache.crunch.Tuple4; +import org.apache.crunch.TupleN; +import org.apache.crunch.test.Person; +import org.apache.crunch.types.PTableType; +import org.apache.crunch.types.PType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +/** + * TODO test Avros.register and Avros.containers + */ +public class AvrosTest { + + @Test + public void testNulls() throws Exception { + Void n = null; + testInputOutputFn(Avros.nulls(), n, n); + } + + @Test + public void testStrings() throws Exception { + String s = "abc"; + Utf8 w = new Utf8(s); + testInputOutputFn(Avros.strings(), s, w); + } + + @Test + public void testInts() throws Exception { + int j = 55; + testInputOutputFn(Avros.ints(), j, j); + } + @Test + public void testLongs() throws Exception { + long j = Long.MAX_VALUE; + testInputOutputFn(Avros.longs(), j, j); + } + @Test + public void testFloats() throws Exception { + float j = Float.MIN_VALUE; + testInputOutputFn(Avros.floats(), j, j); + } + @Test + public void testDoubles() throws Exception { + double j = Double.MIN_VALUE; + testInputOutputFn(Avros.doubles(), j, j); + } + + @Test + public void testBooleans() throws Exception { + boolean j = true; + testInputOutputFn(Avros.booleans(), j, j); + } + + @Test + public void testBytes() throws Exception { + byte[] bytes = new byte[] { 17, 26, -98 }; + ByteBuffer bb = ByteBuffer.wrap(bytes); + testInputOutputFn(Avros.bytes(), bb, bb); + } + + @Test + public void testCollections() throws Exception { + Collection<String> j = Lists.newArrayList(); + j.add("a"); + j.add("b"); + Schema collectionSchema = Schema.createArray( + Schema.createUnion(ImmutableList.of( + Avros.strings().getSchema(), Schema.create(Type.NULL)))); + GenericData.Array<Utf8> w = new GenericData.Array<Utf8>(2, collectionSchema); + w.add(new Utf8("a")); + w.add(new Utf8("b")); + testInputOutputFn(Avros.collections(Avros.strings()), j, w); + } + + @Test + public void testNestedTables() throws Exception { + PTableType<Long, Long> pll = Avros.tableOf(Avros.longs(), Avros.longs()); + String schema = Avros.tableOf(pll, Avros.strings()).getSchema().toString(); + assertNotNull(schema); + } + + @Test + public void testPairs() throws Exception { + AvroType<Pair<String, String>> at = Avros.pairs(Avros.strings(), Avros.strings()); + Pair<String, String> j = Pair.of("a", "b"); + GenericData.Record w = new GenericData.Record(at.getSchema()); + w.put(0, new Utf8("a")); + w.put(1, new Utf8("b")); + testInputOutputFn(at, j, w); + } + + @Test + public void testPairEquals() throws Exception { + AvroType<Pair<Long, ByteBuffer>> at1 = Avros.pairs(Avros.longs(), Avros.bytes()); + AvroType<Pair<Long, ByteBuffer>> at2 = Avros.pairs(Avros.longs(), Avros.bytes()); + assertEquals(at1, at2); + assertEquals(at1.hashCode(), at2.hashCode()); + } + + @Test + @SuppressWarnings("rawtypes") + public void testTriples() throws Exception { + AvroType at = Avros.triples(Avros.strings(), Avros.strings(), Avros.strings()); + Tuple3 j = Tuple3.of("a", "b", "c"); + GenericData.Record w = new GenericData.Record(at.getSchema()); + w.put(0, new Utf8("a")); + w.put(1, new Utf8("b")); + w.put(2, new Utf8("c")); + testInputOutputFn(at, j, w); + } + + @Test + @SuppressWarnings("rawtypes") + public void testQuads() throws Exception { + AvroType at = Avros.quads(Avros.strings(), Avros.strings(), Avros.strings(), Avros.strings()); + Tuple4 j = Tuple4.of("a", "b", "c", "d"); + GenericData.Record w = new GenericData.Record(at.getSchema()); + w.put(0, new Utf8("a")); + w.put(1, new Utf8("b")); + w.put(2, new Utf8("c")); + w.put(3, new Utf8("d")); + testInputOutputFn(at, j, w); + } + + @Test + @SuppressWarnings("rawtypes") + public void testTupleN() throws Exception { + AvroType at = Avros.tuples(Avros.strings(), Avros.strings(), Avros.strings(), Avros.strings(), + Avros.strings()); + TupleN j = new TupleN("a", "b", "c", "d", "e"); + GenericData.Record w = new GenericData.Record(at.getSchema()); + w.put(0, new Utf8("a")); + w.put(1, new Utf8("b")); + w.put(2, new Utf8("c")); + w.put(3, new Utf8("d")); + w.put(4, new Utf8("e")); + testInputOutputFn(at, j, w); + + } + + @Test + @SuppressWarnings("rawtypes") + public void testWritables() throws Exception { + AvroType at = Avros.writables(LongWritable.class); + LongWritable lw = new LongWritable(1729L); + assertEquals(lw, at.getInputMapFn().map(at.getOutputMapFn().map(lw))); + } + + @Test + @SuppressWarnings("rawtypes") + public void testTableOf() throws Exception { + AvroType at = Avros.tableOf(Avros.strings(), Avros.strings()); + Pair<String, String> j = Pair.of("a", "b"); + org.apache.avro.mapred.Pair w = new org.apache.avro.mapred.Pair(at.getSchema()); + w.put(0, new Utf8("a")); + w.put(1, new Utf8("b")); + // TODO update this after resolving the o.a.a.m.Pair.equals issue + initialize(at); + assertEquals(j, at.getInputMapFn().map(w)); + org.apache.avro.mapred.Pair converted = + (org.apache.avro.mapred.Pair) at.getOutputMapFn().map(j); + assertEquals(w.key(), converted.key()); + assertEquals(w.value(), converted.value()); + } + + private static void initialize(PType ptype) { + ptype.getInputMapFn().initialize(); + ptype.getOutputMapFn().initialize(); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + protected static void testInputOutputFn(PType ptype, Object java, Object avro) { + initialize(ptype); + assertEquals(java, ptype.getInputMapFn().map(avro)); + assertEquals(avro, ptype.getOutputMapFn().map(java)); + } + + @Test + public void testIsPrimitive_True() { + assertTrue(Avros.isPrimitive(Avros.ints())); + } + + @Test + public void testIsPrimitive_False() { + assertFalse(Avros.isPrimitive(Avros.reflects(Person.class))); + } + +}
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/java/org/apache/crunch/types/writable/WritableGroupedTableTypeTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/types/writable/WritableGroupedTableTypeTest.java b/crunch/src/test/java/org/apache/crunch/types/writable/WritableGroupedTableTypeTest.java new file mode 100644 index 0000000..6094152 --- /dev/null +++ b/crunch/src/test/java/org/apache/crunch/types/writable/WritableGroupedTableTypeTest.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.types.writable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import java.util.List; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import org.apache.crunch.Pair; +import org.apache.crunch.types.PGroupedTableType; +import com.google.common.collect.Lists; + +public class WritableGroupedTableTypeTest { + + @Test + public void testGetDetachedValue() { + Integer integerValue = 42; + Text textValue = new Text("forty-two"); + Iterable<Text> inputTextIterable = Lists.newArrayList(textValue); + Pair<Integer, Iterable<Text>> pair = Pair.of(integerValue, inputTextIterable); + + PGroupedTableType<Integer, Text> groupedTableType = Writables.tableOf(Writables.ints(), Writables.writables(Text.class)) + .getGroupedTableType(); + + Pair<Integer, Iterable<Text>> detachedPair = groupedTableType.getDetachedValue(pair); + + assertSame(integerValue, detachedPair.first()); + List<Text> textList = Lists.newArrayList(detachedPair.second()); + assertEquals(inputTextIterable, textList); + assertNotSame(textValue, textList.get(0)); + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/java/org/apache/crunch/types/writable/WritableTableTypeTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/types/writable/WritableTableTypeTest.java b/crunch/src/test/java/org/apache/crunch/types/writable/WritableTableTypeTest.java new file mode 100644 index 0000000..4c14fc1 --- /dev/null +++ b/crunch/src/test/java/org/apache/crunch/types/writable/WritableTableTypeTest.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.types.writable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import org.apache.crunch.Pair; + +public class WritableTableTypeTest { + + @Test + public void testGetDetachedValue() { + Integer integerValue = 42; + Text textValue = new Text("forty-two"); + Pair<Integer, Text> pair = Pair.of(integerValue, textValue); + + WritableTableType<Integer, Text> tableType = Writables.tableOf(Writables.ints(), + Writables.writables(Text.class)); + + Pair<Integer, Text> detachedPair = tableType.getDetachedValue(pair); + + assertSame(integerValue, detachedPair.first()); + assertEquals(textValue, detachedPair.second()); + assertNotSame(textValue, detachedPair.second()); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/java/org/apache/crunch/types/writable/WritableTypeTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/types/writable/WritableTypeTest.java b/crunch/src/test/java/org/apache/crunch/types/writable/WritableTypeTest.java new file mode 100644 index 0000000..ea0d11a --- /dev/null +++ b/crunch/src/test/java/org/apache/crunch/types/writable/WritableTypeTest.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.types.writable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +public class WritableTypeTest { + + @Test + public void testGetDetachedValue_AlreadyMappedWritable() { + WritableType<String, Text> stringType = Writables.strings(); + String value = "test"; + assertSame(value, stringType.getDetachedValue(value)); + } + + @Test + public void testGetDetachedValue_CustomWritable() { + WritableType<Text, Text> textWritableType = Writables.writables(Text.class); + Text value = new Text("test"); + + Text detachedValue = textWritableType.getDetachedValue(value); + assertEquals(value, detachedValue); + assertNotSame(value, detachedValue); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java b/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java new file mode 100644 index 0000000..593b773 --- /dev/null +++ b/crunch/src/test/java/org/apache/crunch/types/writable/WritablesTest.java @@ -0,0 +1,279 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.types.writable; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collection; + +import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.FloatWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.junit.Test; + +import org.apache.crunch.Pair; +import org.apache.crunch.Tuple3; +import org.apache.crunch.Tuple4; +import org.apache.crunch.TupleN; +import org.apache.crunch.types.PTableType; +import org.apache.crunch.types.PType; +import com.google.common.collect.Lists; + +public class WritablesTest { + + @Test + public void testNulls() throws Exception { + Void n = null; + NullWritable nw = NullWritable.get(); + testInputOutputFn(Writables.nulls(), n, nw); + } + + @Test + public void testStrings() throws Exception { + String s = "abc"; + Text text = new Text(s); + testInputOutputFn(Writables.strings(), s, text); + } + + @Test + public void testInts() throws Exception { + int j = 55; + IntWritable w = new IntWritable(j); + testInputOutputFn(Writables.ints(), j, w); + } + + @Test + public void testLongs() throws Exception { + long j = 55; + LongWritable w = new LongWritable(j); + testInputOutputFn(Writables.longs(), j, w); + } + + @Test + public void testFloats() throws Exception { + float j = 55.5f; + FloatWritable w = new FloatWritable(j); + testInputOutputFn(Writables.floats(), j, w); + } + + @Test + public void testDoubles() throws Exception { + double j = 55.5d; + DoubleWritable w = new DoubleWritable(j); + testInputOutputFn(Writables.doubles(), j, w); + } + + @Test + public void testBoolean() throws Exception { + boolean j = false; + BooleanWritable w = new BooleanWritable(j); + testInputOutputFn(Writables.booleans(), j, w); + } + + @Test + public void testBytes() throws Exception { + byte[] bytes = new byte[] { 17, 26, -98 }; + BytesWritable bw = new BytesWritable(bytes); + ByteBuffer bb = ByteBuffer.wrap(bytes); + testInputOutputFn(Writables.bytes(), bb, bw); + } + + @Test + public void testCollections() throws Exception { + String s = "abc"; + Collection<String> j = Lists.newArrayList(); + j.add(s); + GenericArrayWritable<Text> w = new GenericArrayWritable<Text>(Text.class); + w.set(new Text[]{ + new Text(s) + }); + testInputOutputFn(Writables.collections(Writables.strings()), j, w); + } + + @Test + public void testPairs() throws Exception { + Pair<String, String> j = Pair.of("a", "b"); + TupleWritable w = new TupleWritable(new Text[] { + new Text("a"), + new Text("b"), + }); + w.setWritten(0); + w.setWritten(1); + testInputOutputFn(Writables.pairs(Writables.strings(), Writables.strings()), j, w); + } + + @Test + public void testNestedTables() throws Exception { + PTableType<Long, Long> pll = Writables.tableOf(Writables.longs(), Writables.longs()); + PTableType<Pair<Long, Long>, String> nest = Writables.tableOf(pll, Writables.strings()); + assertNotNull(nest); + } + + @Test + public void testPairEquals() throws Exception { + PType<Pair<Long, ByteBuffer>> t1 = Writables.pairs(Writables.longs(), Writables.bytes()); + PType<Pair<Long, ByteBuffer>> t2 = Writables.pairs(Writables.longs(), Writables.bytes()); + assertEquals(t1, t2); + assertEquals(t1.hashCode(), t2.hashCode()); + } + + @Test + @SuppressWarnings("rawtypes") + public void testTriples() throws Exception { + Tuple3 j = Tuple3.of("a", "b", "c"); + TupleWritable w = new TupleWritable(new Text[] { + new Text("a"), + new Text("b"), + new Text("c"), + }); + w.setWritten(0); + w.setWritten(1); + w.setWritten(2); + WritableType<?, ?> wt = Writables.triples(Writables.strings(), + Writables.strings(), Writables.strings()); + testInputOutputFn(wt, j, w); + } + + @Test + @SuppressWarnings("rawtypes") + public void testQuads() throws Exception { + Tuple4 j = Tuple4.of("a", "b", "c", "d"); + TupleWritable w = new TupleWritable(new Text[] { + new Text("a"), + new Text("b"), + new Text("c"), + new Text("d"), + }); + w.setWritten(0); + w.setWritten(1); + w.setWritten(2); + w.setWritten(3); + WritableType<?, ?> wt = Writables.quads(Writables.strings(), Writables.strings(), + Writables.strings(), Writables.strings()); + testInputOutputFn(wt, j, w); + } + + @Test + public void testTupleN() throws Exception { + TupleN j = new TupleN("a", "b", "c", "d", "e"); + TupleWritable w = new TupleWritable(new Text[] { + new Text("a"), + new Text("b"), + new Text("c"), + new Text("d"), + new Text("e"), + }); + w.setWritten(0); + w.setWritten(1); + w.setWritten(2); + w.setWritten(3); + w.setWritten(4); + WritableType<?, ?> wt = Writables.tuples(Writables.strings(), Writables.strings(), + Writables.strings(), Writables.strings(), Writables.strings()); + testInputOutputFn(wt, j, w); + } + + protected static class TestWritable implements Writable { + String left; + int right; + @Override + public void write(DataOutput out) throws IOException { + out.writeUTF(left); + out.writeInt(right); + } + @Override + public void readFields(DataInput in) throws IOException { + left = in.readUTF(); + right = in.readInt(); + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + TestWritable other = (TestWritable) obj; + if (left == null) { + if (other.left != null) + return false; + } else if (!left.equals(other.left)) + return false; + if (right != other.right) + return false; + return true; + } + + } + @Test + public void testRecords() throws Exception { + TestWritable j = new TestWritable(); + j.left = "a"; + j.right = 1; + TestWritable w = new TestWritable(); + w.left = "a"; + w.right = 1; + WritableType<?, ?> wt = Writables.records(TestWritable.class); + testInputOutputFn(wt, j, w); + } + + @Test + public void testTableOf() throws Exception { + Pair<String, String> j = Pair.of("a", "b"); + Pair<Text, Text> w = Pair.of(new Text("a"), new Text("b")); + WritableTableType<String, String> wtt = Writables.tableOf(Writables.strings(), Writables.strings()); + testInputOutputFn(wtt, j, w); + } + + @Test + public void testRegister() throws Exception { + WritableType<TestWritable, TestWritable> wt = Writables.writables(TestWritable.class); + Writables.register(TestWritable.class, wt); + assertSame(Writables.records(TestWritable.class), wt); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + protected static void testInputOutputFn(PType ptype, Object java, Object writable) { + ptype.getInputMapFn().initialize(); + ptype.getOutputMapFn().initialize(); + assertEquals(java, ptype.getInputMapFn().map(writable)); + assertEquals(writable, ptype.getOutputMapFn().map(java)); + } + + @Test + public void testDeepCopy() { + Text text = new Text("Test"); + Text copiedText = Writables.deepCopy(text, Text.class); + assertEquals(text, copiedText); + assertNotSame(text, copiedText); + } +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java b/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java new file mode 100644 index 0000000..eda45c4 --- /dev/null +++ b/crunch/src/test/java/org/apache/crunch/util/DistCacheTest.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class DistCacheTest { + + // A temporary folder used to hold files created for the test. + @Rule + public TemporaryFolder testFolder = new TemporaryFolder(); + + // A configuration and lists of paths to use in tests. + private Configuration testConf; + private String[] testFilePaths; + private String[] testFileQualifiedPaths; + + /** + * Setup resources for tests. These include: + * 1. A Hadoop configuration. + * 2. A directory of temporary files that includes 3 .jar files and 1 other file. + * 3. Arrays containing the canonical paths and qualified paths to the test files. + */ + @Before + public void setup() throws IOException { + // Create a configuration for tests. + testConf = new Configuration(); + + // Create the test files and add their paths to the list of test file paths. + testFilePaths = new String[3]; + testFilePaths[0] = testFolder.newFile("jar1.jar").getCanonicalPath(); + testFilePaths[1] = testFolder.newFile("jar2.jar").getCanonicalPath(); + testFilePaths[2] = testFolder.newFile("jar3.jar").getCanonicalPath(); + testFolder.newFile("notJar.other"); + + // Populate a list of qualified paths from the test file paths. + testFileQualifiedPaths = new String[3]; + for (int i = 0; i < testFilePaths.length; i++) { + testFileQualifiedPaths[i] = "file:" + testFilePaths[i]; + } + } + + /** + * Tests adding jars one-by-one to a job's configuration. + * + * @throws IOException If there is a problem adding the jars. + */ + @Test + public void testAddJar() throws IOException { + // Add each valid jar path to the distributed cache configuration, and verify each was + // added correctly in turn. + for (int i = 0; i < testFilePaths.length; i++) { + DistCache.addJarToDistributedCache(testConf, testFilePaths[i]); + assertEquals("tmpjars configuration var does not contain expected value.", + StringUtils.join(testFileQualifiedPaths, ",", 0, i + 1), testConf.get("tmpjars")); + } + } + + /** + * Tests that attempting to add the path to a jar that does not exist to the configuration + * throws an exception. + * + * @throws IOException If the added jar path does not exist. This exception is expected. + */ + @Test(expected = IOException.class) + public void testAddJarThatDoesntExist() throws IOException { + DistCache.addJarToDistributedCache(testConf, "/garbage/doesntexist.jar"); + } + + /** + * Tests that adding a directory of jars to the configuration works as expected. .jar files + * under the added directory should be added to the configuration, + * and all other files should be skipped. + * + * @throws IOException If there is a problem adding the jar directory to the configuration. + */ + @Test + public void testAddJarDirectory() throws IOException { + DistCache.addJarDirToDistributedCache(testConf, testFolder.getRoot().getCanonicalPath()); + // Throw the added jar paths in a set to detect duplicates. + String[] splitJarPaths = StringUtils.split(testConf.get("tmpjars"), ","); + Set<String> addedJarPaths = new HashSet<String>(); + for (String path: splitJarPaths) { + addedJarPaths.add(path); + } + assertEquals("Incorrect number of jar paths added.", testFilePaths.length, + addedJarPaths.size()); + + // Ensure all expected paths were added. + for (int i = 0; i < testFileQualifiedPaths.length; i++) { + assertTrue("Expected jar path missing from jar paths added to tmpjars: " + + testFileQualifiedPaths[i], addedJarPaths.contains(testFileQualifiedPaths[i])); + } + } + + /** + * Tests that adding a jar directory that does not exist to the configuration throws an + * exception. + * + * @throws IOException If the added jar directory does not exist. This exception is expected. + */ + @Test(expected = IOException.class) + public void testAddJarDirectoryThatDoesntExist() throws IOException { + DistCache.addJarDirToDistributedCache(testConf, "/garbage/doesntexist"); + } + + /** + * Tests that adding a jar directory that is not a directory to the configuration throws an + * exception. + * + * @throws IOException If the added jar directory is not a directory. This exception is expected. + */ + @Test(expected = IOException.class) + public void testAddJarDirectoryNotDirectory() throws IOException { + DistCache.addJarDirToDistributedCache(testConf, testFilePaths[0]); + } +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/resources/customers.txt ---------------------------------------------------------------------- diff --git a/crunch/src/test/resources/customers.txt b/crunch/src/test/resources/customers.txt new file mode 100644 index 0000000..98f3f3d --- /dev/null +++ b/crunch/src/test/resources/customers.txt @@ -0,0 +1,4 @@ +111|John Doe +222|Jane Doe +333|Someone Else +444|Has No Orders \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/resources/docs.txt ---------------------------------------------------------------------- diff --git a/crunch/src/test/resources/docs.txt b/crunch/src/test/resources/docs.txt new file mode 100644 index 0000000..90a3f65 --- /dev/null +++ b/crunch/src/test/resources/docs.txt @@ -0,0 +1,6 @@ +A this doc has this text +A and this text as well +A but also this +B this doc has some text +B but not as much as the last +B doc http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/resources/employee.avro ---------------------------------------------------------------------- diff --git a/crunch/src/test/resources/employee.avro b/crunch/src/test/resources/employee.avro new file mode 100644 index 0000000..3563df9 --- /dev/null +++ b/crunch/src/test/resources/employee.avro @@ -0,0 +1,9 @@ +{ +"namespace": "com.cloudera.crunch.test", +"name": "Employee", +"type": "record", +"fields": [ + {"name": "name", "type": ["string", "null"] }, + {"name": "salary", "type": "int"}, + {"name": "department", "type": ["string", "null"] } ] +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/resources/emptyTextFile.txt ---------------------------------------------------------------------- diff --git a/crunch/src/test/resources/emptyTextFile.txt b/crunch/src/test/resources/emptyTextFile.txt new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/resources/letters.txt ---------------------------------------------------------------------- diff --git a/crunch/src/test/resources/letters.txt b/crunch/src/test/resources/letters.txt new file mode 100644 index 0000000..916bfc9 --- /dev/null +++ b/crunch/src/test/resources/letters.txt @@ -0,0 +1,2 @@ +a +bb \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/83acb813/crunch/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/crunch/src/test/resources/log4j.properties b/crunch/src/test/resources/log4j.properties new file mode 100644 index 0000000..c8173d7 --- /dev/null +++ b/crunch/src/test/resources/log4j.properties @@ -0,0 +1,11 @@ +# ***** Set root logger level to INFO and its only appender to A. +log4j.logger.org.apache.crunch=info, A + +# Log warnings on Hadoop for the local runner when testing +log4j.logger.org.apache.hadoop=warn, A + +# ***** A is set to be a ConsoleAppender. +log4j.appender.A=org.apache.log4j.ConsoleAppender +# ***** A uses PatternLayout. +log4j.appender.A.layout=org.apache.log4j.PatternLayout +log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
