Author: todd Date: Thu Aug 14 04:53:36 2014 New Revision: 1617878 URL: http://svn.apache.org/r1617878 Log: MAPREDUCE-6026. native-task: fix logging. Contributed by Manu Zhang.
Added: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/log4j.properties Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt?rev=1617878&r1=1617877&r2=1617878&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt Thu Aug 14 04:53:36 2014 @@ -13,3 +13,4 @@ MAPREDUCE-5984. native-task: Reuse lz4 s MAPREDUCE-5976. native-task: should not fail to build if snappy is missing (Manu Zhang) MAPREDUCE-5978. native-task: remove test case for not supported codec Bzip2Codec and DefaultCodec (Manu Zhang) MAPREDUCE-6006. native-task: add native tests to maven and fix bug in pom.xml (Binglin Chang via todd) +MAPREDUCE-6026. native-task: fix logging (Manu Zhang via todd) Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java?rev=1617878&r1=1617877&r2=1617878&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java Thu Aug 14 04:53:36 2014 @@ -19,6 +19,8 @@ package org.apache.hadoop.mapred.nativet import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.ByteWritable; import org.apache.hadoop.io.BytesWritable; @@ -33,10 +35,9 @@ import org.apache.hadoop.io.VLongWritabl import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.nativetask.serde.*; -import org.apache.log4j.Logger; public class HadoopPlatform extends Platform { - private static final Logger LOG = Logger.getLogger(HadoopPlatform.class); + private static final Log LOG = LogFactory.getLog(HadoopPlatform.class); public HadoopPlatform() throws IOException { } Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java?rev=1617878&r1=1617877&r2=1617878&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java Thu Aug 14 04:53:36 2014 @@ -20,11 +20,12 @@ package org.apache.hadoop.mapred.nativet import java.io.IOException; import java.util.ServiceLoader; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer; import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization; -import org.apache.log4j.Logger; /** @@ -34,7 +35,7 @@ import org.apache.log4j.Logger; */ public class Platforms { - private static final Logger LOG = Logger.getLogger(Platforms.class); + private static final Log LOG = LogFactory.getLog(Platforms.class); private static final ServiceLoader<Platform> platforms = ServiceLoader.load(Platform.class); public static void init(Configuration conf) throws IOException { Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java?rev=1617878&r1=1617877&r2=1617878&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java Thu Aug 14 04:53:36 2014 @@ -19,6 +19,8 @@ package org.apache.hadoop.mapred.nativet import static org.junit.Assert.assertEquals; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -34,6 +36,7 @@ import org.apache.hadoop.mapreduce.Job; import org.junit.Test; public class LargeKVCombinerTest { + private static final Log LOG = LogFactory.getLog(LargeKVCombinerTest.class); @Test public void testLargeValueCombiner(){ @@ -57,7 +60,7 @@ public class LargeKVCombinerTest { int max = i; int min = Math.max(i / 4, max - 10); - System.out.println("===KV Size Test: min size: " + min + ", max size: " + max); + LOG.info("===KV Size Test: min size: " + min + ", max size: " + max); normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min)); normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max)); Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java?rev=1617878&r1=1617877&r2=1617878&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java Thu Aug 14 04:53:36 2014 @@ -23,6 +23,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -37,6 +39,8 @@ import org.junit.runners.Parameterized.P @RunWith(Parameterized.class) public class KVTest { + private static final Log LOG = LogFactory.getLog(KVTest.class); + private static Class<?>[] keyclasses = null; private static Class<?>[] valueclasses = null; private static String[] keyclassNames = null; @@ -53,7 +57,7 @@ public class KVTest { public static Iterable<Class<?>[]> data() { final String valueclassesStr = nativekvtestconf .get(TestConstants.NATIVETASK_KVTEST_VALUECLASSES); - System.out.println(valueclassesStr); + LOG.info(valueclassesStr); valueclassNames = valueclassesStr.replaceAll("\\s", "").split(";");// delete // " " final ArrayList<Class<?>> tmpvalueclasses = new ArrayList<Class<?>>(); @@ -69,7 +73,7 @@ public class KVTest { } valueclasses = tmpvalueclasses.toArray(new Class[tmpvalueclasses.size()]); final String keyclassesStr = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_KEYCLASSES); - System.out.println(keyclassesStr); + LOG.info(keyclassesStr); keyclassNames = keyclassesStr.replaceAll("\\s", "").split(";");// delete // " " final ArrayList<Class<?>> tmpkeyclasses = new ArrayList<Class<?>>(); Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java?rev=1617878&r1=1617877&r2=1617878&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java Thu Aug 14 04:53:36 2014 @@ -21,6 +21,8 @@ import static org.junit.Assert.assertEqu import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -32,6 +34,7 @@ import org.apache.hadoop.mapred.nativeta import org.junit.Test; public class LargeKVTest { + private static final Log LOG = LogFactory.getLog(LargeKVTest.class); @Test public void testKeySize() { @@ -69,8 +72,8 @@ public class LargeKVTest { normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min)); normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max)); - System.out.println("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: " - + keyClass.getName() + ", valueClass: " + valueClass.getName()); + LOG.info("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: " + + keyClass.getName() + ", valueClass: " + valueClass.getName()); final String nativeOutPut = runNativeLargeKVTest("Test Large Value Size:" + String.valueOf(i), keyClass, valueClass, nativeConf); Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java?rev=1617878&r1=1617877&r2=1617878&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java Thu Aug 14 04:53:36 2014 @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.HashMap; import java.util.Random; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -40,7 +42,8 @@ import org.apache.hadoop.mapred.nativeta public class TestInputFile { - + private static Log LOG = LogFactory.getLog(TestInputFile.class); + public static class KVSizeScope { private static final int DefaultMinNum = 1; private static final int DefaultMaxNum = 64; @@ -120,8 +123,8 @@ public class TestInputFile { } public void createSequenceTestFile(String filepath, int base, byte start) throws Exception { - System.out.println("create file " + filepath); - System.out.println(keyClsName + " " + valueClsName); + LOG.info("create file " + filepath); + LOG.info(keyClsName + " " + valueClsName); Class<?> tmpkeycls, tmpvaluecls; try { tmpkeycls = Class.forName(keyClsName); Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java?rev=1617878&r1=1617877&r2=1617878&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java Thu Aug 14 04:53:36 2014 @@ -19,9 +19,12 @@ package org.apache.hadoop.mapred.nativet import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator; public class EnforceNativeOutputCollectorDelegator<K, V> extends NativeMapOutputCollectorDelegator<K, V> { + private static final Log LOG = LogFactory.getLog(EnforceNativeOutputCollectorDelegator.class); private boolean nativetaskloaded = false; @Override @@ -32,8 +35,7 @@ public class EnforceNativeOutputCollecto nativetaskloaded = true; } catch (final Exception e) { nativetaskloaded = false; - System.err.println("load nativetask lib failed, Native-Task Delegation is disabled"); - e.printStackTrace(); + LOG.error("load nativetask lib failed, Native-Task Delegation is disabled", e); } } Added: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/log4j.properties URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/log4j.properties?rev=1617878&view=auto ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/log4j.properties (added) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/log4j.properties Thu Aug 14 04:53:36 2014 @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n