Repository: hadoop Updated Branches: refs/heads/trunk a0ad975ea -> 3c9181722
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java new file mode 100644 index 0000000..b665971 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.nativetask.testutil; + +import java.io.IOException; +import java.util.zip.CRC32; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; + +public class ResultVerifier { + /** + * verify the result + * + * @param sample + * :nativetask output + * @param source + * :yuanwenjian + * @throws Exception + */ + public static boolean verify(String sample, String source) throws Exception { + FSDataInputStream sourcein = null; + FSDataInputStream samplein = null; + + final Configuration conf = new Configuration(); + final FileSystem fs = FileSystem.get(conf); + final Path hdfssource = new Path(source); + final Path[] sourcepaths = FileUtil.stat2Paths(fs.listStatus(hdfssource)); + + final Path hdfssample = new Path(sample); + final Path[] samplepaths = FileUtil.stat2Paths(fs.listStatus(hdfssample)); + if (sourcepaths == null) { + throw new Exception("source file can not be found"); + } + if (samplepaths == null) { + throw new Exception("sample file can not be found"); + } + if (sourcepaths.length != samplepaths.length) { + return false; + } + for (int i = 0; i < sourcepaths.length; i++) { + final Path sourcepath = sourcepaths[i]; + // op result file start with "part-r" like part-r-00000 + + if (!sourcepath.getName().startsWith("part-r")) { + continue; + } + Path samplepath = null; + for (int j = 0; j < samplepaths.length; j++) { + if (samplepaths[i].getName().equals(sourcepath.getName())) { + samplepath = samplepaths[i]; + break; + } + } + if (samplepath == null) { + throw new Exception("cound not found file " + samplepaths[0].getParent() + "/" + sourcepath.getName() + + " , as sourcepaths has such file"); + } + + // compare + try { + if (fs.exists(sourcepath) && fs.exists(samplepath)) { + sourcein = fs.open(sourcepath); + samplein = fs.open(samplepath); + } else { + System.err.println("result file not found:" + sourcepath + " or " + samplepath); + return false; + } + + CRC32 sourcecrc, samplecrc; + samplecrc = new CRC32(); + sourcecrc = new CRC32(); + final byte[] bufin = new byte[1 << 16]; + int readnum = 0; + int totalRead = 0; + while (samplein.available() > 0) { + readnum = samplein.read(bufin); + totalRead += readnum; + samplecrc.update(bufin, 0, readnum); + } + + if (0 == totalRead) { + throw new Exception("source " + sample + " is empty file"); + } + + totalRead = 0; + while (sourcein.available() > 0) { + readnum = sourcein.read(bufin); + totalRead += readnum; + sourcecrc.update(bufin, 0, readnum); + } + if (0 == totalRead) { + throw new Exception("source " + sample + " is empty file"); + } + + if (samplecrc.getValue() == sourcecrc.getValue()) { + ; + } else { + return false; + } + } catch (final IOException e) { + throw new Exception("verify exception :", e); + } finally { + + try { + if (samplein != null) { + samplein.close(); + } + if (sourcein != null) { + sourcein.close(); + } + } catch (final IOException e) { + e.printStackTrace(); + } + + } + } + return true; + } + + public static void main(String[] args) { + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java new file mode 100644 index 0000000..9cc473d --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.nativetask.testutil; + +import org.apache.hadoop.conf.Configuration; + +public class ScenarioConfiguration extends Configuration { + public ScenarioConfiguration() { + super(); + this.addResource(TestConstants.COMMON_CONF_PATH); + } + + public void addcombinerConf() { + this.addResource(TestConstants.COMBINER_CONF_PATH); + } + + public void addKVTestConf() { + this.addResource(TestConstants.KVTEST_CONF_PATH); + } + + public void addNonSortTestConf() { + this.addResource(TestConstants.NONSORT_TEST_CONF); + } + + public void addNativeConf() { + this.set(TestConstants.NATIVETASK_COLLECTOR_DELEGATOR, + TestConstants.NATIVETASK_COLLECTOR_DELEGATOR_CLASS); + } + + public static Configuration getNormalConfiguration() { + Configuration normalConf = new Configuration(); + normalConf.addResource("common_conf.xml"); + normalConf.addResource("normal_conf.xml"); + return normalConf; + } + + public static Configuration getNativeConfiguration() { + Configuration nativeConf = new Configuration(); + nativeConf.addResource("common_conf.xml"); + nativeConf.addResource("native_conf.xml"); + return nativeConf; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java new file mode 100644 index 0000000..3beb082 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.nativetask.testutil; + +public class TestConstants { + // conf path + public static final String COMBINER_CONF_PATH = "test-combiner-conf.xml"; + public static final String KVTEST_CONF_PATH = "kvtest-conf.xml"; + public static final String NONSORT_TEST_CONF = "test-nonsort-conf.xml"; + + public static final String NATIVETASK_KVSIZE_MIN = "nativetask.kvsize.min"; + public static final String NATIVETASK_KVSIZE_MAX = "nativetask.kvsize.max"; + + public static final String NATIVETASK_KVTEST_INPUTDIR = "nativetask.kvtest.inputdir"; + public static final String NATIVETASK_KVTEST_OUTPUTDIR = "nativetask.kvtest.outputdir"; + public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = "normal.kvtest.outputdir"; + public static final String NATIVETASK_KVTEST_CREATEFILE = "nativetask.kvtest.createfile"; + public static final String NATIVETASK_KVTEST_FILE_RECORDNUM = "nativetask.kvtest.file.recordnum"; + public static final String NATIVETASK_KVTEST_KEYCLASSES = "nativetask.kvtest.keyclasses"; + public static final String NATIVETASK_KVTEST_VALUECLASSES = "nativetask.kvtest.valueclasses"; + public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class"; + public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator"; + + public static final String SNAPPY_COMPRESS_CONF_PATH = "test-snappy-compress-conf.xml"; + public static final String GZIP_COMPRESS_CONF_PATH = "test-gzip-compress-conf.xml"; + public static final String BZIP2_COMPRESS_CONF_PATH = "test-bzip2-compress-conf.xml"; + public static final String DEFAULT_COMPRESS_CONF_PATH = "test-default-compress-conf.xml"; + public static final String LZ4_COMPRESS_CONF_PATH = "test-lz4-compress-conf.xml"; + public static final String NATIVETASK_COMPRESS_FILESIZE = "nativetask.compress.filesize"; + + public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_KEY = "nativetask.combinertest.inputpath"; + public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV = "./combinertest/input"; + public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH = "nativetask.combinertest.outputdir"; + public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/native"; + public static final String NORMAL_TEST_COMBINER_OUTPUTPATH = "normal.combinertest.outputdir"; + public static final String NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/normal"; + public static final String OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH = "oldAPI.nativetask.combinertest.outputdir"; + public static final String OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH = "oldAPI.normal.combinertest.outputdir"; + public static final String NATIVETASK_COMBINER_WORDCOUNT_FILESIZE = "nativetask.combiner.wordcount.filesize"; + public static final String NATIVETASK_NONSORTTEST_FILESIZE = "nativetask.nonsorttest.filesize"; + + public static final String COMMON_CONF_PATH = "common_conf.xml"; + + public static final String FILESIZE_KEY = "kvtest.file.size"; + public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv"; + + public static final String NATIVETASK_MAP_OUTPUT_SORT = "mapreduce.sort.avoidance"; + public static final String NONSORT_TEST_INPUTDIR = "nativetask.nonsorttest.inputpath"; + public static final String NONSORT_TEST_NATIVE_OUTPUT = "nonsorttest.native.outputdir"; + public static final String NONSORT_TEST_NORMAL_OUTPUT = "nonsorttest.normal.outputdir"; + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml new file mode 100644 index 0000000..c09e17a --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/common_conf.xml @@ -0,0 +1,39 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> +<property> +<name>mapred.job.tracker</name> +<value>local</value> +</property> + +<property> +<name>keep.failed.task.files</name> +<value>true</value> +</property> + + <property> + <name>keep.task.files.pattern</name> + <value>.*_m_</value> + <description>Keep all files from tasks whose task names match the given + regular expression. Defaults to none.</description> + </property> + +<property> +<name>nativetask.kvsize.max.largekv</name> +<value>1048576</value> +</property> + +<property> +<name>native.processor.buffer.kb</name> +<value>128</value> +</property> + +<property> +<name>nativetask.output.manager</name> +<value>org.apache.hadoop.mapred.nativetask.util.LocalJobOutputFiles</value> +</property> + +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml new file mode 100644 index 0000000..addcbc7 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/kvtest-conf.xml @@ -0,0 +1,87 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> +<property> +<name>nativetask.kvtest.inputdir</name> +<value>./kvtest/input</value> +</property> +<property> +<name>nativetask.kvtest.outputdir</name> +<value>./kvtest/output/native</value> +</property> +<property> +<name>normal.kvtest.outputdir</name> +<value>./kvtest/output/normal</value> +</property> +<property> +<name>kvtest.file.size</name> +<value>10485760</value> +</property> +<property> +<name>mapreduce.reduce.class</name> +<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value> +</property> +<property> +<name>mapred.output.value.class</name> +<value>org.apache.hadoop.io.IntWritable</value> +</property> +<property> +<name>nativetask.kvtest.keyclasses</name> +<value> +org.apache.hadoop.io.BytesWritable; +org.apache.hadoop.io.BooleanWritable; +org.apache.hadoop.io.ByteWritable; +org.apache.hadoop.io.DoubleWritable; +org.apache.hadoop.io.FloatWritable; +org.apache.hadoop.io.IntWritable; +org.apache.hadoop.io.LongWritable; +org.apache.hadoop.io.Text; +org.apache.hadoop.io.VIntWritable; +org.apache.hadoop.io.VLongWritable; +</value> +</property> +<property> +<name>nativetask.kvtest.valueclasses</name> +<value> +org.apache.hadoop.io.BytesWritable; +org.apache.hadoop.io.BooleanWritable; +org.apache.hadoop.io.ByteWritable; +org.apache.hadoop.io.DoubleWritable; +org.apache.hadoop.io.FloatWritable; +org.apache.hadoop.io.IntWritable; +org.apache.hadoop.io.LongWritable; +org.apache.hadoop.io.Text; +org.apache.hadoop.io.VIntWritable; +org.apache.hadoop.io.VLongWritable; +org.apache.hadoop.mapred.nativetask.testutil.MockValueClass; +</value> +</property> + +<property> +<name>mapred.output.compress</name> +<value>false</value> +</property> + + +<property> +<name>mapred.output.compression.type</name> +<value>BLOCK</value> +</property> +<property> +<name>mapred.compress.map.output</name> +<value>true</value> +</property> +<property> +<name>mapred.map.output.compression.codec</name> +<value>org.apache.hadoop.io.compress.SnappyCodec</value> +</property> + + +<property> +<name>hadoop.native.lib</name> +<value>true</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml new file mode 100644 index 0000000..0634d05 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/native_conf.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration> + +<property> +<name>mapreduce.job.map.output.collector.class</name> +<value>org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator</value> +</property> + +<property> +<name>io.sort.mb</name> +<value>5</value> +</property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml new file mode 100644 index 0000000..38cd7d4 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/normal_conf.xml @@ -0,0 +1,10 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration> + +<property> +<name>io.sort.mb</name> +<value>30</value> +</property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-bzip2-compress-conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-bzip2-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-bzip2-compress-conf.xml new file mode 100644 index 0000000..2484984 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-bzip2-compress-conf.xml @@ -0,0 +1,39 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> +<property> +<name>mapred.output.compress</name> +<value>false</value> +</property> +<property> +<name>nativetask.compress.filesize</name> +<value>100000</value> +</property> +<property> +<name>mapreduce.reduce.class</name> +<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value> +</property> +<property> +<name>mapred.output.value.class</name> +<value>org.apache.hadoop.io.IntWritable</value> +</property> +<property> +<name>mapred.compress.map.output</name> +<value>true</value> +</property> +<property> +<name>mapred.output.compression.type</name> +<value>BLOCK</value> +</property> +<property> +<name>mapred.map.output.compression.codec</name> +<value>org.apache.hadoop.io.compress.BZip2Codec</value> +</property> +<property> +<name>hadoop.native.lib</name> +<value>true</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml new file mode 100644 index 0000000..53370fa --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-combiner-conf.xml @@ -0,0 +1,36 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> +<property> +<name>nativetask.combinertest.inputpath</name> +<value>./combinertest/input</value> +</property> +<property> +<name>nativetask.combiner.wordcount.filesize</name> +<value>20971520</value> +</property> + +<property> +<name>nativetask.combinertest.outputdir</name> +<value>./combinertest/output/native</value> +</property> +<property> +<name>normal.combinertest.outputdir</name> +<value>./combinertest/output/normal</value> +</property> +<property> +<name>oldAPI.nativetask.combinertest.outputdir</name> +<value>./combinertest/oldapi/output/native</value> +</property> +<property> +<name>oldAPI.normal.combinertest.outputdir</name> +<value>./combinertest/oldapi/output/normal</value> +</property> +<property> +<name>mapred.job.tracker</name> +<value>local</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-default-compress-conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-default-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-default-compress-conf.xml new file mode 100644 index 0000000..765daf1 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-default-compress-conf.xml @@ -0,0 +1,39 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> +<property> +<name>mapred.output.compress</name> +<value>false</value> +</property> +<property> +<name>mapred.output.value.class</name> +<value>org.apache.hadoop.io.IntWritable</value> +</property> +<property> +<name>nativetask.compress.filesize</name> +<value>100000</value> +</property> +<property> +<name>mapreduce.reduce.class</name> +<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value> +</property> +<property> +<name>mapred.compress.map.output</name> +<value>true</value> +</property> +<property> +<name>mapred.output.compression.type</name> +<value>BLOCK</value> +</property> +<property> +<name>mapred.map.output.compression.codec</name> +<value>org.apache.hadoop.io.compress.DefaultCodec</value> +</property> +<property> +<name>hadoop.native.lib</name> +<value>true</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-gzip-compress-conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-gzip-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-gzip-compress-conf.xml new file mode 100644 index 0000000..ed43034 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-gzip-compress-conf.xml @@ -0,0 +1,39 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> +<property> +<name>mapred.output.compress</name> +<value>false</value> +</property> +<property> +<name>mapred.output.value.class</name> +<value>org.apache.hadoop.io.IntWritable</value> +</property> +<property> +<name>nativetask.compress.filesize</name> +<value>100000</value> +</property> +<property> +<name>mapreduce.reduce.class</name> +<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value> +</property> +<property> +<name>mapred.compress.map.output</name> +<value>true</value> +</property> +<property> +<name>mapred.output.compression.type</name> +<value>BLOCK</value> +</property> +<property> +<name>mapred.map.output.compression.codec</name> +<value>org.apache.hadoop.io.compress.GzipCodec</value> +</property> +<property> +<name>hadoop.native.lib</name> +<value>true</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-lz4-compress-conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-lz4-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-lz4-compress-conf.xml new file mode 100644 index 0000000..6e54c0b --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-lz4-compress-conf.xml @@ -0,0 +1,39 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> +<property> +<name>mapred.output.compress</name> +<value>false</value> +</property> +<property> +<name>mapred.output.value.class</name> +<value>org.apache.hadoop.io.IntWritable</value> +</property> +<property> +<name>nativetask.compress.filesize</name> +<value>100000</value> +</property> +<property> +<name>mapreduce.reduce.class</name> +<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value> +</property> +<property> +<name>mapred.compress.map.output</name> +<value>true</value> +</property> +<property> +<name>mapred.output.compression.type</name> +<value>BLOCK</value> +</property> +<property> +<name>mapred.map.output.compression.codec</name> +<value>org.apache.hadoop.io.compress.Lz4Codec</value> +</property> +<property> +<name>hadoop.native.lib</name> +<value>true</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml new file mode 100644 index 0000000..f1308c3 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-nonsort-conf.xml @@ -0,0 +1,31 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> +<property> +<name>nativetask.nonsorttest.inputpath</name> +<value>./sorttest/input</value> +</property> +<property> +<name>nativetask.nonsorttest.filesize</name> +<value>4194304</value> +</property> +<property> +<name>nonsorttest.native.outputdir.tmp</name> +<value>./sorttest/output/tmp</value> +</property> +<property> +<name>nonsorttest.native.outputdir</name> +<value>./sorttest/output/native</value> +</property> +<property> +<name>nonsorttest.normal.outputdir</name> +<value>./sort/output/normal</value> +</property> +<property> +<name>mapred.job.tracker</name> +<value>local</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml new file mode 100644 index 0000000..46b3ea7 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/resources/test-snappy-compress-conf.xml @@ -0,0 +1,41 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> +<property> +<name>mapred.output.compress</name> +<value>false</value> +</property> +<property> +<name>mapreduce.reduce.class</name> +<value>org.apache.hadoop.mapred.nativetask.kvtest.HashSumReducer</value> +</property> + +<property> +<name>mapred.output.value.class</name> +<value>org.apache.hadoop.io.IntWritable</value> +</property> +<property> +<name>nativetask.compress.filesize</name> +<value>100000</value> +</property> + +<property> +<name>mapred.compress.map.output</name> +<value>true</value> +</property> +<property> +<name>mapred.output.compression.type</name> +<value>BLOCK</value> +</property> +<property> +<name>mapred.map.output.compression.codec</name> +<value>org.apache.hadoop.io.compress.SnappyCodec</value> +</property> +<property> +<name>hadoop.native.lib</name> +<value>true</value> +</property> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index 72b9cd4..5a16820 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -187,5 +187,6 @@ <module>hadoop-mapreduce-client-jobclient</module> <module>hadoop-mapreduce-client-hs</module> <module>hadoop-mapreduce-client-hs-plugins</module> + <module>hadoop-mapreduce-client-nativetask</module> </modules> </project>
