http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java deleted file mode 100644 index b665971..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred.nativetask.testutil; - -import java.io.IOException; -import java.util.zip.CRC32; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; - -public class ResultVerifier { - /** - * verify the result - * - * @param sample - * :nativetask output - * @param source - * :yuanwenjian - * @throws Exception - */ - public static boolean verify(String sample, String source) throws Exception { - FSDataInputStream sourcein = null; - FSDataInputStream samplein = null; - - final Configuration conf = new Configuration(); - final FileSystem fs = FileSystem.get(conf); - final Path hdfssource = new Path(source); - final Path[] sourcepaths = FileUtil.stat2Paths(fs.listStatus(hdfssource)); - - final Path hdfssample = new Path(sample); - final Path[] samplepaths = FileUtil.stat2Paths(fs.listStatus(hdfssample)); - if (sourcepaths == null) { - throw new Exception("source file can not be found"); - } - if (samplepaths == null) { - throw new Exception("sample file can not be found"); - } - if (sourcepaths.length != samplepaths.length) { - return false; - } - for (int i = 0; i < sourcepaths.length; i++) { - final Path sourcepath = sourcepaths[i]; - // op result file start with "part-r" like part-r-00000 - - if (!sourcepath.getName().startsWith("part-r")) { - continue; - } - Path samplepath = null; - for (int j = 0; j < samplepaths.length; j++) { - if (samplepaths[i].getName().equals(sourcepath.getName())) { - samplepath = samplepaths[i]; - break; - } - } - if (samplepath == null) { - throw new Exception("cound not found file " + samplepaths[0].getParent() + "/" + sourcepath.getName() - + " , as sourcepaths has such file"); - } - - // compare - try { - if (fs.exists(sourcepath) && fs.exists(samplepath)) { - sourcein = fs.open(sourcepath); - samplein = fs.open(samplepath); - } else { - System.err.println("result file not found:" + sourcepath + " or " + samplepath); - return false; - } - - CRC32 sourcecrc, samplecrc; - samplecrc = new CRC32(); - sourcecrc = new CRC32(); - final byte[] bufin = new byte[1 << 16]; - int readnum = 0; - int totalRead = 0; - while (samplein.available() > 0) { - readnum = samplein.read(bufin); - totalRead += readnum; - samplecrc.update(bufin, 0, readnum); - } - - if (0 == totalRead) { - throw new Exception("source " + sample + " is empty file"); - } - - totalRead = 0; - while (sourcein.available() > 0) { - readnum = sourcein.read(bufin); - totalRead += readnum; - sourcecrc.update(bufin, 0, readnum); - } - if (0 == totalRead) { - throw new Exception("source " + sample + " is empty file"); - } - - if (samplecrc.getValue() == sourcecrc.getValue()) { - ; - } else { - return false; - } - } catch (final IOException e) { - throw new Exception("verify exception :", e); - } finally { - - try { - if (samplein != null) { - samplein.close(); - } - if (sourcein != null) { - sourcein.close(); - } - } catch (final IOException e) { - e.printStackTrace(); - } - - } - } - return true; - } - - public static void main(String[] args) { - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java deleted file mode 100644 index 9cc473d..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred.nativetask.testutil; - -import org.apache.hadoop.conf.Configuration; - -public class ScenarioConfiguration extends Configuration { - public ScenarioConfiguration() { - super(); - this.addResource(TestConstants.COMMON_CONF_PATH); - } - - public void addcombinerConf() { - this.addResource(TestConstants.COMBINER_CONF_PATH); - } - - public void addKVTestConf() { - this.addResource(TestConstants.KVTEST_CONF_PATH); - } - - public void addNonSortTestConf() { - this.addResource(TestConstants.NONSORT_TEST_CONF); - } - - public void addNativeConf() { - this.set(TestConstants.NATIVETASK_COLLECTOR_DELEGATOR, - TestConstants.NATIVETASK_COLLECTOR_DELEGATOR_CLASS); - } - - public static Configuration getNormalConfiguration() { - Configuration normalConf = new Configuration(); - normalConf.addResource("common_conf.xml"); - normalConf.addResource("normal_conf.xml"); - return normalConf; - } - - public static Configuration getNativeConfiguration() { - Configuration nativeConf = new Configuration(); - nativeConf.addResource("common_conf.xml"); - nativeConf.addResource("native_conf.xml"); - return nativeConf; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java deleted file mode 100644 index 3beb082..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.mapred.nativetask.testutil; - -public class TestConstants { - // conf path - public static final String COMBINER_CONF_PATH = "test-combiner-conf.xml"; - public static final String KVTEST_CONF_PATH = "kvtest-conf.xml"; - public static final String NONSORT_TEST_CONF = "test-nonsort-conf.xml"; - - public static final String NATIVETASK_KVSIZE_MIN = "nativetask.kvsize.min"; - public static final String NATIVETASK_KVSIZE_MAX = "nativetask.kvsize.max"; - - public static final String NATIVETASK_KVTEST_INPUTDIR = "nativetask.kvtest.inputdir"; - public static final String NATIVETASK_KVTEST_OUTPUTDIR = "nativetask.kvtest.outputdir"; - public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = "normal.kvtest.outputdir"; - public static final String NATIVETASK_KVTEST_CREATEFILE = "nativetask.kvtest.createfile"; - public static final String NATIVETASK_KVTEST_FILE_RECORDNUM = "nativetask.kvtest.file.recordnum"; - public static final String NATIVETASK_KVTEST_KEYCLASSES = "nativetask.kvtest.keyclasses"; - public static final String NATIVETASK_KVTEST_VALUECLASSES = "nativetask.kvtest.valueclasses"; - public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class"; - public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator"; - - public static final String SNAPPY_COMPRESS_CONF_PATH = "test-snappy-compress-conf.xml"; - public static final String GZIP_COMPRESS_CONF_PATH = "test-gzip-compress-conf.xml"; - public static final String BZIP2_COMPRESS_CONF_PATH = "test-bzip2-compress-conf.xml"; - public static final String DEFAULT_COMPRESS_CONF_PATH = "test-default-compress-conf.xml"; - public static final String LZ4_COMPRESS_CONF_PATH = "test-lz4-compress-conf.xml"; - public static final String NATIVETASK_COMPRESS_FILESIZE = "nativetask.compress.filesize"; - - public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_KEY = "nativetask.combinertest.inputpath"; - public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV = "./combinertest/input"; - public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH = "nativetask.combinertest.outputdir"; - public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/native"; - public static final String NORMAL_TEST_COMBINER_OUTPUTPATH = "normal.combinertest.outputdir"; - public static final String NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/normal"; - public static final String OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH = "oldAPI.nativetask.combinertest.outputdir"; - public static final String OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH = "oldAPI.normal.combinertest.outputdir"; - public static final String NATIVETASK_COMBINER_WORDCOUNT_FILESIZE = "nativetask.combiner.wordcount.filesize"; - public static final String NATIVETASK_NONSORTTEST_FILESIZE = "nativetask.nonsorttest.filesize"; - - public static final String COMMON_CONF_PATH = "common_conf.xml"; - - public static final String FILESIZE_KEY = "kvtest.file.size"; - public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv"; - - public static final String NATIVETASK_MAP_OUTPUT_SORT = "mapreduce.sort.avoidance"; - public static final String NONSORT_TEST_INPUTDIR = "nativetask.nonsorttest.inputpath"; - public static final String NONSORT_TEST_NATIVE_OUTPUT = "nonsorttest.native.outputdir"; - public static final String NONSORT_TEST_NORMAL_OUTPUT = "nonsorttest.normal.outputdir"; - -}
