Import initial code for MAPREDUCE-2841 (native output collector) Contributed by: Binglin Chang Yang Dong Sean Zhong Manu Zhang Zhongliang Zhu Vincent Wang Yan Dong Cheng Lian Xusen Yin Fangqin Dai Jiang Weihua Gansha Wu Avik Dey
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1611413 13f79535-47bb-0310-9956-ffa450edef68 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2551c06 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2551c06 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2551c06 Branch: refs/heads/trunk Commit: b2551c06a09fb80a9e69adbc01c4c34b93ad0139 Parents: cc57df3 Author: Todd Lipcon <t...@apache.org> Authored: Thu Jul 17 17:44:55 2014 +0000 Committer: Todd Lipcon <t...@apache.org> Committed: Thu Jul 17 17:44:55 2014 +0000 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/io/Text.java | 6 +- hadoop-dist/pom.xml | 1 + .../java/org/apache/hadoop/mapred/MapTask.java | 14 +- .../hadoop-mapreduce-client-nativetask/LICENSE | 288 + .../hadoop-mapreduce-client-nativetask/pom.xml | 186 + .../sdk/example/CustomModule/README.txt | 1 + .../sdk/example/CustomModule/pom.xml | 131 + .../platform/custom/CustomPlatform.java | 39 + .../nativetask/serde/custom/CustomWritable.java | 75 + .../serde/custom/CustomWritableSerializer.java | 33 + .../src/main/native/src/CustomComparator.cpp | 88 + .../src/CMakeLists.txt | 279 + .../src/JNIFlags.cmake | 118 + .../src/config.h.cmake | 23 + .../hadoop/mapred/nativetask/Command.java | 49 + .../mapred/nativetask/CommandDispatcher.java | 38 + .../hadoop/mapred/nativetask/Constants.java | 59 + .../hadoop/mapred/nativetask/DataChannel.java | 38 + .../hadoop/mapred/nativetask/DataReceiver.java | 37 + .../mapred/nativetask/HadoopPlatform.java | 81 + .../mapred/nativetask/ICombineHandler.java | 43 + .../mapred/nativetask/INativeComparable.java | 50 + .../mapred/nativetask/INativeHandler.java | 59 + .../mapred/nativetask/NativeBatchProcessor.java | 289 + .../mapred/nativetask/NativeDataSource.java | 53 + .../mapred/nativetask/NativeDataTarget.java | 51 + .../NativeMapOutputCollectorDelegator.java | 170 + .../hadoop/mapred/nativetask/NativeRuntime.java | 216 + .../hadoop/mapred/nativetask/Platform.java | 102 + .../hadoop/mapred/nativetask/Platforms.java | 75 + .../mapred/nativetask/StatusReportChecker.java | 103 + .../hadoop/mapred/nativetask/TaskContext.java | 91 + .../mapred/nativetask/buffer/BufferType.java | 26 + .../nativetask/buffer/ByteBufferDataReader.java | 264 + .../nativetask/buffer/ByteBufferDataWriter.java | 252 + .../nativetask/buffer/DataInputStream.java | 25 + .../nativetask/buffer/DataOutputStream.java | 42 + .../nativetask/buffer/DirectBufferPool.java | 93 + .../mapred/nativetask/buffer/InputBuffer.java | 121 + .../mapred/nativetask/buffer/OutputBuffer.java | 74 + .../nativetask/handlers/BufferPullee.java | 118 + .../nativetask/handlers/BufferPuller.java | 187 + .../nativetask/handlers/BufferPushee.java | 147 + .../nativetask/handlers/BufferPusher.java | 89 + .../nativetask/handlers/CombinerHandler.java | 140 + .../mapred/nativetask/handlers/IDataLoader.java | 35 + .../handlers/NativeCollectorOnlyHandler.java | 161 + .../serde/BoolWritableSerializer.java | 33 + .../serde/ByteWritableSerializer.java | 33 + .../serde/BytesWritableSerializer.java | 45 + .../nativetask/serde/DefaultSerializer.java | 67 + .../serde/DoubleWritableSerializer.java | 33 + .../serde/FloatWritableSerializer.java | 33 + .../mapred/nativetask/serde/IKVSerializer.java | 71 + .../nativetask/serde/INativeSerializer.java | 45 + .../nativetask/serde/IntWritableSerializer.java | 33 + .../mapred/nativetask/serde/KVSerializer.java | 108 + .../serde/LongWritableSerializer.java | 32 + .../nativetask/serde/NativeSerialization.java | 86 + .../serde/NullWritableSerializer.java | 33 + .../serde/SerializationFramework.java | 32 + .../mapred/nativetask/serde/TextSerializer.java | 54 + .../serde/VIntWritableSerializer.java | 26 + .../serde/VLongWritableSerializer.java | 26 + .../mapred/nativetask/util/BytesUtil.java | 267 + .../mapred/nativetask/util/ConfigUtil.java | 43 + .../nativetask/util/LocalJobOutputFiles.java | 189 + .../nativetask/util/NativeTaskOutput.java | 137 + .../nativetask/util/NativeTaskOutputFiles.java | 199 + .../mapred/nativetask/util/OutputUtil.java | 46 + .../mapred/nativetask/util/ReadWriteBuffer.java | 150 + .../mapred/nativetask/util/SizedWritable.java | 53 + .../mapred/nativetask/util/SnappyUtil.java | 32 + .../src/main/native/COPYING | 83 + .../src/main/native/cityhash/city.cc | 307 + .../src/main/native/cityhash/city.h | 90 + .../src/main/native/gtest/gtest-all.cc | 9120 ++++++++ .../src/main/native/gtest/gtest.h | 19541 +++++++++++++++++ .../src/main/native/gtest/gtest_main.cc | 39 + .../src/main/native/lz4/lz4.c | 740 + .../src/main/native/lz4/lz4.h | 96 + .../src/main/native/src/NativeTask.h | 646 + .../src/main/native/src/codec/BlockCodec.cc | 180 + .../src/main/native/src/codec/BlockCodec.h | 98 + .../src/main/native/src/codec/GzipCodec.cc | 199 + .../src/main/native/src/codec/GzipCodec.h | 82 + .../src/main/native/src/codec/Lz4Codec.cc | 106 + .../src/main/native/src/codec/Lz4Codec.h | 45 + .../src/main/native/src/codec/SnappyCodec.cc | 93 + .../src/main/native/src/codec/SnappyCodec.h | 46 + .../src/main/native/src/codec/snappy-c.h | 138 + .../src/main/native/src/codec/snappy.h | 163 + .../native/src/handler/AbstractMapHandler.cc | 35 + .../native/src/handler/AbstractMapHandler.h | 102 + .../src/main/native/src/handler/BatchHandler.cc | 354 + .../src/main/native/src/handler/BatchHandler.h | 157 + .../main/native/src/handler/CombineHandler.cc | 256 + .../main/native/src/handler/CombineHandler.h | 90 + .../src/handler/MCollectorOutputHandler.cc | 97 + .../src/handler/MCollectorOutputHandler.h | 53 + ...oop_mapred_nativetask_NativeBatchProcessor.h | 54 + .../src/main/native/src/lib/BufferStream.cc | 155 + .../src/main/native/src/lib/BufferStream.h | 193 + .../src/main/native/src/lib/Buffers.cc | 304 + .../src/main/native/src/lib/Buffers.h | 694 + .../src/main/native/src/lib/Combiner.cc | 73 + .../src/main/native/src/lib/Combiner.h | 86 + .../src/main/native/src/lib/Compressions.cc | 136 + .../src/main/native/src/lib/Compressions.h | 108 + .../src/main/native/src/lib/Constants.h | 26 + .../src/main/native/src/lib/FileSystem.cc | 278 + .../src/main/native/src/lib/FileSystem.h | 128 + .../src/main/native/src/lib/IFile.cc | 229 + .../src/main/native/src/lib/IFile.h | 165 + .../src/main/native/src/lib/Iterator.cc | 91 + .../src/main/native/src/lib/Iterator.h | 47 + .../src/main/native/src/lib/Log.cc | 30 + .../src/main/native/src/lib/Log.h | 48 + .../main/native/src/lib/MapOutputCollector.cc | 378 + .../main/native/src/lib/MapOutputCollector.h | 163 + .../src/main/native/src/lib/MapOutputSpec.cc | 64 + .../src/main/native/src/lib/MapOutputSpec.h | 81 + .../src/main/native/src/lib/MemoryBlock.cc | 65 + .../src/main/native/src/lib/MemoryBlock.h | 168 + .../src/main/native/src/lib/MemoryPool.h | 90 + .../src/main/native/src/lib/Merge.cc | 181 + .../src/main/native/src/lib/Merge.h | 254 + .../src/main/native/src/lib/MinHeap.h | 64 + .../src/main/native/src/lib/NativeLibrary.cc | 90 + .../src/main/native/src/lib/NativeLibrary.h | 57 + .../main/native/src/lib/NativeObjectFactory.cc | 453 + .../main/native/src/lib/NativeObjectFactory.h | 93 + .../main/native/src/lib/NativeRuntimeJniImpl.cc | 226 + .../src/main/native/src/lib/NativeTask.cc | 247 + .../src/main/native/src/lib/PartitionBucket.cc | 76 + .../src/main/native/src/lib/PartitionBucket.h | 144 + .../native/src/lib/PartitionBucketIterator.cc | 109 + .../native/src/lib/PartitionBucketIterator.h | 54 + .../src/main/native/src/lib/Path.cc | 53 + .../src/main/native/src/lib/Path.h | 38 + .../src/main/native/src/lib/SpillInfo.cc | 73 + .../src/main/native/src/lib/SpillInfo.h | 106 + .../main/native/src/lib/SpillOutputService.h | 44 + .../src/main/native/src/lib/Streams.cc | 122 + .../src/main/native/src/lib/Streams.h | 221 + .../src/main/native/src/lib/TaskCounters.cc | 49 + .../src/main/native/src/lib/TaskCounters.h | 52 + .../src/main/native/src/lib/TrackingCollector.h | 50 + .../src/main/native/src/lib/commons.h | 49 + .../src/main/native/src/lib/jniutils.cc | 111 + .../src/main/native/src/lib/jniutils.h | 57 + ...che_hadoop_mapred_nativetask_NativeRuntime.h | 58 + .../src/main/native/src/lib/primitives.h | 287 + .../src/main/native/src/util/Checksum.cc | 749 + .../src/main/native/src/util/Checksum.h | 78 + .../main/native/src/util/DualPivotQuickSort.h | 137 + .../src/main/native/src/util/Hash.cc | 24 + .../src/main/native/src/util/Hash.h | 73 + .../src/main/native/src/util/Random.cc | 285 + .../src/main/native/src/util/Random.h | 140 + .../src/main/native/src/util/StringUtil.cc | 217 + .../src/main/native/src/util/StringUtil.h | 66 + .../src/main/native/src/util/SyncUtils.cc | 161 + .../src/main/native/src/util/SyncUtils.h | 299 + .../src/main/native/src/util/Timer.cc | 117 + .../src/main/native/src/util/Timer.h | 56 + .../src/main/native/src/util/WritableUtils.cc | 309 + .../src/main/native/src/util/WritableUtils.h | 124 + .../src/main/native/test.sh | 18 + .../src/main/native/test/TestCommand.cc | 38 + .../src/main/native/test/TestCompressions.cc | 284 + .../src/main/native/test/TestConfig.cc | 67 + .../src/main/native/test/TestCounter.cc | 35 + .../src/main/native/test/TestFileSystem.cc | 81 + .../src/main/native/test/TestIFile.cc | 189 + .../src/main/native/test/TestMain.cc | 103 + .../src/main/native/test/TestPrimitives.cc | 304 + .../src/main/native/test/TestSort.cc | 323 + .../src/main/native/test/lib/TestByteArray.cc | 43 + .../src/main/native/test/lib/TestByteBuffer.cc | 46 + .../lib/TestComparatorForDualPivotQuickSort.cc | 88 + .../native/test/lib/TestComparatorForStdSort.cc | 88 + .../native/test/lib/TestFixSizeContainer.cc | 52 + .../src/main/native/test/lib/TestIterator.cc | 120 + .../src/main/native/test/lib/TestKVBuffer.cc | 53 + .../native/test/lib/TestMemBlockIterator.cc | 107 + .../src/main/native/test/lib/TestMemoryBlock.cc | 107 + .../src/main/native/test/lib/TestMemoryPool.cc | 50 + .../main/native/test/lib/TestPartitionBucket.cc | 212 + .../src/main/native/test/lib/TestReadBuffer.cc | 67 + .../main/native/test/lib/TestReadWriteBuffer.cc | 72 + .../native/test/lib/TestTrackingCollector.cc | 37 + .../src/main/native/test/test_commons.cc | 344 + .../src/main/native/test/test_commons.h | 150 + .../src/main/native/test/util/TestChecksum.cc | 43 + .../src/main/native/test/util/TestHash.cc | 52 + .../src/main/native/test/util/TestProcess.cc | 33 + .../src/main/native/test/util/TestStringUtil.cc | 59 + .../src/main/native/test/util/TestSyncUtils.cc | 116 + .../main/native/test/util/TestWritableUtils.cc | 49 + .../main/native/testData/testGlibCBugSpill.out | 2 + ...org.apache.hadoop.mapred.nativetask.Platform | 14 + .../mapred/nativetask/TestTaskContext.java | 44 + .../nativetask/buffer/TestBufferPushPull.java | 278 + .../buffer/TestByteBufferReadWrite.java | 181 + .../nativetask/buffer/TestDirectBufferPool.java | 201 + .../nativetask/buffer/TestInputBuffer.java | 53 + .../nativetask/buffer/TestOutputBuffer.java | 45 + .../nativetask/handlers/TestCombineHandler.java | 76 + .../TestNativeCollectorOnlyHandler.java | 111 + .../nativetask/serde/TestKVSerializer.java | 142 + .../serde/TestNativeSerialization.java | 113 + .../mapred/nativetask/testutil/TestInput.java | 60 + .../mapred/nativetask/utils/TestBytesUtil.java | 63 + .../nativetask/utils/TestReadWriteBuffer.java | 62 + .../nativetask/utils/TestSizedWritable.java | 34 + .../test/java/system/data/testGlibcBugSpill.out | 2 + .../nativetask/combinertest/CombinerTest.java | 122 + .../combinertest/LargeKVCombinerTest.java | 96 + .../combinertest/OldAPICombinerTest.java | 107 + .../nativetask/combinertest/WordCount.java | 89 + .../combinertest/WordCountWithOldAPI.java | 65 + .../nativetask/compresstest/CompressMapper.java | 69 + .../nativetask/compresstest/CompressTest.java | 134 + .../nativetask/kvtest/HashSumReducer.java | 48 + .../hadoop/mapred/nativetask/kvtest/KVJob.java | 97 + .../hadoop/mapred/nativetask/kvtest/KVTest.java | 181 + .../mapred/nativetask/kvtest/LargeKVTest.java | 129 + .../mapred/nativetask/kvtest/TestInputFile.java | 213 + .../nativetask/nonsorttest/NonSortTest.java | 99 + .../nativetask/nonsorttest/NonSortTestMR.java | 71 + .../nativetask/testutil/BytesFactory.java | 101 + .../EnforceNativeOutputCollectorDelegator.java | 48 + .../nativetask/testutil/MockValueClass.java | 72 + .../nativetask/testutil/ResultVerifier.java | 141 + .../testutil/ScenarioConfiguration.java | 58 + .../nativetask/testutil/TestConstants.java | 67 + .../src/test/resources/common_conf.xml | 39 + .../src/test/resources/kvtest-conf.xml | 87 + .../src/test/resources/native_conf.xml | 15 + .../src/test/resources/normal_conf.xml | 10 + .../test/resources/test-bzip2-compress-conf.xml | 39 + .../src/test/resources/test-combiner-conf.xml | 36 + .../resources/test-default-compress-conf.xml | 39 + .../test/resources/test-gzip-compress-conf.xml | 39 + .../test/resources/test-lz4-compress-conf.xml | 39 + .../src/test/resources/test-nonsort-conf.xml | 31 + .../resources/test-snappy-compress-conf.xml | 41 + .../hadoop-mapreduce-client/pom.xml | 1 + 249 files changed, 56959 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java index e4490f1..e4e72a4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java @@ -226,6 +226,10 @@ public class Text extends BinaryComparable this.length = len; } + public void setLength(int len) { + this.length = len; + } + /** * Append a range of bytes to the end of the given text * @param utf8 the data to copy from @@ -260,7 +264,7 @@ public class Text extends BinaryComparable * @param len the number of bytes we need * @param keepData should the old data be kept */ - private void setCapacity(int len, boolean keepData) { + public void setCapacity(int len, boolean keepData) { if (bytes == null || bytes.length < len) { if (bytes != null && keepData) { bytes = Arrays.copyOf(bytes, Math.max(len,length << 1)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-dist/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 149f404..62a27f4 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -122,6 +122,7 @@ run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${project.version}/* . run cp -r $ROOT/hadoop-yarn-project/target/hadoop-yarn-project-${project.version}/* . run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* . + run cp -r $ROOT/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/target/hadoop-mapreduce-client-nativetask-${project.version}/* . run cp -r $ROOT/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${project.version}/* . echo echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}" http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java index 84fdd92..c4c48e1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java @@ -385,11 +385,19 @@ public class MapTask extends Task { ReflectionUtils.newInstance( job.getClass(JobContext.MAP_OUTPUT_COLLECTOR_CLASS_ATTR, MapOutputBuffer.class, MapOutputCollector.class), job); - LOG.info("Map output collector class = " + collector.getClass().getName()); MapOutputCollector.Context context = new MapOutputCollector.Context(this, job, reporter); - collector.init(context); - return collector; + + try { + collector.init(context); + } catch (Exception e) { + LOG.warn("Nativetask falling back to Java MapOutputCollector", e); + collector = new MapOutputBuffer(); + collector.init(context); + } finally { + LOG.info("Map output collector class = " + collector.getClass().getName()); + return collector; + } } @SuppressWarnings("unchecked") http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE new file mode 100644 index 0000000..612fdf9 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/LICENSE @@ -0,0 +1,288 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +Licenses for third party projects used by this project: + +CityHash src/main/native/cityhash +--------------------------------------------------------------------- +// Copyright (c) 2011 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +GoogleTest src/main/native/gtest +--------------------------------------------------------------------- +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +LZ4 src/main/native/lz4 +--------------------------------------------------------------------- + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011, Yann Collet. + BSD License + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml new file mode 100644 index 0000000..6f46fd5 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml @@ -0,0 +1,186 @@ +<?xml version="1.0"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>hadoop-mapreduce-client</artifactId> + <groupId>org.apache.hadoop</groupId> + <version>3.0.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-nativetask</artifactId> + <version>3.0.0-SNAPSHOT</version> + <name>hadoop-mapreduce-client-nativetask</name> + + <properties> + <!-- Needed for generating FindBugs warnings using parent pom --> + <mr.basedir>${project.parent.basedir}/../</mr.basedir> + </properties> + + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <properties> + <property> + <name>listener</name> + <value>org.apache.hadoop.test.TimedOutTestsListener</value> + </property> + </properties> + <includes> + <include>**/TestTaskContext.java</include> + <include>**/buffer/Test*.java</include> + <include>**/handlers/Test*.java</include> + <include>**/serde/Test*.java</include> + <include>**/combinertest/*Test.java</include> + <include>**/compresstest/*Test.java</include> + <include>**/nonsorttest/*Test.java</include> + <include>**/kvtest/*Test.java</include> + </includes> + <additionalClasspathElements> + <additionalClasspathElement>${basedir}/../hadoop-mapreduce-client-common/target/classes</additionalClasspathElement> + </additionalClasspathElements> + </configuration> + </plugin> + </plugins> + </build> + + <profiles> + <profile> + <id>native</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <properties> + <snappy.prefix></snappy.prefix> + <snappy.lib></snappy.lib> + <snappy.include></snappy.include> + <require.snappy>false</require.snappy> + </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>enforce-os</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireOS> + <family>mac</family> + <family>unix</family> + <message>native build only supported on Mac or + Unix</message> + </requireOS> + </rules> + <fail>true</fail> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>native-maven-plugin</artifactId> + <executions> + <execution> + <phase>compile</phase> + <goals> + <goal>javah</goal> + </goals> + <configuration> + <javahPath>${env.JAVA_HOME}/bin/javah</javahPath> + <javahClassNames> + <javaClassName>org.apache.hadoop.mapred.nativetask.NativeBatchProcessor</javaClassName> + <javaClassName>org.apache.hadoop.mapred.nativetask.NativeRuntime</javaClassName> + </javahClassNames> + <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>make</id> + <phase>compile</phase> + <goals><goal>run</goal></goals> + <configuration> + <target> + <copy todir="${project.build.directory}/native/test/testData" overwrite="true"> + <fileset dir="${basedir}/src/main/native/testData" /> + </copy> + <copy file="${basedir}/src/main/native/test.sh" todir="${project.build.directory}/native/test" /> + <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true"> + <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/> + </exec> + <exec executable="make" dir="${project.build.directory}/native" failonerror="true"> + <arg line="VERBOSE=1"/> + </exec> + <!-- The second make is a workaround for HADOOP-9215. It can + be removed when version 2.6 of cmake is no longer supported . --> + <exec executable="make" dir="${project.build.directory}/native" failonerror="true"></exec> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt new file mode 100644 index 0000000..0ad6f1e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/README.txt @@ -0,0 +1 @@ +This project is depend on hadoop and hadoop-nativetask, so in order to complete the build, you have to specify the hadoop dir first. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml new file mode 100644 index 0000000..5bf67c7 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/pom.xml @@ -0,0 +1,131 @@ +<?xml version="1.0"?> +<!-- Licensed under the Apache License, Version 2.0 (the "License"); you + may not use this file except in compliance with the License. You may obtain + a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless + required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + See accompanying LICENSE file. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.hadoop</groupId> + <artifactId>nativetask-sdk</artifactId> + <version>2.2.0</version> + <name>nativetask-sdk</name> + + <properties> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-nativetask</artifactId> + <version>2.2.0</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + </configuration> + </plugin> + </plugins> + </build> + + <profiles> + <profile> + <id>native</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>enforce-os</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireOS> + <family>mac</family> + <family>unix</family> + <message>native build only supported on Mac or + Unix</message> + </requireOS> + </rules> + <fail>true</fail> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>make</id> + <phase>compile</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <exec executable="sh" dir="${basedir}/src/main/native" + failonerror="true"> + <arg value="${basedir}/src/main/native/build.sh" /> + <arg value="all" /> + </exec> + <copy toDir="${project.build.directory}/native/target/usr/local/lib" overwrite="true"> + <fileset dir="${basedir}/src/main/native/.libs"> + <exclude name="**/*.o"/> + <exclude name="nttest"/> + </fileset> + </copy> + <copy todir="${project.build.directory}/${project.artifactId}-${project.version}/lib/native/sdk/" overwrite="true"> + <fileset dir="${basedir}/src/main/native/.libs"> + <exclude name="**/*.o"/> + <exclude name="nttest"/> + </fileset> + <fileset file="${basedir}/src/main/native/src/NativeTask.h" /> + </copy> + </target> + </configuration> + </execution> + <execution> + <id>make-clean</id> + <phase>clean</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <exec executable="sh" dir="${basedir}/src/main/native" + failonerror="true"> + <arg value="${basedir}/src/main/native/build.sh" /> + <arg value="clean" /> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/platform/custom/CustomPlatform.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/platform/custom/CustomPlatform.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/platform/custom/CustomPlatform.java new file mode 100644 index 0000000..1d67f43 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/platform/custom/CustomPlatform.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.nativetask.platform.custom; + +import java.io.IOException; + +import org.apache.hadoop.mapred.nativetask.Platform; +import org.apache.hadoop.nativetask.serde.custom.CustomWritable; +import org.apache.hadoop.nativetask.serde.custom.CustomWritableSerializer; + +public class CustomPlatform extends Platform{ + + @Override + public void init() throws IOException { + registerKey(CustomWritable.class.getName(), + CustomWritableSerializer.class); + } + + @Override + public String name() { + return "CustomPlatform"; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritable.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritable.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritable.java new file mode 100644 index 0000000..28162f9 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritable.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.nativetask.serde.custom; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.WritableComparable; + +public class CustomWritable implements WritableComparable<CustomWritable>{ + + private int Id_a; + private long Id_b; + + public CustomWritable() { + this.Id_a = 0; + this.Id_b = 0; + } + + public CustomWritable(int a, long b){ + this.Id_a = a; + this.Id_b = b; + } + + @Override + public void readFields(DataInput in) throws IOException { + Id_a = in.readInt(); + Id_b = in.readLong(); + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(Id_a); + out.writeLong(Id_b); + } + + @Override + public int compareTo(CustomWritable that) { + if(Id_a > that.Id_a){ + return 1; + } + if(Id_a < that.Id_a){ + return -1; + } + if(Id_b > that.Id_b){ + return 1; + } + if(Id_b < that.Id_b){ + return -1; + } + return 0; + } + + @Override + public String toString() { + return Id_a + "\t" + Id_b; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritableSerializer.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritableSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritableSerializer.java new file mode 100644 index 0000000..4424a2e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/java/org/apache/hadoop/nativetask/serde/custom/CustomWritableSerializer.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.nativetask.serde.custom; + +import java.io.IOException; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.nativetask.INativeComparable; +import org.apache.hadoop.mapred.nativetask.serde.DefaultSerializer; + +public class CustomWritableSerializer extends DefaultSerializer implements + INativeComparable{ + + @Override + public int getLength(Writable w) throws IOException { + return 12; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/native/src/CustomComparator.cpp ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/native/src/CustomComparator.cpp b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/native/src/CustomComparator.cpp new file mode 100644 index 0000000..c8aa21d --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/sdk/example/CustomModule/src/main/native/src/CustomComparator.cpp @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <iostream> +#include "NativeTask.h" + +using namespace NativeTask; + +namespace Custom { +using namespace std; + +inline uint32_t bswap(uint32_t val) { + __asm__("bswap %0" : "=r" (val) : "0" (val)); + return val; +} + +int32_t ReadInt(const char * src) { + return (int32_t) bswap(*(uint32_t*) src); +} + +inline uint64_t bswap64(uint64_t val) { +#ifdef __X64 + __asm__("bswapq %0" : "=r" (val) : "0" (val)); +#else + + uint64_t lower = val & 0xffffffffU; + uint32_t higher = (val >> 32) & 0xffffffffU; + + lower = bswap(lower); + higher = bswap(higher); + + return (lower << 32) + higher; + +#endif + return val; +} + +int64_t ReadLong(const char * src) { + return (int64_t) bswap64(*(uint64_t*) src); +} + +int CustomComparator(const char * src, uint32_t srcLength, const char * dest, + uint32_t destLength) { + int32_t src_IDa = ReadInt(src); + int64_t src_IDb = ReadLong(src+4); + int32_t dest_IDa = ReadInt(dest); + int64_t dest_IDb = ReadLong(dest+4); + if(src_IDa > dest_IDa){ + return 1; + } + if(src_IDa < dest_IDa){ + return -1; + } + if(src_IDb > dest_IDb){ + return 1; + } + if(src_IDb < dest_IDb){ + return -1; + } + return 0; +}; + +DEFINE_NATIVE_LIBRARY(Custom) { + REGISTER_FUNCTION(CustomComparator,Custom); +} + +} + + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt new file mode 100644 index 0000000..0247a54 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt @@ -0,0 +1,279 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) + +# Default to release builds +set(CMAKE_BUILD_TYPE, Release) + +include(JNIFlags.cmake NO_POLICY_SCOPE) + +# Compile a library with both shared and static variants +function(add_dual_library LIBNAME) + add_library(${LIBNAME} SHARED ${ARGN}) + add_library(${LIBNAME}_static STATIC ${ARGN}) + set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME}) +endfunction(add_dual_library) + +# Link both a static and a dynamic target against some libraries +function(target_link_dual_libraries LIBNAME) + target_link_libraries(${LIBNAME} ${ARGN}) + target_link_libraries(${LIBNAME}_static ${ARGN}) +endfunction(target_link_dual_libraries) + +function(output_directory TGT DIR) + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") +endfunction(output_directory TGT DIR) + +function(dual_output_directory TGT DIR) + output_directory(${TGT} "${DIR}") + output_directory(${TGT}_static "${DIR}") +endfunction(dual_output_directory TGT DIR) + +# +# This macro alters the behavior of find_package and find_library. +# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable. +# You should save that variable before calling this function and restore it +# after you have accomplished your goal. +# +# The behavior is altered in two ways: +# 1. We always find shared libraries, never static; +# 2. We find shared libraries with the given version number. +# +# On Windows this function is a no-op. Windows does not encode +# version number information information into library path names. +# +macro(set_find_shared_library_version LVERS) + IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + # Mac OS uses .dylib + SET(CMAKE_FIND_LIBRARY_SUFFIXES ".${LVERS}.dylib") + ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") + # FreeBSD has always .so installed. + SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so") + ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + # Windows doesn't support finding shared libraries by version. + ELSE() + # Most UNIX variants use .so + SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${LVERS}") + ENDIF() +endmacro(set_find_shared_library_version LVERS) + +if (NOT GENERATED_JAVAH) + #Must identify where the generated headers have been placed + MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH") +endif (NOT GENERATED_JAVAH) +find_package(JNI REQUIRED) + +SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES) +set_find_shared_library_version("1") +SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES) + +# primitive configs +set(PRFLAGS "-DSIMPLE_MEMCPY") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -Wall") +set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0 + -L${_JAVA_HOME}/jre/lib/amd64/server -ljvm") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -g -O2 -DNDEBUG -fPIC") +set(D main/native/) + +SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES) +set_find_shared_library_version("1") +SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES) + +INCLUDE(CheckFunctionExists) +INCLUDE(CheckCSourceCompiles) +#INCLUDE(CheckLibraryExists) +INCLUDE(CheckIncludeFiles) +#CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE) +#CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE) +#CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL) +CHECK_INCLUDE_FILES(fcntl.h HAVE_FCNTL_H) +CHECK_INCLUDE_FILES(malloc.h HAVE_MALLOC_H) +CHECK_INCLUDE_FILES(mach/mach.h HAVE_MACH_MACH_H) +CHECK_INCLUDE_FILES(memory.h HAVE_MEMORY_H) +CHECK_INCLUDE_FILES(stddef.h HAVE_STDDEF_H) +CHECK_INCLUDE_FILES(stdint.h HAVE_STDINT_H) +CHECK_INCLUDE_FILES(stdlib.h HAVE_STDLIB_H) +CHECK_INCLUDE_FILES(string.h HAVE_STRING_H) +CHECK_INCLUDE_FILES(unistd.h HAVE_UNITSTD_H) +CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME) +CHECK_FUNCTION_EXISTS(localtime_r HAVE_LOCALTIME_R) +CHECK_FUNCTION_EXISTS(memset HAVE_MEMSET) +CHECK_FUNCTION_EXISTS(strchr HAVE_STRCHR) +CHECK_FUNCTION_EXISTS(strtoul HAVE_STRTOUL) + + +SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES) +set_find_shared_library_version("1") +find_library(SNAPPY_LIBRARY + NAMES snappy + PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib + ${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB}) +SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES) +find_path(SNAPPY_INCLUDE_DIR + NAMES snappy.h + PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include + ${CUSTOM_SNAPPY_INCLUDE}) +if (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR) + GET_FILENAME_COMPONENT(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME) + set(SNAPPY_SOURCE_FILES + "${D}/src/codec/SnappyCodec.cc") +else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR) + set(SNAPPY_INCLUDE_DIR "") + set(SNAPPY_SOURCE_FILES "") + IF(REQUIRE_SNAPPY) + MESSAGE(FATAL_ERROR "Required snappy library could not be found. SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}") + ENDIF(REQUIRE_SNAPPY) +endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR) + +include_directories( + ${GENERATED_JAVAH} + ${D} + ${D}/src + ${D}/src/util + ${D}/src/lib + ${D}/test + ${CMAKE_CURRENT_SOURCE_DIR} + #${CMAKE_CURRENT_SOURCE_DIR}/src + #${CMAKE_BINARY_DIR} + ${JNI_INCLUDE_DIRS} + ${SNAPPY_INCLUDE_DIR} +) + +#SET(CMAKE_SOURCE_DIR "/cygdrive/c/Users/tianlunz/repo/hadoop-2.2.0-src/hadoop-common-project/hadoop-common/src") +CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) + + +SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) + +add_dual_library(nativetask + ${D}/lz4/lz4.c + ${D}/cityhash/city.cc + ${D}/src/codec/BlockCodec.cc + ${D}/src/codec/GzipCodec.cc + ${D}/src/codec/Lz4Codec.cc + ${SNAPPY_SOURCE_FILES} + ${D}/src/handler/BatchHandler.cc + ${D}/src/handler/MCollectorOutputHandler.cc + ${D}/src/handler/AbstractMapHandler.cc + ${D}/src/handler/CombineHandler.cc + ${D}/src/lib/Buffers.cc + ${D}/src/lib/BufferStream.cc + ${D}/src/lib/Compressions.cc + ${D}/src/lib/PartitionBucket.cc + ${D}/src/lib/PartitionBucketIterator.cc + ${D}/src/lib/FileSystem.cc + ${D}/src/lib/IFile.cc + ${D}/src/lib/jniutils.cc + ${D}/src/lib/Log.cc + ${D}/src/lib/MapOutputCollector.cc + ${D}/src/lib/MapOutputSpec.cc + ${D}/src/lib/MemoryBlock.cc + ${D}/src/lib/Merge.cc + ${D}/src/lib/NativeLibrary.cc + ${D}/src/lib/Iterator.cc + ${D}/src/lib/NativeObjectFactory.cc + ${D}/src/lib/NativeRuntimeJniImpl.cc + ${D}/src/lib/NativeTask.cc + ${D}/src/lib/SpillInfo.cc + ${D}/src/lib/Path.cc + ${D}/src/lib/Streams.cc + ${D}/src/lib/Combiner.cc + ${D}/src/lib/TaskCounters.cc + ${D}/src/util/Checksum.cc + ${D}/src/util/Hash.cc + ${D}/src/util/Random.cc + ${D}/src/util/StringUtil.cc + ${D}/src/util/SyncUtils.cc + ${D}/src/util/Timer.cc + ${D}/src/util/WritableUtils.cc +) +target_link_libraries(nativetask + #${LIB_DL} + dl + rt + pthread + z + ${SNAPPY_LIBRARY} + ${JAVA_JVM_LIBRARY} +) + +add_executable(nttest + ${D}/gtest/gtest-all.cc + ${D}/test/lib/TestByteArray.cc + ${D}/test/lib/TestByteBuffer.cc + ${D}/test/lib/TestComparatorForDualPivotQuickSort.cc + ${D}/test/lib/TestComparatorForStdSort.cc + ${D}/test/lib/TestFixSizeContainer.cc + ${D}/test/lib/TestMemoryPool.cc + ${D}/test/lib/TestIterator.cc + ${D}/test/lib/TestKVBuffer.cc + ${D}/test/lib/TestMemBlockIterator.cc + ${D}/test/lib/TestMemoryBlock.cc + ${D}/test/lib/TestPartitionBucket.cc + ${D}/test/lib/TestReadBuffer.cc + ${D}/test/lib/TestReadWriteBuffer.cc + ${D}/test/lib/TestTrackingCollector.cc + ${D}/test/util/TestChecksum.cc + ${D}/test/util/TestHash.cc + ${D}/test/util/TestStringUtil.cc + ${D}/test/util/TestSyncUtils.cc + ${D}/test/util/TestWritableUtils.cc + ${D}/test/TestCommand.cc + ${D}/test/TestConfig.cc + ${D}/test/TestCounter.cc + ${D}/test/TestCompressions.cc + ${D}/test/TestFileSystem.cc + ${D}/test/TestIFile.cc + ${D}/test/TestPrimitives.cc + ${D}/test/TestSort.cc + ${D}/test/TestMain.cc + ${D}/test/test_commons.cc) +set(CMAKE_EXE_LINKER_FLAGS "-L${_JAVA_HOME}/jre/lib/amd64/server -ljvm") +target_link_libraries(nttest + nativetask_static + dl + rt + pthread + z + ${SNAPPY_LIBRARY} + # ${JAVA_JVM_LIBRARY} +) +#if (NEED_LINK_DL) +# set(LIB_DL dl) +#endif (NEED_LINK_DL) + +IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + # + # By embedding '$ORIGIN' into the RPATH of libnativetask.so, + # dlopen will look in the directory containing libnativetask.so. + # However, $ORIGIN is not supported by all operating systems. + # + SET_TARGET_PROPERTIES(nativetask + PROPERTIES INSTALL_RPATH "\$ORIGIN/") +ENDIF() + +SET(LIBNATIVETASK_VERSION "1.0.0") +SET_TARGET_PROPERTIES(nativetask PROPERTIES SOVERSION ${LIBNATIVETASK_VERSION}) +dual_output_directory(nativetask target/usr/local/lib) +output_directory(nttest test) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake new file mode 100644 index 0000000..0100b06 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake @@ -0,0 +1,118 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) + +# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit. +# This variable is set by maven. +if (JVM_ARCH_DATA_MODEL EQUAL 32) + # Force 32-bit code generation on amd64/x86_64, ppc64, sparc64 + if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_SYSTEM_PROCESSOR MATCHES ".*64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") + set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32") + endif () + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") + # Set CMAKE_SYSTEM_PROCESSOR to ensure that find_package(JNI) will use + # the 32-bit version of libjvm.so. + set(CMAKE_SYSTEM_PROCESSOR "i686") + endif () +endif (JVM_ARCH_DATA_MODEL EQUAL 32) + +# Determine float ABI of JVM on ARM Linux +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + find_program(READELF readelf) + if (READELF MATCHES "NOTFOUND") + message(WARNING "readelf not found; JVM float ABI detection disabled") + else (READELF MATCHES "NOTFOUND") + execute_process( + COMMAND ${READELF} -A ${JAVA_JVM_LIBRARY} + OUTPUT_VARIABLE JVM_ELF_ARCH + ERROR_QUIET) + if (NOT JVM_ELF_ARCH MATCHES "Tag_ABI_VFP_args: VFP registers") + message("Soft-float JVM detected") + + # Test compilation with -mfloat-abi=softfp using an arbitrary libc function + # (typically fails with "fatal error: bits/predefs.h: No such file or directory" + # if soft-float dev libraries are not installed) + include(CMakePushCheckState) + cmake_push_check_state() + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -mfloat-abi=softfp") + include(CheckSymbolExists) + check_symbol_exists(exit stdlib.h SOFTFP_AVAILABLE) + if (NOT SOFTFP_AVAILABLE) + message(FATAL_ERROR "Soft-float dev libraries required (e.g. 'apt-get install libc6-dev-armel' on Debian/Ubuntu)") + endif (NOT SOFTFP_AVAILABLE) + cmake_pop_check_state() + + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfloat-abi=softfp") + endif () + endif (READELF MATCHES "NOTFOUND") +endif (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + +IF("${CMAKE_SYSTEM}" MATCHES "Linux") + # + # Locate JNI_INCLUDE_DIRS and JNI_LIBRARIES. + # Since we were invoked from Maven, we know that the JAVA_HOME environment + # variable is valid. So we ignore system paths here and just use JAVA_HOME. + # + FILE(TO_CMAKE_PATH "$ENV{JAVA_HOME}" _JAVA_HOME) + IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$") + SET(_java_libarch "i386") + ELSEIF (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") + SET(_java_libarch "amd64") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") + SET(_java_libarch "arm") + ELSE() + SET(_java_libarch ${CMAKE_SYSTEM_PROCESSOR}) + ENDIF() + SET(_JDK_DIRS "${_JAVA_HOME}/jre/lib/${_java_libarch}/*" + "${_JAVA_HOME}/jre/lib/${_java_libarch}" + "${_JAVA_HOME}/jre/lib/*" + "${_JAVA_HOME}/jre/lib" + "${_JAVA_HOME}/lib/*" + "${_JAVA_HOME}/lib" + "${_JAVA_HOME}/include/*" + "${_JAVA_HOME}/include" + "${_JAVA_HOME}" + ) + FIND_PATH(JAVA_INCLUDE_PATH + NAMES jni.h + PATHS ${_JDK_DIRS} + NO_DEFAULT_PATH) + #In IBM java, it's jniport.h instead of jni_md.h + FIND_PATH(JAVA_INCLUDE_PATH2 + NAMES jni_md.h jniport.h + PATHS ${_JDK_DIRS} + NO_DEFAULT_PATH) + SET(JNI_INCLUDE_DIRS ${JAVA_INCLUDE_PATH} ${JAVA_INCLUDE_PATH2}) + FIND_LIBRARY(JAVA_JVM_LIBRARY + NAMES rt jvm + PATHS ${_JDK_DIRS} + NO_DEFAULT_PATH) + SET(JNI_LIBRARIES ${JAVA_JVM_LIBRARY}) + MESSAGE("JAVA_HOME=${JAVA_HOME}, JAVA_JVM_LIBRARY=${JAVA_JVM_LIBRARY}") + MESSAGE("JAVA_INCLUDE_PATH=${JAVA_INCLUDE_PATH}, JAVA_INCLUDE_PATH2=${JAVA_INCLUDE_PATH2}") + IF(JAVA_JVM_LIBRARY AND JAVA_INCLUDE_PATH AND JAVA_INCLUDE_PATH2) + MESSAGE("Located all JNI components successfully.") + ELSE() + MESSAGE(FATAL_ERROR "Failed to find a viable JVM installation under JAVA_HOME.") + ENDIF() +ELSE() + find_package(JNI REQUIRED) +ENDIF() http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/config.h.cmake ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/config.h.cmake b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/config.h.cmake new file mode 100644 index 0000000..e69dc43 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/config.h.cmake @@ -0,0 +1,23 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#ifndef CONFIG_H +#define CONFIG_H + +#cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@" + +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Command.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Command.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Command.java new file mode 100644 index 0000000..80a5658 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Command.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred.nativetask; + +public class Command { + + private int id; + private String description; + + public Command(int id) { + this.id = id; + } + + public Command(int id, String description) { + this.id = id; + this.description = description; + } + + public int id() { + return this.id; + } + + public String description() { + return this.description; + } + + @Override + public boolean equals(Object other) { + if (other instanceof Command) { + return this.id == ((Command)other).id; + } + return false; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/CommandDispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/CommandDispatcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/CommandDispatcher.java new file mode 100644 index 0000000..52af730 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/CommandDispatcher.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred.nativetask; + +import java.io.IOException; +import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer; + +/** + * a CommandDispatcher receives {@link Command} from upstream + * and performs corresponding operations + */ +public interface CommandDispatcher { + + /** + * + * @param command + * @param parameter + * @return + * @throws IOException + */ + public ReadWriteBuffer onCall(Command command, ReadWriteBuffer parameter) throws IOException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java new file mode 100644 index 0000000..e71326f --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Constants.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred.nativetask; + +public class Constants { + + public static final String MAP_SORT_CLASS = "map.sort.class"; + public static final String MAPRED_COMBINER_CLASS = "mapred.combiner.class"; + + public static final String MAPRED_MAPTASK_DELEGATOR_CLASS = "mapreduce.map.task.delegator.class"; + public static final String MAPRED_REDUCETASK_DELEGATOR_CLASS = "mapreduce.reduce.task.delegator.class"; + public static final String NATIVE_TASK_ENABLED = "native.task.enabled"; + public static final String NATIVE_LOG_DEVICE = "native.log.device"; + public static final String NATIVE_HADOOP_VERSION = "native.hadoop.version"; + + public static final String NATIVE_MAPPER_CLASS = "native.mapper.class"; + public static final String NATIVE_REDUCER_CLASS = "native.reducer.class"; + public static final String NATIVE_PARTITIONER_CLASS = "native.partitioner.class"; + public static final String NATIVE_COMBINER_CLASS = "native.combiner.class"; + public static final String NATIVE_INPUT_SPLIT = "native.input.split"; + + public static final String NATIVE_RECORDREADER_CLASS = "native.recordreader.class"; + public static final String NATIVE_RECORDWRITER_CLASS = "native.recordwriter.class"; + public static final String NATIVE_OUTPUT_FILE_NAME = "native.output.file.name"; + + public static final String NATIVE_PROCESSOR_BUFFER_KB = "native.processor.buffer.kb"; + public static int NATIVE_PROCESSOR_BUFFER_KB_DEFAULT = 64; + public static int NATIVE_ASYNC_PROCESSOR_BUFFER_KB_DEFAULT = 1024; + + public static final String NATIVE_STATUS_UPDATE_INTERVAL = "native.update.interval"; + public static int NATIVE_STATUS_UPDATE_INTERVAL_DEFVAL = 3000; + + public static final String SERIALIZATION_FRAMEWORK = "SerializationFramework"; + public static int SIZEOF_PARTITION_LENGTH = 4; + public static int SIZEOF_KEY_LENGTH = 4; + public static int SIZEOF_VALUE_LENGTH = 4; + public static int SIZEOF_KV_LENGTH = SIZEOF_KEY_LENGTH + SIZEOF_VALUE_LENGTH; + + public static final String NATIVE_CLASS_LIBRARY = "native.class.library"; + public static final String NATIVE_CLASS_LIBRARY_CUSTOM = "native.class.library.custom"; + public static final String NATIVE_CLASS_LIBRARY_BUILDIN = "native.class.library.buildin"; + public static final String NATIVE_MAPOUT_KEY_COMPARATOR = "native.map.output.key.comparator"; +}
