http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h index 60bb6f5..ba026f5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/NativeTask.h @@ -38,13 +38,6 @@ using std::pair; enum NativeObjectType { UnknownObjectType = 0, BatchHandlerType = 1, - MapperType = 2, - ReducerType = 3, - PartitionerType = 4, - CombinerType = 5, - FolderType = 6, - RecordReaderType = 7, - RecordWriterType = 8 }; /** @@ -69,7 +62,6 @@ enum Endium { #define INPUT_LINE_KV_SEPERATOR "mapreduce.input.keyvaluelinerecordreader.key.value.separator" #define MAPRED_TEXTOUTPUT_FORMAT_SEPERATOR "mapreduce.output.textoutputformat.separator" #define MAPRED_WORK_OUT_DIR "mapreduce.task.output.dir" -#define NATIVE_OUTPUT_FILE_NAME "native.output.file.name" #define MAPRED_COMPRESS_OUTPUT "mapreduce.output.fileoutputformat.compress" #define MAPRED_OUTPUT_COMPRESSION_CODEC "mapreduce.output.fileoutputformat.compress.codec" #define TOTAL_ORDER_PARTITIONER_PATH "total.order.partitioner.path" @@ -386,32 +378,6 @@ public: virtual bool next(Buffer & key, Buffer & value) = 0; }; -class RecordReader : public KVIterator, public Configurable, public Progress { -public: - virtual NativeObjectType type() { - return RecordReaderType; - } - - virtual bool next(Buffer & key, Buffer & value) = 0; - - virtual float getProgress() = 0; - - virtual void close() = 0; -}; - -class RecordWriter : public Collector, public Configurable { -public: - virtual NativeObjectType type() { - return RecordWriterType; - } - - virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen) { - } - - virtual void close() { - } - -}; class ProcessorBase : public Configurable { protected: @@ -444,36 +410,6 @@ public: } }; -class Mapper : public ProcessorBase { -public: - virtual NativeObjectType type() { - return MapperType; - } - - /** - * Map interface, default IdenticalMapper - */ - virtual void map(const char * key, uint32_t keyLen, const char * value, uint32_t valueLen) { - collect(key, keyLen, value, valueLen); - } -}; - -class Partitioner : public Configurable { -public: - virtual NativeObjectType type() { - return PartitionerType; - } - - /** - * Partition interface - * @param key key buffer - * @param keyLen key length, can be modified to smaller value - * to truncate key - * @return partition number - */ - virtual uint32_t getPartition(const char * key, uint32_t & keyLen, uint32_t numPartition); -}; - enum KeyGroupIterState { SAME_KEY, NEW_KEY, @@ -502,80 +438,7 @@ public: virtual const char * nextValue(uint32_t & len) = 0; }; -class Reducer : public ProcessorBase { -public: - virtual NativeObjectType type() { - return ReducerType; - } - - /** - * Reduce interface, default IdenticalReducer - */ - virtual void reduce(KeyGroupIterator & input) { - const char * key; - const char * value; - uint32_t keyLen; - uint32_t valueLen; - key = input.getKey(keyLen); - while (NULL != (value = input.nextValue(valueLen))) { - collect(key, keyLen, value, valueLen); - } - } -}; - -/** - * Folder API used for hashtable based aggregation - * Folder will be used in this way: - * on(key, value): - * state = hashtable.get(key) - * if state == None: - * size = size() - * if size == -1: - * state = init(null, -1) - * elif size > 0: - * state = fixallocator.get(key) - * init(state, size) - * folder(state, value, value.len) - * - * final(): - * for k,state in hashtable: - * final(key, key.len, state) - */ -class Folder : public ProcessorBase { -public: - virtual NativeObjectType type() { - return FolderType; - } - - /** - * Get aggregator state size - * @return state storage size - * -1 size not fixed or unknown, default - * e.g. list map tree - * 0 don't need to store state - * >0 fixed sized state - * e.g. int32 int64 float. - */ - virtual int32_t size() { - return -1; - } - /** - * Create and/or init new state - */ - virtual void * init(const char * key, uint32_t keyLen) { - return NULL; - } - - /** - * Aggregation function - */ - virtual void folder(void * dest, const char * value, uint32_t valueLen) { - } - - virtual void final(const char * key, uint32_t keyLen, void * dest) { - } -}; enum KeyValueType { TextType = 0,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc index 59cd8fb..ce36239 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "NativeTask.h" #include "BlockCodec.h" @@ -79,9 +79,8 @@ BlockDecompressStream::BlockDecompressStream(InputStream * stream, uint32_t buff } void BlockDecompressStream::init() { - _tempBufferSize = maxCompressedLength(_blockMax) + 8; + _tempBufferSize = maxCompressedLength(_blockMax) + 8; _tempBuffer = (char*)malloc(_tempBufferSize); - } BlockDecompressStream::~BlockDecompressStream() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h index 0412bb0..ffa4417 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/BlockCodec.h @@ -19,7 +19,7 @@ #ifndef BLOCKCODEC_H_ #define BLOCKCODEC_H_ -#include "Compressions.h" +#include "lib/Compressions.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc index bb699e9..b0b9852 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.cc @@ -18,7 +18,7 @@ #include <zconf.h> #include <zlib.h> -#include "commons.h" +#include "lib/commons.h" #include "GzipCodec.h" #include <iostream> http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h index 96721e5..a5f986e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/GzipCodec.h @@ -19,7 +19,7 @@ #ifndef GZIPCODEC_H_ #define GZIPCODEC_H_ -#include "Compressions.h" +#include "lib/Compressions.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc index 8dc4c64..75bfad1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "lz4.h" #include "NativeTask.h" #include "Lz4Codec.h" http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h index d300dda..f3dd168 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/Lz4Codec.h @@ -19,7 +19,7 @@ #ifndef LZ4CODEC_H_ #define LZ4CODEC_H_ -#include "Compressions.h" +#include "lib/Compressions.h" #include "BlockCodec.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc index 0504033..a0417e0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc @@ -19,7 +19,7 @@ #include "config.h" #if defined HADOOP_SNAPPY_LIBRARY -#include "commons.h" +#include "lib/commons.h" #include "NativeTask.h" #include "SnappyCodec.h" http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h index 3ba06b8..0673cf3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.h @@ -19,7 +19,7 @@ #ifndef SNAPPYCODEC_H_ #define SNAPPYCODEC_H_ -#include "Compressions.h" +#include "lib/Compressions.h" #include "BlockCodec.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc index 7ed363b..37e0d42 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/AbstractMapHandler.cc @@ -16,20 +16,19 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "util/StringUtil.h" #include "MCollectorOutputHandler.h" -#include "NativeObjectFactory.h" -#include "MapOutputCollector.h" +#include "lib/NativeObjectFactory.h" +#include "lib/MapOutputCollector.h" #include "CombineHandler.h" using std::string; using std::vector; namespace NativeTask { - const Command AbstractMapHandler::GET_OUTPUT_PATH(100, "GET_OUTPUT_PATH"); const Command AbstractMapHandler::GET_OUTPUT_INDEX_PATH(101, "GET_OUTPUT_INDEX_PATH"); const Command AbstractMapHandler::GET_SPILL_PATH(102, "GET_SPILL_PATH"); const Command AbstractMapHandler::GET_COMBINE_HANDLER(103, "GET_COMBINE_HANDLER"); -} //namespace +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc index b5b7451..ecb217c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/BatchHandler.cc @@ -19,11 +19,11 @@ #ifndef QUICK_BUILD #include "org_apache_hadoop_mapred_nativetask_NativeBatchProcessor.h" #endif -#include "commons.h" +#include "lib/commons.h" #include "jni_md.h" -#include "jniutils.h" +#include "lib/jniutils.h" #include "BatchHandler.h" -#include "NativeObjectFactory.h" +#include "lib/NativeObjectFactory.h" /////////////////////////////////////////////////////////////// // NativeBatchProcessor jni util methods @@ -130,7 +130,8 @@ void BatchHandler::onSetup(Config * config, char * inputBuffer, uint32_t inputBu _out.reset(outputBuffer, outputBufferCapacity); _out.rewind(0, outputBufferCapacity); - LOG("[BatchHandler::onSetup] input Capacity %d, output capacity %d", inputBufferCapacity, _out.limit()); + LOG("[BatchHandler::onSetup] input Capacity %d, output capacity %d", + inputBufferCapacity, _out.limit()); } configure(_config); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc index 83da238..5f3863e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.cc @@ -208,7 +208,6 @@ void CombineHandler::write(char * buf, uint32_t length) { outputRecordCount++; remain -= kv->length(); pos += kv->length(); - ; } _combineOutputRecordCount += outputRecordCount; @@ -242,7 +241,8 @@ void CombineHandler::combine(CombineContext type, KVIterator * kvIterator, IFile this->_writer = writer; call(COMBINE, NULL); - LOG("[CombineHandler] input Record Count: %d, input Bytes: %d, output Record Count: %d, output Bytes: %d", + LOG("[CombineHandler] input Record Count: %d, input Bytes: %d, " + "output Record Count: %d, output Bytes: %d", _combineInputRecordCount, _combineInputBytes, _combineOutputRecordCount, _combineOutputBytes); return; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h index 7f10552..eb43bd4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/CombineHandler.h @@ -18,7 +18,7 @@ #ifndef _COMBINEHANDLER_H_ #define _COMBINEHANDLER_H_ -#include "Combiner.h" +#include "lib/Combiner.h" #include "BatchHandler.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc index 4df5e64..7e4ae44 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/handler/MCollectorOutputHandler.cc @@ -16,12 +16,12 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "util/StringUtil.h" #include "lib/TaskCounters.h" #include "MCollectorOutputHandler.h" -#include "NativeObjectFactory.h" -#include "MapOutputCollector.h" +#include "lib/NativeObjectFactory.h" +#include "lib/MapOutputCollector.h" #include "CombineHandler.h" using std::string; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc index 33a1de9..e459b46 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.cc @@ -16,123 +16,11 @@ * limitations under the License. */ -#include "commons.h" -#include "BufferStream.h" +#include "lib/commons.h" +#include "lib/BufferStream.h" namespace NativeTask { -BufferedInputStream::BufferedInputStream(InputStream * stream, uint32_t bufferSize) - : FilterInputStream(stream), _buff(NULL), _position(0), _limit(0), _capacity(0) { - _buff = (char*)malloc(bufferSize); - if (NULL != _buff) { - LOG("[BuferStream] malloc failed when create BufferedInputStream with buffersize %u", - bufferSize); - _capacity = bufferSize; - } -} - -BufferedInputStream::~BufferedInputStream() { - if (NULL != _buff) { - free(_buff); - _buff = NULL; - _position = 0; - _limit = 0; - _capacity = 0; - } -} - -void BufferedInputStream::seek(uint64_t position) { - if (_limit - _position > 0) { - THROW_EXCEPTION(IOException, "temporary buffered data exists when fseek()"); - } - _stream->seek(position); -} - -uint64_t BufferedInputStream::tell() { - return _stream->tell() - (_limit - _position); -} - -int32_t BufferedInputStream::read(void * buff, uint32_t length) { - uint32_t rest = _limit - _position; - if (rest > 0) { - // have some data in buffer, read from buffer - uint32_t cp = rest < length ? rest : length; - memcpy(buff, _buff + _position, cp); - _position += cp; - return cp; - } else if (length >= _capacity / 2) { - // dest buffer big enough, read to dest buffer directly - return _stream->read(buff, length); - } else { - // read to buffer first, then copy part of it to dest - _limit = 0; - do { - int32_t rd = _stream->read(_buff + _limit, _capacity - _limit); - if (rd <= 0) { - break; - } - } while (_limit < _capacity / 2); - if (_limit == 0) { - return -1; - } - uint32_t cp = _limit < length ? _limit : length; - memcpy(buff, _buff, cp); - _position = cp; - return cp; - } -} - -///////////////////////////////////////////////////////////////// - -BufferedOutputStream::BufferedOutputStream(InputStream * stream, uint32_t bufferSize) - : FilterOutputStream(_stream), _buff(NULL), _position(0), _capacity(0) { - _buff = (char*)malloc(bufferSize + sizeof(uint64_t)); - if (NULL != _buff) { - LOG("[BuferStream] malloc failed when create BufferedOutputStream with buffersize %u", - bufferSize); - _capacity = bufferSize; - } -} - -BufferedOutputStream::~BufferedOutputStream() { - if (NULL != _buff) { - free(_buff); - _buff = NULL; - _position = 0; - _capacity = 0; - } -} - -uint64_t BufferedOutputStream::tell() { - return _stream->tell() + _position; -} - -void BufferedOutputStream::write(const void * buff, uint32_t length) { - if (length < _capacity / 2) { - uint32_t rest = _capacity - _position; - if (length < rest) { - simple_memcpy(_buff + _position, buff, length); - _position += length; - } else { - flush(); - simple_memcpy(_buff, buff, length); - _position = length; - } - } else { - flush(); - _stream->write(buff, length); - } -} - -void BufferedOutputStream::flush() { - if (_position > 0) { - _stream->write(_buff, _position); - _position = 0; - } -} - -/////////////////////////////////////////////////////////// - int32_t InputBuffer::read(void * buff, uint32_t length) { uint32_t rd = _capacity - _position < length ? _capacity - _position : length; if (rd > 0) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h index 541bab5..e945b95 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/BufferStream.h @@ -20,49 +20,12 @@ #define BUFFERSTREAM_H_ #include <string> -#include "Streams.h" +#include "lib/Streams.h" namespace NativeTask { using std::string; -class BufferedInputStream : public FilterInputStream { -protected: - char * _buff; - uint32_t _position; - uint32_t _limit; - uint32_t _capacity; -public: - BufferedInputStream(InputStream * stream, uint32_t bufferSize = 64 * 1024); - - virtual ~BufferedInputStream(); - - virtual void seek(uint64_t position); - - virtual uint64_t tell(); - - virtual int32_t read(void * buff, uint32_t length); -}; - -class BufferedOutputStream : public FilterOutputStream { -protected: - char * _buff; - uint32_t _position; - uint32_t _capacity; - -public: - BufferedOutputStream(InputStream * stream, uint32_t bufferSize = 64 * 1024); - - virtual ~BufferedOutputStream(); - - virtual uint64_t tell(); - - virtual void write(const void * buff, uint32_t length); - - virtual void flush(); - -}; - class InputBuffer : public InputStream { protected: const char * _buff; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc index d07f2fb..d40abb9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.cc @@ -16,84 +16,15 @@ * limitations under the License. */ -#include "commons.h" +#include <string> + +#include "lib/commons.h" #include "util/StringUtil.h" #include "util/WritableUtils.h" -#include "Buffers.h" +#include "lib/Buffers.h" namespace NativeTask { -DynamicBuffer::DynamicBuffer() - : _data(NULL), _capacity(0), _size(0), _used(0) { -} - -DynamicBuffer::DynamicBuffer(uint32_t capacity) - : _data(NULL), _capacity(0), _size(0), _used(0) { - reserve(capacity); -} - -DynamicBuffer::~DynamicBuffer() { - release(); -} - -void DynamicBuffer::release() { - if (_data != NULL) { - free(_data); - _data = NULL; - _capacity = 0; - _used = 0; - } -} - -void DynamicBuffer::reserve(uint32_t capacity) { - if (_data != NULL) { - if (capacity > _capacity) { - char * newdata = (char*)realloc(_data, capacity); - if (newdata == NULL) { - THROW_EXCEPTION_EX(OutOfMemoryException, "DynamicBuffer reserve realloc %u failed", - capacity); - } - _data = newdata; - _capacity = capacity; - } - return; - } - release(); - char * newdata = (char*)malloc(capacity); - if (newdata == NULL) { - THROW_EXCEPTION_EX(OutOfMemoryException, "DynamicBuffer reserve new %u failed", capacity); - } - _data = newdata; - _capacity = capacity; - _size = 0; - _used = 0; -} - -int32_t DynamicBuffer::refill(InputStream * stream) { - if (_data == NULL || freeSpace() == 0) { - THROW_EXCEPTION(IOException, "refill DynamicBuffer failed, no space left"); - } - int32_t rd = stream->read(_data + _size, freeSpace()); - if (rd > 0) { - _size += rd; - } - return rd; -} - -void DynamicBuffer::cleanUsed() { - if (_used > 0) { - uint32_t needToMove = _size - _used; - if (needToMove > 0) { - memmove(_data, _data + _used, needToMove); - _size = needToMove; - } else { - _size = 0; - } - _used = 0; - } -} - -/////////////////////////////////////////////////////////// ReadBuffer::ReadBuffer() : _buff(NULL), _remain(0), _size(0), _capacity(0), _stream(NULL), _source(NULL) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h index 0da23af..13cd545 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Buffers.h @@ -19,69 +19,12 @@ #ifndef BUFFERS_H_ #define BUFFERS_H_ -#include "Streams.h" -#include "Compressions.h" -#include "Constants.h" +#include "lib/Streams.h" +#include "lib/Compressions.h" +#include "lib/Constants.h" namespace NativeTask { -class DynamicBuffer { -protected: - char * _data; - uint32_t _capacity; - uint32_t _size; - uint32_t _used; -public: - DynamicBuffer(); - - DynamicBuffer(uint32_t capacity); - - ~DynamicBuffer(); - - void reserve(uint32_t capacity); - - void release(); - - uint32_t capacity() { - return _capacity; - } - - char * data() { - return _data; - } - - uint32_t size() { - return _size; - } - - uint32_t used() { - return _used; - } - - char * current() { - return _data + _used; - } - - char * end() { - return _data + _size; - } - - uint32_t remain() { - return _size - _used; - } - - uint32_t freeSpace() { - return _capacity - _size; - } - - void use(uint32_t count) { - _used += count; - } - - void cleanUsed(); - - int32_t refill(InputStream * stream); -}; /** * A lightweight read buffer, act as buffered input stream http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc deleted file mode 100644 index b426fc3..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.cc +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "Combiner.h" -#include "StringUtil.h" - -namespace NativeTask { - -NativeCombineRunner::NativeCombineRunner(Config * config, ObjectCreatorFunc combinerCreator) - : _config(config), _combinerCreator(combinerCreator), _keyGroupCount(0) { - if (NULL == _combinerCreator) { - THROW_EXCEPTION_EX(UnsupportException, "Create combiner failed"); - } -} - -KeyGroupIterator * NativeCombineRunner::createKeyGroupIterator(KVIterator * iter) { - return new KeyGroupIteratorImpl(iter); -} - -void NativeCombineRunner::combine(CombineContext context, KVIterator * iterator, - IFileWriter * writer) { - Configurable * combiner = (Configurable *)(_combinerCreator()); - if (NULL != combiner) { - combiner->configure(_config); - } - - NativeObjectType type = combiner->type(); - switch (type) { - case MapperType: { - Mapper * mapper = (Mapper*)combiner; - mapper->setCollector(writer); - - Buffer key; - Buffer value; - while (iterator->next(key, value)) { - mapper->map(key.data(), key.length(), value.data(), value.length()); - } - mapper->close(); - delete mapper; - } - break; - case ReducerType: { - Reducer * reducer = (Reducer*)combiner; - reducer->setCollector(writer); - KeyGroupIterator * kg = createKeyGroupIterator(iterator); - while (kg->nextKey()) { - _keyGroupCount++; - reducer->reduce(*kg); - } - reducer->close(); - delete reducer; - } - break; - default: - THROW_EXCEPTION(UnsupportException, "Combiner type not support"); - } -} - -} /* namespace NativeTask */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h index 09f4eb0..a7309ff 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Combiner.h @@ -18,7 +18,7 @@ #ifndef COMBINER_H_ #define COMBINER_H_ #include "commons.h" -#include "IFile.h" +#include "lib/IFile.h" namespace NativeTask { @@ -66,21 +66,5 @@ public: } }; -class NativeCombineRunner : public ICombineRunner { -private: - Config * _config; - ObjectCreatorFunc _combinerCreator; - uint32_t _keyGroupCount; - -public: - NativeCombineRunner(Config * config, ObjectCreatorFunc objectCreator); - -public: - void combine(CombineContext type, KVIterator * kvIterator, IFileWriter * writer); - -private: - KeyGroupIterator * createKeyGroupIterator(KVIterator * iter); -}; - } /* namespace NativeTask */ #endif /* COMBINER_H_ */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc index 0f954e0..879c016 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.cc @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "config.h" -#include "SyncUtils.h" -#include "Compressions.h" +#include "lib/Compressions.h" +#include "util/SyncUtils.h" #include "codec/GzipCodec.h" #include "codec/SnappyCodec.h" #include "codec/Lz4Codec.h" http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h index f70adb6..96c5084 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Compressions.h @@ -21,7 +21,7 @@ #include <string> #include <vector> -#include "Streams.h" +#include "lib/Streams.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc index e137883..e0a698c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.cc @@ -20,14 +20,14 @@ #include <fcntl.h> #include <dirent.h> #include <sys/stat.h> -#include "commons.h" +#include "lib/commons.h" #include "util/StringUtil.h" -#include "jniutils.h" +#include "lib/jniutils.h" #include "NativeTask.h" -#include "TaskCounters.h" -#include "NativeObjectFactory.h" -#include "Path.h" -#include "FileSystem.h" +#include "lib/TaskCounters.h" +#include "lib/NativeObjectFactory.h" +#include "lib/Path.h" +#include "lib/FileSystem.h" namespace NativeTask { @@ -122,14 +122,15 @@ void FileOutputStream::close() { ///////////////////////////////////////////////////////////// class RawFileSystem : public FileSystem { -protected: + protected: string getRealPath(const string & path) { if (StringUtil::StartsWith(path, "file:")) { return path.substr(5); } return path; } -public: + + public: InputStream * open(const string & path) { return new FileInputStream(getRealPath(path)); } @@ -206,7 +207,7 @@ public: struct stat sb; if (stat(np.c_str(), &sb) == 0) { - if (S_ISDIR (sb.st_mode) == 0) { + if (S_ISDIR(sb.st_mode) == 0) { return 1; } return 0; @@ -226,7 +227,7 @@ public: if (mkdir(npath, nmode)) { return 1; } - } else if (S_ISDIR (sb.st_mode) == 0) { + } else if (S_ISDIR(sb.st_mode) == 0) { return 1; } *p++ = '/'; /* restore slash */ @@ -259,4 +260,4 @@ FileSystem & FileSystem::getLocal() { return RawFileSystemInstance; } -} // namespace Hadoap +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h index 7649947..8eaf32d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/FileSystem.h @@ -21,7 +21,7 @@ #include <string> #include "NativeTask.h" -#include "Streams.h" +#include "lib/Streams.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc index 4a5edda..2d3e0b5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.cc @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "util/StringUtil.h" -#include "IFile.h" -#include "Compressions.h" +#include "lib/IFile.h" +#include "lib/Compressions.h" #include "lib/FileSystem.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h index d86e98a..e397f90 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/IFile.h @@ -19,11 +19,11 @@ #ifndef IFILE_H_ #define IFILE_H_ -#include "Checksum.h" -#include "Buffers.h" -#include "WritableUtils.h" -#include "SpillInfo.h" -#include "MapOutputSpec.h" +#include "util/Checksum.h" +#include "lib/Buffers.h" +#include "util/WritableUtils.h" +#include "lib/SpillInfo.h" +#include "lib/MapOutputSpec.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc index dfde456..3d0516a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Iterator.cc @@ -15,8 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "Iterator.h" -#include "commons.h" +#include "lib/Iterator.h" +#include "lib/commons.h" namespace NativeTask { @@ -88,4 +88,4 @@ bool KeyGroupIteratorImpl::next() { return result; } -} //namespace NativeTask +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc index 2eaa342..1e36e18 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Log.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "Log.h" +#include "lib/Log.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc index d151f1f..1175b23 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.cc @@ -16,19 +16,21 @@ * limitations under the License. */ -#include "commons.h" +#include <string> + +#include "lib/commons.h" #include "util/Timer.h" #include "util/StringUtil.h" -#include "FileSystem.h" -#include "NativeObjectFactory.h" -#include "MapOutputCollector.h" -#include "Merge.h" +#include "lib/FileSystem.h" +#include "lib/NativeObjectFactory.h" +#include "lib/MapOutputCollector.h" +#include "lib/Merge.h" #include "NativeTask.h" -#include "WritableUtils.h" +#include "util/WritableUtils.h" #include "util/DualPivotQuickSort.h" -#include "Combiner.h" -#include "TaskCounters.h" -#include "MinHeap.h" +#include "lib/Combiner.h" +#include "lib/TaskCounters.h" +#include "lib/MinHeap.h" namespace NativeTask { @@ -36,22 +38,18 @@ ICombineRunner * CombineRunnerWrapper::createCombiner() { ICombineRunner * combineRunner = NULL; if (NULL != _config->get(NATIVE_COMBINER)) { - const char * combinerClass = _config->get(NATIVE_COMBINER); - ObjectCreatorFunc objectCreater = NativeObjectFactory::GetObjectCreator(combinerClass); - if (NULL == objectCreater) { - THROW_EXCEPTION_EX(UnsupportException, "Combiner not found: %s", combinerClass); - } else { - LOG("[MapOutputCollector::configure] native combiner is enabled: %s", combinerClass); - } - combineRunner = new NativeCombineRunner(_config, objectCreater); + // Earlier versions of this code supported user-defined + // native Combiner implementations. This simplified version + // no longer supports it. + THROW_EXCEPTION_EX(UnsupportException, "Native Combiners not supported"); + } + + CombineHandler * javaCombiner = _spillOutput->getJavaCombineHandler(); + if (NULL != javaCombiner) { + _isJavaCombiner = true; + combineRunner = (ICombineRunner *)javaCombiner; } else { - CombineHandler * javaCombiner = _spillOutput->getJavaCombineHandler(); - if (NULL != javaCombiner) { - _isJavaCombiner = true; - combineRunner = (ICombineRunner *)javaCombiner; - } else { - LOG("[MapOutputCollector::getCombiner] cannot get combine handler from java"); - } + LOG("[MapOutputCollector::getCombiner] cannot get combine handler from java"); } return combineRunner; } @@ -118,7 +116,7 @@ void MapOutputCollector::init(uint32_t defaultBlockSize, uint32_t memoryCapacity _pool->init(memoryCapacity); - //TODO: add support for customized comparator + // TODO: add support for customized comparator this->_keyComparator = keyComparator; _buckets = new PartitionBucket*[_numPartitions]; @@ -160,7 +158,8 @@ void MapOutputCollector::configure(Config * config) { uint32_t capacity = config->getInt(MAPRED_IO_SORT_MB, 300) * 1024 * 1024; uint32_t defaultBlockSize = getDefaultBlockSize(capacity, _numPartitions, maxBlockSize); - LOG("Native Total MemoryBlockPool: num_partitions %u, min_block_size %uK, max_block_size %uK, capacity %uM", _numPartitions, defaultBlockSize / 1024, + LOG("Native Total MemoryBlockPool: num_partitions %u, min_block_size %uK, " + "max_block_size %uK, capacity %uM", _numPartitions, defaultBlockSize / 1024, maxBlockSize / 1024, capacity / 1024 / 1024); ComparatorPtr comparator = getComparator(config, _spec); @@ -179,7 +178,8 @@ void MapOutputCollector::configure(Config * config) { KVBuffer * MapOutputCollector::allocateKVBuffer(uint32_t partitionId, uint32_t kvlength) { PartitionBucket * partition = getPartition(partitionId); if (NULL == partition) { - THROW_EXCEPTION_EX(IOException, "Partition is NULL, partition_id: %d, num_partitions: %d", partitionId, _numPartitions); + THROW_EXCEPTION_EX(IOException, "Partition is NULL, partition_id: %d, num_partitions: %d", + partitionId, _numPartitions); } KVBuffer * dest = partition->allocateKVBuffer(kvlength); @@ -301,7 +301,7 @@ void MapOutputCollector::middleSpill(const std::string & spillOutput, info->path = spillOutput; uint64_t spillTime = timer.now() - timer.last() - metrics.sortTime; - const uint64_t M = 1000000; //million + const uint64_t M = 1000000; // million LOG("%s-spill: { id: %d, collect: %"PRIu64" ms, " "in-memory sort: %"PRIu64" ms, in-memory records: %"PRIu64", " "merge&spill: %"PRIu64" ms, uncompressed size: %"PRIu64", " @@ -369,7 +369,7 @@ void MapOutputCollector::finalSpill(const std::string & filepath, uint64_t recordCount; writer->getStatistics(outputSize, realOutputSize, recordCount); - const uint64_t M = 1000000; //million + const uint64_t M = 1000000; // million LOG("Final-merge-spill: { id: %d, in-memory sort: %"PRIu64" ms, " "in-memory records: %"PRIu64", merge&spill: %"PRIu64" ms, " "records: %"PRIu64", uncompressed size: %"PRIu64", " http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h index 474b8b1..ee447ed 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputCollector.h @@ -20,14 +20,14 @@ #define MAP_OUTPUT_COLLECTOR_H_ #include "NativeTask.h" -#include "MemoryPool.h" -#include "Timer.h" -#include "Buffers.h" -#include "MapOutputSpec.h" -#include "IFile.h" -#include "SpillInfo.h" -#include "Combiner.h" -#include "PartitionBucket.h" +#include "lib/MemoryPool.h" +#include "util/Timer.h" +#include "lib/Buffers.h" +#include "lib/MapOutputSpec.h" +#include "lib/IFile.h" +#include "lib/SpillInfo.h" +#include "lib/Combiner.h" +#include "lib/PartitionBucket.h" #include "lib/SpillOutputService.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc index 5088206..8874d02 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.cc @@ -16,8 +16,8 @@ * limitations under the License. */ -#include "commons.h" -#include "MapOutputSpec.h" +#include "lib/commons.h" +#include "lib/MapOutputSpec.h" #include "NativeTask.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h index fac96ab..ce43997 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MapOutputSpec.h @@ -20,8 +20,8 @@ #define MAPOUTPUTSPEC_H_ #include <string> -#include "Checksum.h" -#include "WritableUtils.h" +#include "util/Checksum.h" +#include "util/WritableUtils.h" #include "NativeTask.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc index 29f59b9..b3734a1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryBlock.cc @@ -15,18 +15,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#include <algorithm> + #include "NativeTask.h" -#include "commons.h" -#include "Timer.h" -#include "Buffers.h" -#include "MapOutputSpec.h" -#include "IFile.h" -#include "SpillInfo.h" -#include "Combiner.h" -#include "MapOutputSpec.h" +#include "lib/commons.h" +#include "util/Timer.h" +#include "lib/Buffers.h" +#include "lib/MapOutputSpec.h" +#include "lib/IFile.h" +#include "lib/SpillInfo.h" +#include "lib/Combiner.h" -#include "MemoryBlock.h" -#include "MemoryPool.h" +#include "lib/MemoryBlock.h" +#include "lib/MemoryPool.h" #include "util/DualPivotQuickSort.h" namespace NativeTask { @@ -62,4 +64,4 @@ void MemoryBlock::sort(SortAlgorithm type, ComparatorPtr comparator) { } _sorted = true; } -} //namespace NativeTask +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h index 0303fb2..1aa9cc2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MemoryPool.h @@ -19,8 +19,8 @@ #ifndef MEMORYPOOL_H_ #define MEMORYPOOL_H_ -#include "Buffers.h" -#include "MapOutputSpec.h" +#include "lib/Buffers.h" +#include "lib/MapOutputSpec.h" #include "NativeTask.h" #include "util/StringUtil.h" http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc index 126b82d..0d18db8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.cc @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "util/Timer.h" #include "util/StringUtil.h" -#include "Merge.h" +#include "lib/Merge.h" #include "lib/FileSystem.h" namespace NativeTask { @@ -34,7 +34,6 @@ Merger::Merger(IFileWriter * writer, Config * config, ComparatorPtr comparator, ICombineRunner * combineRunner) : _writer(writer), _config(config), _combineRunner(combineRunner), _first(true), _comparator(comparator) { - } Merger::~Merger() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h index 956fa41..cf1f2be 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Merge.h @@ -20,10 +20,10 @@ #define MERGE_H_ #include "NativeTask.h" -#include "Buffers.h" -#include "MapOutputCollector.h" -#include "IFile.h" -#include "MinHeap.h" +#include "lib/Buffers.h" +#include "lib/MapOutputCollector.h" +#include "lib/IFile.h" +#include "lib/MinHeap.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h index ab434f6..5885f34 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/MinHeap.h @@ -20,7 +20,7 @@ #define MIN_HEAP_H_ #include "NativeTask.h" -#include "Buffers.h" +#include "lib/Buffers.h" template<typename T, typename Compare> void heapify(T* first, int rt, int heap_len, Compare & Comp) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc index 237b55e..5d69b26 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeLibrary.cc @@ -18,9 +18,9 @@ #include <dlfcn.h> -#include "commons.h" -#include "NativeObjectFactory.h" -#include "NativeLibrary.h" +#include "lib/commons.h" +#include "lib/NativeObjectFactory.h" +#include "lib/NativeLibrary.h" namespace NativeTask { @@ -30,7 +30,6 @@ namespace NativeTask { NativeLibrary::NativeLibrary(const string & path, const string & name) : _path(path), _name(name), _getObjectCreatorFunc(NULL), _functionGetter(NULL) { - } bool NativeLibrary::init() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc index ce3799d..2185798 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeObjectFactory.cc @@ -20,11 +20,11 @@ #ifndef __CYGWIN__ #include <execinfo.h> #endif -#include "commons.h" +#include "lib/commons.h" #include "NativeTask.h" -#include "NativeObjectFactory.h" -#include "NativeLibrary.h" -#include "BufferStream.h" +#include "lib/NativeObjectFactory.h" +#include "lib/NativeLibrary.h" +#include "lib/BufferStream.h" #include "util/StringUtil.h" #include "util/SyncUtils.h" #include "util/WritableUtils.h" @@ -53,19 +53,10 @@ extern "C" void handler(int sig) { } DEFINE_NATIVE_LIBRARY(NativeTask) { - //signal(SIGSEGV, handler); REGISTER_CLASS(BatchHandler, NativeTask); REGISTER_CLASS(CombineHandler, NativeTask); REGISTER_CLASS(MCollectorOutputHandler, NativeTask); - REGISTER_CLASS(Mapper, NativeTask); - REGISTER_CLASS(Reducer, NativeTask); - REGISTER_CLASS(Partitioner, NativeTask); - REGISTER_CLASS(Folder, NativeTask); NativeObjectFactory::SetDefaultClass(BatchHandlerType, "NativeTask.BatchHandler"); - NativeObjectFactory::SetDefaultClass(MapperType, "NativeTask.Mapper"); - NativeObjectFactory::SetDefaultClass(ReducerType, "NativeTask.Reducer"); - NativeObjectFactory::SetDefaultClass(PartitionerType, "NativeTask.Partitioner"); - NativeObjectFactory::SetDefaultClass(FolderType, "NativeTask.Folder"); } namespace NativeTask { @@ -132,7 +123,7 @@ bool NativeObjectFactory::Init() { } const char * version = GetConfig().get(NATIVE_HADOOP_VERSION); LOG("[NativeObjectLibrary] NativeTask library initialized with hadoop %s", - version==NULL?"unkown":version); + version == NULL ? "unkown" : version); } return true; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc index 28aeaf3..be682e8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeRuntimeJniImpl.cc @@ -20,9 +20,9 @@ #include "org_apache_hadoop_mapred_nativetask_NativeRuntime.h" #endif #include "config.h" -#include "commons.h" -#include "jniutils.h" -#include "NativeObjectFactory.h" +#include "lib/commons.h" +#include "lib/jniutils.h" +#include "lib/NativeObjectFactory.h" using namespace NativeTask; @@ -36,8 +36,7 @@ using namespace NativeTask; * Signature: ([B)Z */ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_supportsCompressionCodec - (JNIEnv *jenv, jclass clazz, jbyteArray codec) -{ + (JNIEnv *jenv, jclass clazz, jbyteArray codec) { const std::string codecString = JNU_ByteArrayToString(jenv, codec); if ("org.apache.hadoop.io.compress.GzipCodec" == codecString) { return JNI_TRUE; @@ -58,7 +57,8 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntim * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime * Method: JNIRelease * Signature: ()V - */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease( + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRelease( JNIEnv * jenv, jclass nativeRuntimeClass) { try { NativeTask::NativeObjectFactory::Release(); @@ -82,7 +82,8 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntim * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime * Method: JNIConfigure * Signature: ([[B)V - */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIConfigure( + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIConfigure( JNIEnv * jenv, jclass nativeRuntimeClass, jobjectArray configs) { try { NativeTask::Config & config = NativeTask::NativeObjectFactory::GetConfig(); @@ -139,7 +140,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime * Method: JNICreateDefaultNativeObject * Signature: ([B)J - */JNIEXPORT jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateDefaultNativeObject( + */ +JNIEXPORT jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateDefaultNativeObject( JNIEnv * jenv, jclass nativeRuntimeClass, jbyteArray type) { try { std::string typeString = JNU_ByteArrayToString(jenv, type); @@ -166,7 +168,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime * Method: JNIReleaseNativeObject * Signature: (J)V - */JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIReleaseNativeObject( + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIReleaseNativeObject( JNIEnv * jenv, jclass nativeRuntimeClass, jlong objectAddr) { try { NativeTask::NativeObject * nobj = ((NativeTask::NativeObject *)objectAddr); @@ -196,7 +199,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime * Method: JNIRegisterModule * Signature: ([B[B)I - */JNIEXPORT jint JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRegisterModule( + */ +JNIEXPORT jint JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIRegisterModule( JNIEnv * jenv, jclass nativeRuntimeClass, jbyteArray modulePath, jbyteArray moduleName) { try { std::string pathString = JNU_ByteArrayToString(jenv, modulePath); @@ -225,7 +229,8 @@ jlong JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNICreateNa * Class: org_apache_hadoop_mapred_nativetask_NativeRuntime * Method: JNIUpdateStatus * Signature: ()[B - */JNIEXPORT jbyteArray JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIUpdateStatus( + */ +JNIEXPORT jbyteArray JNICALL Java_org_apache_hadoop_mapred_nativetask_NativeRuntime_JNIUpdateStatus( JNIEnv * jenv, jclass nativeRuntimeClass) { try { std::string statusData; http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc index 5c42e56..5dc880c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/NativeTask.cc @@ -18,11 +18,10 @@ #ifndef __CYGWIN__ #include <execinfo.h> #endif -#include "commons.h" -#include "util/Hash.h" +#include "lib/commons.h" #include "util/StringUtil.h" #include "NativeTask.h" -#include "NativeObjectFactory.h" +#include "lib/NativeObjectFactory.h" namespace NativeTask { @@ -34,20 +33,6 @@ const string NativeObjectTypeToString(NativeObjectType type) { switch (type) { case BatchHandlerType: return string("BatchHandlerType"); - case MapperType: - return string("MapperType"); - case ReducerType: - return string("ReducerType"); - case PartitionerType: - return string("PartitionerType"); - case CombinerType: - return string("CombinerType"); - case FolderType: - return string("FolderType"); - case RecordReaderType: - return string("RecordReaderType"); - case RecordWriterType: - return string("RecordWriterType"); default: return string("UnknownObjectType"); } @@ -56,20 +41,6 @@ const string NativeObjectTypeToString(NativeObjectType type) { NativeObjectType NativeObjectTypeFromString(const string type) { if (type == "BatchHandlerType") { return BatchHandlerType; - } else if (type == "MapperType") { - return MapperType; - } else if (type == "ReducerType") { - return ReducerType; - } else if (type == "PartitionerType") { - return PartitionerType; - } else if (type == "CombinerType") { - return CombinerType; - } else if (type == "FolderType") { - return CombinerType; - } else if (type == "RecordReaderType") { - return RecordReaderType; - } else if (type == "RecordWriterType") { - return RecordWriterType; } return UnknownObjectType; } @@ -96,7 +67,7 @@ HadoopException::HadoopException(const string & what) { #ifndef __CYGWIN__ size = backtrace(array, 64); char ** traces = backtrace_symbols(array, size); - for (size_t i = 0; i< size;i++) { + for (size_t i = 0; i < size; i++) { _reason.append("\n\t"); _reason.append(traces[i]); } @@ -235,13 +206,6 @@ Counter * ProcessorBase::getCounter(const string & group, const string & name) { return NULL; } -uint32_t Partitioner::getPartition(const char * key, uint32_t & keyLen, uint32_t numPartition) { - if (numPartition == 1) { - return 0; - } - return (Hash::BytesHash(key, keyLen) & 0x7fffffff) % numPartition; -} - /////////////////////////////////////////////////////////// -} +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc index 55ed4b2..33c2403 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.cc @@ -16,19 +16,19 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "util/Timer.h" #include "util/StringUtil.h" -#include "NativeObjectFactory.h" -#include "PartitionBucket.h" -#include "Merge.h" +#include "lib/NativeObjectFactory.h" +#include "lib/PartitionBucket.h" +#include "lib/Merge.h" #include "NativeTask.h" -#include "WritableUtils.h" +#include "util/WritableUtils.h" #include "util/DualPivotQuickSort.h" -#include "Combiner.h" -#include "TaskCounters.h" -#include "MinHeap.h" -#include "PartitionBucketIterator.h" +#include "lib/Combiner.h" +#include "lib/TaskCounters.h" +#include "lib/MinHeap.h" +#include "lib/PartitionBucketIterator.h" namespace NativeTask { @@ -39,7 +39,8 @@ KVIterator * PartitionBucket::getIterator() { return new PartitionBucketIterator(this, _keyComparator); } -void PartitionBucket::spill(IFileWriter * writer) throw (IOException, UnsupportException) { +void PartitionBucket::spill(IFileWriter * writer) + throw(IOException, UnsupportException) { KVIterator * iterator = getIterator(); if (NULL == iterator || NULL == writer) { return; @@ -71,6 +72,4 @@ void PartitionBucket::sort(SortAlgorithm type) { _sorted = true; } -} -; -// namespace NativeTask +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h index 3be52ff..cd9ec97 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucket.h @@ -20,14 +20,14 @@ #define PARTITION_BUCKET_H_ #include "NativeTask.h" -#include "MemoryPool.h" -#include "MemoryBlock.h" -#include "Timer.h" -#include "Buffers.h" -#include "MapOutputSpec.h" -#include "IFile.h" -#include "SpillInfo.h" -#include "Combiner.h" +#include "lib/MemoryPool.h" +#include "lib/MemoryBlock.h" +#include "util/Timer.h" +#include "lib/Buffers.h" +#include "lib/MapOutputSpec.h" +#include "lib/IFile.h" +#include "lib/SpillInfo.h" +#include "lib/Combiner.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc index 0b26f21..ade042e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.cc @@ -16,18 +16,20 @@ * limitations under the License. */ -#include "commons.h" +#include <algorithm> + +#include "lib/commons.h" #include "util/Timer.h" #include "util/StringUtil.h" -#include "NativeObjectFactory.h" -#include "PartitionBucketIterator.h" -#include "Merge.h" +#include "lib/NativeObjectFactory.h" +#include "lib/PartitionBucketIterator.h" +#include "lib/Merge.h" #include "NativeTask.h" -#include "WritableUtils.h" +#include "util/WritableUtils.h" #include "util/DualPivotQuickSort.h" -#include "Combiner.h" -#include "TaskCounters.h" -#include "MinHeap.h" +#include "lib/Combiner.h" +#include "lib/TaskCounters.h" +#include "lib/MinHeap.h" namespace NativeTask { @@ -108,7 +110,5 @@ bool PartitionBucketIterator::next(Buffer & key, Buffer & value) { } return false; } -} -; -// namespace NativeTask +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h index beabff8..be78497 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/PartitionBucketIterator.h @@ -20,14 +20,14 @@ #define PARTITION_BUCKET_ITERATOR_H_ #include "NativeTask.h" -#include "MemoryPool.h" -#include "Timer.h" -#include "Buffers.h" -#include "MapOutputSpec.h" -#include "IFile.h" -#include "SpillInfo.h" -#include "Combiner.h" -#include "PartitionBucket.h" +#include "lib/MemoryPool.h" +#include "util/Timer.h" +#include "lib/Buffers.h" +#include "lib/MapOutputSpec.h" +#include "lib/IFile.h" +#include "lib/SpillInfo.h" +#include "lib/Combiner.h" +#include "lib/PartitionBucket.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc index 6d85fe2..78321e2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Path.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "Path.h" +#include "lib/Path.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc index bce2cb2..9cff529 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/SpillInfo.cc @@ -16,11 +16,11 @@ * limitations under the License. */ -#include "commons.h" -#include "Streams.h" -#include "FileSystem.h" -#include "Buffers.h" -#include "SpillInfo.h" +#include "lib/commons.h" +#include "lib/Streams.h" +#include "lib/FileSystem.h" +#include "lib/Buffers.h" +#include "lib/SpillInfo.h" namespace NativeTask { @@ -69,5 +69,5 @@ void SingleSpillInfo::writeSpillInfo(const std::string & filepath) { delete fout; } -} +} // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc index c2d64c9..9e8ffba 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/Streams.cc @@ -16,9 +16,9 @@ * limitations under the License. */ -#include "commons.h" -#include "Checksum.h" -#include "Streams.h" +#include "lib/commons.h" +#include "util/Checksum.h" +#include "lib/Streams.h" namespace NativeTask { http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc index 39a7c43..b70db4c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TaskCounters.cc @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "TaskCounters.h" +#include "lib/TaskCounters.h" namespace NativeTask { @@ -36,6 +36,5 @@ const char * TaskCounters::FILESYSTEM_COUNTER_GROUP = "FileSystemCounters"; DEFINE_COUNTER(FILE_BYTES_READ) DEFINE_COUNTER(FILE_BYTES_WRITTEN) -; } // namespace NativeTask http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h deleted file mode 100644 index 3f96faf..0000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/TrackingCollector.h +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef TRACKING_COLLECTOR_H -#define TRACKING_COLLECTOR_H - -#include <stdint.h> -#include <string> - -namespace NativeTask { - -class TrackingCollector : public Collector { -protected: - Collector * _collector; - Counter * _counter; -public: - TrackingCollector(Collector * collector, Counter * counter) - : _collector(collector), _counter(counter) { - } - - virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen) { - _counter->increase(); - _collector->collect(key, keyLen, value, valueLen); - } - - virtual void collect(const void * key, uint32_t keyLen, const void * value, uint32_t valueLen, - int32_t partition) { - _counter->increase(); - _collector->collect(key, keyLen, value, valueLen, partition); - } -}; - -} //namespace NativeTask - -#endif //TRACKING_COLLECTOR_H http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h index 4212533..57500b7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/commons.h @@ -41,13 +41,12 @@ #include <map> #include <algorithm> -#include "primitives.h" -#include "Log.h" +#include "lib/primitives.h" +#include "lib/Log.h" #include "NativeTask.h" -#include "Constants.h" +#include "lib/Constants.h" -#include "Iterator.h" -#include "TrackingCollector.h" +#include "lib/Iterator.h" #endif /* COMMONS_H_ */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc index 01eaa57..d1c45bd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/lib/jniutils.cc @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "commons.h" +#include "lib/commons.h" #include "util/StringUtil.h" #include "util/SyncUtils.h" -#include "jniutils.h" +#include "lib/jniutils.h" using namespace NativeTask; @@ -53,7 +53,7 @@ JavaVM * JNU_GetJVM(void) { JavaVMOption options[noArgs]; options[0].optionString = optHadoopClassPath; - //Create the VM + // Create the VM JavaVMInitArgs vm_args; vm_args.version = JNI_VERSION_1_6; vm_args.options = options; @@ -108,4 +108,4 @@ std::string JNU_ByteArrayToString(JNIEnv * jenv, jbyteArray src) { return ret; } return std::string(); -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/683987be/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc index 4c4f3cc..191e093 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/util/Checksum.cc @@ -17,7 +17,7 @@ */ #include <assert.h> -#include "Checksum.h" +#include "util/Checksum.h" namespace NativeTask {
