http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc new file mode 100644 index 0000000..4ec1f48 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.cc @@ -0,0 +1,344 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdarg.h> +#include "commons.h" +#include "util/Random.h" +#include "FileSystem.h" +#include "test_commons.h" + +Config TestConfig = Config(); + +const char * GenerateSeed = "generate.seed"; +const char * GenerateChoice = "generate.choice"; +const char * GenerateLen = "generate.len"; +const char * GenerateKeyLen = "generate.key.len"; +const char * GenerateValueLen = "generate.value.len"; +const char * GenerateRange = "generate.range"; +const char * GenerateKeyRange = "generate.key.range"; +const char * GenerateValueRange = "generate.value.range"; + +vector<string> & MakeStringArray(vector<string> & dest, ...) { + va_list al; + va_start(al, dest); + while (true) { + const char * s = va_arg(al, const char *); + if (s == NULL) { + break; + } + dest.push_back(s); + } + va_end(al); + return dest; +} + +GenerateType GetGenerateType(const string & type) { + if (type == "word") { + return GenWord; + } else if (type == "number") { + return GenNumber; + } else if (type == "bytes") { + return GenBytes; + } else { + THROW_EXCEPTION(UnsupportException, "GenerateType not support"); + } +} + +string & GenerateOne(string & dest, Random & r, GenerateType gtype, int64_t choice, int64_t len, + int64_t range) { + switch (gtype) { + case GenWord: + r.nextWord(dest, choice); + break; + case GenNumber: + uint64_t v; + if (choice > 0) { + v = r.next_int32(choice); + } else { + v = r.next_uint64(); + } + if (len > 0) { + dest = StringUtil::ToString(v, '0', len); + } else { + dest = StringUtil::ToString(v); + } + break; + case GenBytes: + if (range < 2) { + if (len > 0) { + dest = r.nextBytes(len, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + } else { + dest = r.nextBytes(r.next_int32(32), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + } + } else { + if (len > 0) { + int64_t nlen = len - range / 2 + r.next_int32(range); + if (nlen > 0) { + dest = r.nextBytes(nlen, "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + } else { + dest = ""; + } + } else { + dest = r.nextBytes(r.next_int32(range), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + } + } + break; + default: + THROW_EXCEPTION(IOException, "GenerateType not support"); + } + return dest; +} + +/** + * Generate random string sequences + * @param dest dest array + * @param size output array size + * @param type string type (word|number|bytes|tera) + */ +vector<string> & Generate(vector<string> & dest, uint64_t size, const string & type) { + Random r; + if (TestConfig.get(GenerateSeed) != NULL) { + r.setSeed(TestConfig.getInt(GenerateSeed, 0)); + } + GenerateType gtype = GetGenerateType(type); + int64_t choice = TestConfig.getInt(GenerateChoice, -1); + int64_t len = TestConfig.getInt(GenerateLen, -1); + int64_t range = TestConfig.getInt(GenerateRange, 1); + string temp; + for (uint64_t i = 0; i < size; i++) { + dest.push_back(GenerateOne(temp, r, gtype, choice, len, range)); + } + return dest; +} + +/** + * Generate random string pair sequences + * @param dest dest array + * @param size output array size + * @param type string type (word|number|bytes|tera) + */ +vector<pair<string, string> > & Generate(vector<pair<string, string> > & dest, uint64_t size, + const string & type) { + Random r; + if (TestConfig.get(GenerateSeed) != NULL) { + r.setSeed(TestConfig.getInt(GenerateSeed, 0)); + } + GenerateType gtype = GetGenerateType(type); + int64_t choice = TestConfig.getInt(GenerateChoice, -1); + int64_t keylen = TestConfig.getInt(GenerateKeyLen, -1); + int64_t valuelen = TestConfig.getInt(GenerateValueLen, -1); + int64_t keyRange = TestConfig.getInt(GenerateKeyRange, 1); + int64_t valueRange = TestConfig.getInt(GenerateValueRange, 1); + string key, value; + for (uint64_t i = 0; i < size; i++) { + GenerateOne(key, r, gtype, choice, keylen, keyRange); + GenerateOne(value, r, gtype, choice, valuelen, valueRange); + dest.push_back(std::make_pair(key, value)); + } + return dest; +} + +/** + * Generate random string pair sequences + * @param dest dest array + * @param length output bytes count + * @param type string type (word|number|bytes|tera) + */ +vector<pair<string, string> > & GenerateLength(vector<pair<string, string> > & dest, + uint64_t length, const string & type) { + Random r; + if (TestConfig.get(GenerateSeed) != NULL) { + r.setSeed(TestConfig.getInt(GenerateSeed, 0)); + } + GenerateType gtype = GetGenerateType(type); + int64_t choice = TestConfig.getInt(GenerateChoice, -1); + int64_t keylen = TestConfig.getInt(GenerateKeyLen, -1); + int64_t valuelen = TestConfig.getInt(GenerateValueLen, -1); + int64_t keyRange = TestConfig.getInt(GenerateKeyRange, 1); + int64_t valueRange = TestConfig.getInt(GenerateValueRange, 1); + string key, value; + dest.reserve((size_t)(length / (keylen + valuelen) * 1.2)); + for (uint64_t i = 0; i < length;) { + GenerateOne(key, r, gtype, choice, keylen, keyRange); + GenerateOne(value, r, gtype, choice, valuelen, valueRange); + dest.push_back(std::make_pair(key, value)); + i += (key.length() + value.length() + 2); + } + return dest; +} + +/** + * Generate random KV text: + * Key0\tValue0\n + * Key1\tValue1\n + * ... + * @param dest dest string contain generated text + * @param size output array size + * @param type string type (word|number|bytes|tera) + */ +string & GenerateKVText(string & dest, uint64_t size, const string & type) { + Random r; + if (TestConfig.get(GenerateSeed) != NULL) { + r.setSeed(TestConfig.getInt(GenerateSeed, 0)); + } + GenerateType gtype = GetGenerateType(type); + int64_t choice = TestConfig.getInt(GenerateChoice, -1); + int64_t keylen = TestConfig.getInt(GenerateKeyLen, -1); + int64_t valuelen = TestConfig.getInt(GenerateValueLen, -1); + int64_t keyRange = TestConfig.getInt(GenerateKeyRange, 1); + int64_t valueRange = TestConfig.getInt(GenerateValueRange, 1); + string key, value; + for (uint64_t i = 0; i < size; i++) { + GenerateOne(key, r, gtype, choice, keylen, keyRange); + GenerateOne(value, r, gtype, choice, valuelen, valueRange); + dest.append(key); + dest.append("\t"); + dest.append(value); + dest.append("\n"); + } + return dest; +} + +/** + * Generate random KV text: + * Key0\tValue0\n + * Key1\tValue1\n + * ... + * @param dest dest string contain generated text + * @param length output string length + * @param type string type (word|number|bytes) + */ +string & GenerateKVTextLength(string & dest, uint64_t length, const string & type) { + Random r; + if (TestConfig.get(GenerateSeed) != NULL) { + r.setSeed(TestConfig.getInt(GenerateSeed, 0)); + } + GenerateType gtype = GetGenerateType(type); + int64_t choice = TestConfig.getInt(GenerateChoice, -1); + int64_t keylen = TestConfig.getInt(GenerateKeyLen, -1); + int64_t valuelen = TestConfig.getInt(GenerateValueLen, -1); + int64_t keyRange = TestConfig.getInt(GenerateKeyRange, 1); + int64_t valueRange = TestConfig.getInt(GenerateValueRange, 1); + string key, value; + while (dest.length() < length) { + GenerateOne(key, r, gtype, choice, keylen, keyRange); + GenerateOne(value, r, gtype, choice, valuelen, valueRange); + dest.append(key); + dest.append("\t"); + dest.append(value); + dest.append("\n"); + } + return dest; +} + +/** + * File <-> String utilities + */ +string & ReadFile(string & dest, const string & path) { + FILE * fin = fopen(path.c_str(), "rb"); + if (NULL == fin) { + THROW_EXCEPTION(IOException, "file not found or can not open for read"); + } + char buff[1024 * 16]; + while (true) { + size_t rd = fread(buff, 1, 1024 * 16, fin); + if (rd <= 0) { + break; + } + dest.append(buff, rd); + } + fclose(fin); + return dest; +} + +void WriteFile(const string & content, const string & path) { + FILE * fout = fopen(path.c_str(), "wb"); + if (NULL == fout) { + THROW_EXCEPTION(IOException, "file can not open for write"); + } + size_t wt = fwrite(content.c_str(), 1, content.length(), fout); + if (wt != content.length()) { + THROW_EXCEPTION(IOException, "write file error"); + } + fclose(fout); +} + +bool FileEqual(const string & lh, const string & rh) { + string lhs, rhs; + ReadFile(lhs, lh); + ReadFile(rhs, rh); + return lhs == rhs; +} + +KVGenerator::KVGenerator(uint32_t keylen, uint32_t vallen, bool unique) + : keylen(keylen), vallen(vallen), unique(unique) { + factor = 2999999; + keyb = new char[keylen + 32]; + valb = new char[vallen + 32]; + snprintf(keyformat, 32, "%%0%ulx", keylen); +} + +KVGenerator::~KVGenerator() { + delete[] keyb; + delete[] valb; +} + +char * KVGenerator::key(uint32_t & kl) { + long v; + if (unique) { + while (true) { + v = lrand48(); + if (old_keys.find(v) == old_keys.end()) { + old_keys.insert(v); + break; + } + } + } else { + v = lrand48(); + } + snprintf(keyb, keylen + 32, keyformat, v); + kl = keylen; + return keyb; +} + +char * KVGenerator::value(uint32_t & vl) { + uint32_t off = 0; + while (off < vallen) { + long v = lrand48(); + v = (v / factor) * factor; + uint32_t wn = snprintf(valb + off, vallen + 32 - off, "%09lx\t", v); + off += wn; + } + vl = vallen; + return valb; +} + +void KVGenerator::write(FILE * fout, int64_t totallen) { + while (totallen > 0) { + uint32_t kl, vl; + char * key = this->key(kl); + char * value = this->value(vl); + fwrite(key, kl, 1, fout); + fputc('\t', fout); + fwrite(value, vl, 1, fout); + fputc('\n', fout); + totallen -= (kl + vl + 2); + } + fflush(fout); +} +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h new file mode 100644 index 0000000..48884ce --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/test_commons.h @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TEST_COMMONS_H_ +#define TEST_COMMONS_H_ + +#include "gtest/gtest.h" +#include "commons.h" +#include "util/Random.h" +#include "util/StringUtil.h" +#include "util/Timer.h" +#include "Buffers.h" +#include "BufferStream.h" + +using std::pair; +using std::vector; +using std::set; +using std::map; +using std::string; + +using namespace NativeTask; + +extern Config TestConfig; + +/** + * e.g. MakeStringArray(dest, "a", "b", "c", NULL) = {"a","b","c"} + */ +vector<string> & MakeStringArray(vector<string> & dest, ...); + +extern const char * GenerateSeed; +extern const char * GenerateChoice; +extern const char * GenerateLen; +extern const char * GenerateKeyLen; +extern const char * GenerateValueLen; +extern const char * GenerateRange; +extern const char * GenerateKeyRange; +extern const char * GenerateValueRange; + +enum GenerateType { + GenWord, + GenNumber, + GenBytes, +}; + +GenerateType GetGenerateType(const string & type); + +string & GenerateOne(string & dest, Random & r, GenerateType gtype, int64_t choice, int64_t len, + int64_t range = 0); +/** + * Generate random string sequences + * @param dest dest array + * @param size output array size + * @param type string type (word|number|bytes|tera) + */ +vector<string> & Generate(vector<string> & dest, uint64_t size, const string & type); + +/** + * Generate random string pair sequences + * @param dest dest array + * @param size output array size + * @param type string type (word|number|bytes|tera) + */ +vector<pair<string, string> > & Generate(vector<pair<string, string> > & dest, uint64_t size, + const string & type); + +/** + * Generate random string pair sequences + * @param dest dest array + * @param length output bytes count + * @param type string type (word|number|bytes|tera) + */ +vector<pair<string, string> > & GenerateLength(vector<pair<string, string> > & dest, + uint64_t length, const string & type); + +/** + * Generate random KV text: + * Key0\tValue0\n + * Key1\tValue1\n + * ... + * @param dest dest string contain generated text + * @param size output kv pair count + * @param type string type (word|number|bytes|tera) + */ +string & GenerateKVText(string & dest, uint64_t size, const string & type); + +/** + * Generate random KV text: + * Key0\tValue0\n + * Key1\tValue1\n + * ... + * @param dest dest string contain generated text + * @param length output string length + * @param type string type (word|number|bytes|tera) + */ +string & GenerateKVTextLength(string & dest, uint64_t length, const string & type); + +/** + * File <-> String utilities + */ +string & ReadFile(string & dest, const string & path); +void WriteFile(const string & content, const string & path); + +/** + * File compare + */ +bool FileEqual(const string & lh, const string & rh); + +/** + * generate k/v pairs with normal compression ratio + * + */ +class KVGenerator { +protected: + uint32_t keylen; + uint32_t vallen; + bool unique; + long factor; + char * keyb; + char * valb; + char keyformat[32]; + set<int64_t> old_keys; + +public: + KVGenerator(uint32_t keylen, uint32_t vallen, bool unique = false); + + ~KVGenerator(); + + char * key(uint32_t & kl); + + char * value(uint32_t & vl); + + void write(FILE * fout, int64_t totallen); +}; + +#endif /* TEST_COMMONS_H_ */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestChecksum.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestChecksum.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestChecksum.cc new file mode 100644 index 0000000..d77656a --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestChecksum.cc @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "util/Checksum.h" +#include "test_commons.h" + +void TestChecksum(ChecksumType type, void * buff, uint32_t len) { + uint32_t chm = Checksum::init(type); + Checksum::update(type, chm, buff, len); +} + +TEST(Perf, CRC) { + uint32_t len = TestConfig.getInt("checksum.perf.size", 1024 * 1024 * 50); + int testTime = TestConfig.getInt("checksum.perf.time", 2); + char * buff = new char[len]; + memset(buff, 1, len); + Timer timer; + for (int i = 0; i < testTime; i++) { + TestChecksum(CHECKSUM_CRC32, buff, len); + } + LOG("%s", timer.getSpeedM("CRC", len * testTime).c_str()); + timer.reset(); + for (int i = 0; i < testTime; i++) { + TestChecksum(CHECKSUM_CRC32C, buff, len); + } + LOG("%s", timer.getSpeedM("CRC32C", len * testTime).c_str()); + delete[] buff; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc new file mode 100644 index 0000000..e2f854c --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestHash.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "util/Hash.h" +#include "test_commons.h" + +static uint64_t test_length(int64_t len, size_t size, size_t loopTime) { + vector<string> data; + TestConfig.setInt(GenerateLen, len); + Generate(data, size, "bytes"); + Timer t; + uint64_t ret; + for (size_t m = 0; m < loopTime; m++) { + for (size_t i = 0; i < data.size(); i++) { + ret += Hash::BytesHash(data[i].c_str(), data[i].length()); + } + } + LOG("%s", t.getInterval(StringUtil::Format("Bytes%3lld", len).c_str()).c_str()); + t.reset(); + for (size_t m = 0; m < loopTime; m++) { + for (size_t i = 0; i < data.size(); i++) { + ret += Hash::CityHash(data[i].c_str(), data[i].length()); + } + } + LOG("%s", t.getInterval(StringUtil::Format(" City%3lld", len).c_str()).c_str()); + return ret; +} + +TEST(Perf, Hash) { + uint64_t ret = 0; + ret += test_length(1, 100, 4000); + ret += test_length(17, 100, 4000); + ret += test_length(64, 100, 4000); + ret += test_length(128, 100, 4000); + ret += test_length(513, 100, 4000); + fprintf(stderr, "%llu\n", (long long unsigned int)ret); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestProcess.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestProcess.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestProcess.cc new file mode 100644 index 0000000..cf857d3 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestProcess.cc @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "util/Process.h" +#include "test_commons.h" + +TEST(Process, Run) { + string out; + string err; + string cmd = TestConfig.get("process.run.cmd", "ls"); + int retcode = Process::Run(cmd, &out, &err); + LOG("cmd: %s", cmd.c_str()); + LOG("stdout: %s", out.c_str()); + LOG("stderr: %s", err.c_str()); + LOG("retcode: %d", retcode); + EXPECT_EQ(0, retcode); +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestStringUtil.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestStringUtil.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestStringUtil.cc new file mode 100644 index 0000000..d6dc93f --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestStringUtil.cc @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "util/StringUtil.h" +#include "test_commons.h" + +TEST(StringUtil, Convertion) { + ASSERT_FLOAT_EQ(StringUtil::toFloat("1.333"), 1.333); + ASSERT_FLOAT_EQ(StringUtil::toFloat(StringUtil::ToString(1.333f)), 1.333); + ASSERT_EQ(StringUtil::ToString(76957382U), "76957382"); + ASSERT_EQ(StringUtil::ToString((uint64_t )76957382234233432ULL), "76957382234233432"); + ASSERT_EQ(StringUtil::ToString(111, ' ', 40), " 111"); +} + +TEST(StringUtil, Format) { + string t = StringUtil::Format("%d %d %d %.3lf %s", 1, 2, 3, 1.333, "aaaaaaaaaaa"); + ASSERT_EQ(t, "1 2 3 1.333 aaaaaaaaaaa"); + string longstring(999, 'a'); + string d = StringUtil::Format("%s", longstring.c_str()); + ASSERT_EQ(longstring, d); +} + +TEST(StringUtil, Trim) { + ASSERT_EQ(StringUtil::Trim(" \taaaa \t "), "aaaa"); + ASSERT_EQ(StringUtil::Trim(" \t \t "), ""); + ASSERT_EQ(StringUtil::Trim(""), ""); +} + +TEST(StringUtil, ToLower) { + ASSERT_EQ(StringUtil::ToLower("111ABabABabAbaB222"), "111abababababab222"); + ASSERT_EQ(StringUtil::ToLower(""), ""); +} + +TEST(StringUtil, JoinSplit) { + vector<string> temp1, temp2, temp3, temp4; + StringUtil::Split("1aaa bbb ccc", " ", temp1, false); + StringUtil::Split(" 1aaa bbb ccc ", " ", temp2, true); + ASSERT_EQ(temp1, temp2); + string j = StringUtil::Join(temp1, ","); + ASSERT_EQ(j, "1aaa,bbb,ccc"); + StringUtil::Split(" a b ", " ", temp3, false); + ASSERT_EQ(temp3, MakeStringArray(temp4, "", "", "a", "b", "", NULL)); +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc new file mode 100644 index 0000000..7641f9c --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestSyncUtils.cc @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "util/SyncUtils.h" +#include "test_commons.h" + +class TestThread : public Thread { + virtual void run() { + for (uint32_t i = 0; i < 5; i++) { + usleep(100); + LOG("sleep %d", i * 100); + } + } +}; + +TEST(SyncUtil, Thread) { + TestThread a, b, c; + a.start(); + b.start(); + c.start(); + a.join(); + b.join(); + c.join(); +} + +class TestBind { +public: + + int get() { + return 100; + } + + void foo() { + for (uint32_t i = 0; i < 2; i++) { + usleep(100); + LOG("usleep %d", i * 100); + } + } + void bar(const char * msg) { + for (uint32_t i = 0; i < 2; i++) { + usleep(100); + LOG("usleep %d %s", i * 100, msg); + } + } + +}; + +TEST(SyncUtil, ThreadBind) { + TestBind a = TestBind(); + Runnable * bind1 = BindNew(a, &TestBind::get); + Thread t = Thread(bind1); + Runnable * bind2 = BindNew(a, &TestBind::bar, "testmsg"); + Thread t2 = Thread(bind2); + t.start(); + t2.start(); + t.join(); + t2.join(); + + delete bind1; + delete bind2; +} + +//class TestParallelFor { +//protected: +// SpinLock lock; +// uint64_t aggr; +//public: +// TestParallelFor() : aggr(0) { +// } +// void add(uint64_t i) { +// lock.lock(); +// aggr += i; +// lock.unlock(); +// } +// void test(uint64_t n, size_t threadnum) { +// aggr = 0; +// ParallelFor(*this, &TestParallelFor::add, 0ULL, n, threadnum); +// ASSERT_EQ(n*(n-1)/2, aggr); +// } +//}; +// +//TEST(SyncUtil, ParallelFor) { +// TestParallelFor tpf; +// tpf.test(100000, 2); +// tpf.test(100000, 3); +// tpf.test(100000, 4); +//} + +TEST(Perf, ThreadOverhead) { + int64_t threadnum = TestConfig.getInt("thread.num", 1000); + Thread * t = new Thread[threadnum]; + Timer timer; + for (uint32_t i = 0; i < threadnum; i++) { + t[i].start(); + } + for (uint32_t i = 0; i < threadnum; i++) { + t[i].join(); + } + LOG("%lld thread %s", (long long int )threadnum, timer.getInterval("start&join").c_str()); + delete[] t; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2551c06/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestWritableUtils.cc ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestWritableUtils.cc b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestWritableUtils.cc new file mode 100644 index 0000000..f692c35 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/test/util/TestWritableUtils.cc @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "util/WritableUtils.h" +#include "test_commons.h" + +void TestVLong(int64_t v) { + char buff[1024]; + char buff2[1024]; + uint32_t dsize = WritableUtils::GetVLongSize(v); + uint32_t wsize = (uint32_t)-1; + WritableUtils::WriteVLong(v, buff, wsize); + ASSERT_EQ(dsize, wsize); + memcpy(buff2, buff, wsize); + uint32_t rsize; + int64_t rv = WritableUtils::ReadVLong(buff2, rsize); + ASSERT_EQ(v, rv); + ASSERT_EQ(rsize, dsize); +} + + +TEST(WritableUtils, VLong) { + int num = TestConfig.getInt("test.size", 3000); + int seed = TestConfig.getInt("test.seed", -1); + Random r(seed); + for (int i = 0; i < num; i++) { + uint64_t v = r.nextLog2(((uint64_t)-1) / 2 - 3); + TestVLong(v); + TestVLong(-v); + } +} + + +
