The hypertable version is hypertable-0.9.4.3
2011/4/15 pan yue <[email protected]> > Thanks for Doug Judd 's reply. the following is what done when > RangerServer leak memory. > > 1. Deployment: I deployed hypertable-0.9.34 on one machine(16GB memory, 8 > cores). > ./start-dfsbroker.sh hadoop > ./start-hyperspace.sh > ./start-master.sh > ./start-rangerserver.sh > > > 2. test insert data: one c++ hypertable client, single thread, here is the > test result: > ./pressClient 0 1 1000000 > output.txt > > TotalBytes: 65266475584B > TotalSeconds: 3285.826 > QPS: 18.943MB/S > > table schema: create table foo (name, age, sex); > > 3. hypertable.cfg > # HDFS Broker > HdfsBroker.Port=38030 > HdfsBroker.fs.default.name=hdfs://dev1:9000 > HdfsBroker.Workers=20 > # Hyperspace > Hyperspace.Replica.Host=localhost > Hyperspace.Replica.Port=38040 > Hyperspace.Replica.Dir=hyperspace > Hyperspace.Replica.Workers=20 > > # Hypertable.Master > Hypertable.Master.Host=localhost > Hypertable.Master.Port=38050 > Hypertable.Master.Workers=20 > > # Hypertable.RangeServer > Hypertable.RangeServer.Port=38060 > Hypertable.RangeServer.MemoryLimit.Percentage=60 > Hypertable.RangeServer.CommitLog.PruneThreshold.Min=50000000 > Hypertable.RangeServer.CommitLog.PruneThreshold.Max=100000000 > Hypertable.RangeServer.Timer.Interval=10000 > Hypertable.RangeServer.Maintenance.Interval=15000 > > Hyperspace.KeepAlive.Interval=30000 > Hyperspace.Lease.Interval=1000000 > Hyperspace.GracePeriod=200000 > > > > 4. memory status: (from Hypertable.RangerServer.log) > > STAT ***** MaintenanceScheduler::schedule() ***** > STAT revision_root 1302839066478990001 > STAT revision_metadata 1302839108088203001 > STAT revision_user 1302844927547261001 > STAT 0/0[..0/0:��](location) cumulative_size 137 <= prune_threshold > 50000000 > STAT 0/0[0/0:��..��](default) cumulative_size 28635 <= prune_threshold > 50000000 > STAT 0/0[0/0:��](location) cumulative_size 28635 <= prune_threshold > 50000000 > STATS user log prune threshold 50000000 > STAT 1/0[..row_key_of_302072](default) cumulative_size 40819432 <= > prune_threshold 50000000 > STAT 1/0[row_key_of_74178..��](default) cumulative_size 40819432 <= > prune_threshold 50000000 > 1302845608 INFO Hypertable.RangeServer : > (/root/src/hypertable/src/cc/Hypertable/RangeServer/RangeServer.cc:2817) > Memory Usage: 216582972 bytes > 1302845618 INFO Hypertable.RangeServer : > (/root/src/hypertable/src/cc/Hypertable/RangeServer/RSStats.h:84) > Maintenance stats scans=(0 0 0 0.000000) updates=(0 0 0 0.000000 0) > 1302845618 INFO Hypertable.RangeServer : > (/root/src/hypertable/src/cc/Hypertable/Lib/CommitLog.cc:255) Purging commit > log fragments with latest revision older than 1302839066478990001 > 1302845618 INFO Hypertable.RangeServer : > (/root/src/hypertable/src/cc/Hypertable/Lib/CommitLog.cc:255) Purging commit > log fragments with latest revision older than 1302839108088203001 > 1302845618 INFO Hypertable.RangeServer : > (/root/src/hypertable/src/cc/Hypertable/Lib/CommitLog.cc:255) Purging commit > log fragments with latest revision older than 1302844927547261001 > 1302845618 INFO Hypertable.RangeServer : > (/root/src/hypertable/src/cc/Hypertable/RangeServer/MaintenanceScheduler.cc:171) > Memory Statistics (MB): VM=10259.90, RSS=8223.74, tracked=216.58, > computed=216.58 limit=8424.00 > 1302845618 INFO Hypertable.RangeServer : > (/root/src/hypertable/src/cc/Hypertable/RangeServer/MaintenanceScheduler.cc:176) > Memory Allocation: BlockCache=0.00% BlockIndex=24.58% BloomFilter=0.05% > CellCache=52.29% ShadowCache=0.00% QueryCache=23.09% > > > 5. I do not use ThriftBroker, I write my own HypertableClient (c++) > according to the implementation of ThriftBroker.cpp > HypertableClient.cpp HypertableClient.h HypertableTypes.h was in > attachment. > > > 6. Here is my test code: PressClient.cpp > > #include "HypertableClient.h" > #include <iostream> > #include <vector> > #include <pca/util/log/log.h> > #include <signal.h> > > using namespace std; > using namespace htclient; > using namespace pca; > > int64_t totalBytes = 0; > int64_t totalUsec = 0; > > void sighandler(int signo) > { > double totalSec = totalUsec*1.0/(1000*1000); > printf("TotalBytes: %ldB\n", totalBytes); > printf("TotalSeconds: %.3lf\n", totalSec); > printf("QPS: %.3fMB/S\n", (totalBytes*1.0/totalSec)/(1024*1024)); > } > > int main(int argc, char *argv[]) > { > if (argc != 4){ > printf("usage: %s type(0:insert, 1:get) start count\n", > argv[0]); > return -1; > } > int type = atoi(argv[1]); > int start = atoi(argv[2]); > int count = atoi(argv[3]); > > printf("type=%d start=%d count=%d\n", type, start, count); > > > PCA_LOG_SETUP("./log4cpp.conf"); > string config_file = "./hypertable.cfg"; > > HypertableClient* client = HypertableClient::get_instance(config_file); > if (client == NULL){ > cout << "get HypertaleClient error!" << endl; > return -1; > }else > cout << "get HypertableClient ok!" << endl; > > string ns = "combo_test"; > int64_t nsid = client->open_namespace(ns); > if (nsid <= 0){ > cout << "open ns 'combo_test' error. nsid = " << nsid << endl; > return -1; > } > > signal(SIGTERM, sighandler); > > string table_name = "foo"; > > if (type == 0){ > int64_t mutator_id = client->open_mutator(nsid, table_name); > if (mutator_id < 0){ > cout << "open mutator error, mutator_id = " << mutator_id << > endl; > return -1; > } > cout << "open mutator ok, mutator_id = " << mutator_id << endl; > > string base_rowkey= "row_key_of"; > string base_name, tmp_base = "name_value_of"; //13B > for (int i = 0; i < 5000; i++) > base_name += tmp_base; > > char *buf = (char *)malloc(102400); //100KB > memset(buf, 0, 102400); > > struct timeval stv, etv; > for (int i = start; i < start + count; i++){ > gettimeofday(&stv, NULL); > vector<Cell> cells; > sprintf(buf, "%s_%d", base_rowkey.c_str(), i); > string row_key = buf; > for (int j = 0; j < 3; j++){ > Cell cell; > cell.row_key = row_key; > //printf("row_key: %s\n", cell.row_key.c_str()); > string value; > if (j == 0){ > cell.column_family = "name"; > sprintf(buf, "%s_%d", base_name.c_str(), i); > value = buf; > }else if (j == 1){ > cell.column_family = "age"; > sprintf(buf, "%d", 1 + i % 100); > value = buf; > }else if (j == 2){ > cell.column_family = "sex"; > value = (i % 2 ? "male" : "female"); > } > > cell.column_qualifier = buf; > cell.set_value((uint8_t*)value.c_str(), value.length()+1); > cell.flag = htclient::KeyFlag::INSERT; > cells.push_back(cell); > memset(buf, 0, 102400); > > totalBytes += (sizeof(cell) + cell.column_family.length() + > value.length() + row_key.length()); > //printf("totalBytes: %ld\n", totalBytes); > } > > if (client->set_cells(mutator_id, cells)) > printf("insert ok for row: %s\n", row_key.c_str()); > else > printf("****insert row error*****\n"); > > usleep(2000); > gettimeofday(&etv, NULL); > int32_t usec = (etv.tv_sec - stv.tv_sec)*1000000 + (etv.tv_usec > - stv.tv_usec); > > totalUsec += usec; > > > } > > client->close_mutator(mutator_id); > free(buf); > > kill(getpid(), SIGTERM); > > }else { > //string table_name = "foo"; > htclient::ScanSpec ss; > ss.columns.push_back("name"); > > int64_t scanner_id = client->open_scanner(nsid, table_name, ss); > > if (scanner_id <= 0){ > cout << "create scanner for table " << table_name << " error, > ret = " << scanner_id << endl; > return -1; > } > cout << "create scanner ok for table " << table_name << endl; > > htclient::Cell cell; > while(client->next_cell(scanner_id, cell) == HTE_OK){ > cout << "-------------get a cell------------- " << endl; > //cout << cell << endl; > cell.clear(); > } > > client->close_scanner(scanner_id); > } > > PCA_LOG_SHUTDOWN(""); > return 0; > } > > > 8. Promblem: the memory always increase when I insert data; when the insert > finished, the RSS alawy 8GB, but RangerServer declared it only used 200M. > what the remain memory going? > > > Thanks! > > > > > > > -- You received this message because you are subscribed to the Google Groups "Hypertable Development" group. To post to this group, send email to [email protected]. To unsubscribe from this group, send email to [email protected]. For more options, visit this group at http://groups.google.com/group/hypertable-dev?hl=en.
