[1/3] incubator-trafodion git commit: [TRAFODION-2478] Reduce the number of memory monitoring threads in Trafodion SQL processes

[selva]

Repository: incubator-trafodion
Updated Branches:
  refs/heads/master 5881cf1c5 -> f6f4402b5


[TRAFODION-2478] Reduce the number of memory monitoring threads in Trafodion 
SQL processes

The memory monitor thread is now moved to mxsscp process and the collected data 
is
stored in RMS shared segment. All SQL proceses have access to it.


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/3d7df2d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/3d7df2d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/3d7df2d2

Branch: refs/heads/master
Commit: 3d7df2d24db818ade2c94d58ed9cd01c1c43f847
Parents: 554cd32
Author: selvaganesang 
Authored: Fri Feb 10 23:56:56 2017 +
Committer: selvaganesang 
Committed: Sat Feb 11 01:53:02 2017 +

--
 core/sql/bin/ex_esp_main.cpp   | 17 ++---
 core/sql/cli/Globals.cpp   | 24 +++
 core/sql/cli/memorymonitor.cpp | 34 +++--
 core/sql/executor/ex_hash_grby.h   |  3 ---
 core/sql/runtimestats/SqlStats.cpp | 12 
 core/sql/runtimestats/SqlStats.h   |  7 +--
 6 files changed, 59 insertions(+), 38 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d7df2d2/core/sql/bin/ex_esp_main.cpp
--
diff --git a/core/sql/bin/ex_esp_main.cpp b/core/sql/bin/ex_esp_main.cpp
index 49d0500..d8bb1eb 100644
--- a/core/sql/bin/ex_esp_main.cpp
+++ b/core/sql/bin/ex_esp_main.cpp
@@ -357,15 +357,18 @@ Int32 runESP(Int32 argc, char** argv, GuaReceiveFastStart 
*guaReceiveFastStart)
   cliGlobals->initiateDefaultContext();
   NAHeap *espIpcHeap = cliGlobals->getIpcHeap();
   IpcEnvironment *ipcEnvPtr = cliGlobals->getEnvironment();
-  //
-  // Start the  memory monitor for dynamic memory management
-  Lng32 memMonitorWindowSize = 10;
-  Lng32 memMonitorSampleInterval = 10;
-  MemoryMonitor memMonitor(memMonitorWindowSize,
+  if (statsGlobals != NULL)
+ cliGlobals->setMemoryMonitor(statsGlobals->getMemoryMonitor());
+  else 
+  {
+ // Start the  memory monitor for dynamic memory management
+ Lng32 memMonitorWindowSize = 10;
+ Lng32 memMonitorSampleInterval = 10;
+ MemoryMonitor memMonitor(memMonitorWindowSize,
memMonitorSampleInterval,
espExecutorHeap);
-  cliGlobals->setMemoryMonitor();
-
+ cliGlobals->setMemoryMonitor();
+  }
   // After CLI globals are initialized but before we begin ESP message
   // processing, have the CLI context set its user identity based on
   // the OS user identity.

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d7df2d2/core/sql/cli/Globals.cpp
--
diff --git a/core/sql/cli/Globals.cpp b/core/sql/cli/Globals.cpp
index 28b970e..95fb000 100644
--- a/core/sql/cli/Globals.cpp
+++ b/core/sql/cli/Globals.cpp
@@ -235,19 +235,7 @@ void CliGlobals::init( NABoolean espProcess,
   if (! espProcess)
   {
 // Create the process global ARKCMP server.
-// In R1.8, Each context has its own mxcmp.
 sharedArkcmp_ = NULL;
-
-//sharedArkcmp_->setShared(FALSE);
-// create the process global memory monitor. For now with
-// defaults of 10 window entries and sampling every 1 second
-Lng32 memMonitorWindowSize = 10;
-Lng32 memMonitorSampleInterval = 1; // reduced from 10 (for M5 - May 2011)
-memMonitor_ = new(_) MemoryMonitor(memMonitorWindowSize,
- memMonitorSampleInterval,
- _);
-
-//nextUniqueContextHandle = 2000;
 nextUniqueContextHandle = DEFAULT_CONTEXT_HANDLE;
 
 arlibHeap_ = new (_) NAHeap("MXARLIB Cache Heap",
@@ -322,6 +310,18 @@ void CliGlobals::init( NABoolean espProcess,
 capacities_.setHeap(defaultContext_->exCollHeap());
 freespaces_.setHeap(defaultContext_->exCollHeap());
 largestFragments_.setHeap(defaultContext_->exCollHeap());
+if (statsGlobals_ != NULL) 
+   memMonitor_ = statsGlobals_->getMemoryMonitor();
+else
+{
+   // create the process global memory monitor. For now with
+   // defaults of 10 window entries and sampling every 1 second
+   Lng32 memMonitorWindowSize = 10;
+   Lng32 memMonitorSampleInterval = 1; // reduced from 10 (for M5 - May 
2011)
+   memMonitor_ = new (_) MemoryMonitor(memMonitorWindowSize,
+ memMonitorSampleInterval,
+ _);
+}
   } // (!espProcess)
 
   else

[3/3] incubator-trafodion git commit: Merge [TRAFODION-2478] PR 953 Reduce the number of memory monitoring threads in Trafodion

[selva]

Merge [TRAFODION-2478] PR 953 Reduce the number of memory monitoring threads in 
Trafodion


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/f6f4402b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/f6f4402b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/f6f4402b

Branch: refs/heads/master
Commit: f6f4402b5a1438b96f551968b9c7371d0d2e2c29
Parents: 5881cf1 8ba597f
Author: selvaganesang 
Authored: Tue Feb 14 21:04:54 2017 +
Committer: selvaganesang 
Committed: Tue Feb 14 21:04:54 2017 +

--
 core/sql/bin/ex_esp_main.cpp   | 18 -
 core/sql/cli/Globals.cpp   | 24 -
 core/sql/cli/memorymonitor.cpp | 47 ++---
 core/sql/cli/memorymonitor.h   | 17 +---
 core/sql/executor/ex_hash_grby.h   |  3 ---
 core/sql/runtimestats/SqlStats.cpp | 12 +
 core/sql/runtimestats/SqlStats.h   |  7 +++--
 7 files changed, 73 insertions(+), 55 deletions(-)
--

[2/3] incubator-trafodion git commit: [TRAFODION-2478] Reduce the number of memory monitoring threads in Trafodion SQL processes

[selva]

[TRAFODION-2478] Reduce the number of memory monitoring threads in Trafodion 
SQL processes

Rework as per the review comments


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/8ba597f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/8ba597f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/8ba597f3

Branch: refs/heads/master
Commit: 8ba597f3d081737e9e62bfdbc861d2bba7bd0a36
Parents: 3d7df2d
Author: selvaganesang 
Authored: Fri Feb 10 23:56:56 2017 +
Committer: selvaganesang 
Committed: Tue Feb 14 00:06:28 2017 +

--
 core/sql/bin/ex_esp_main.cpp   |  5 +++--
 core/sql/cli/memorymonitor.cpp | 29 ++---
 core/sql/cli/memorymonitor.h   | 17 +++--
 3 files changed, 24 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8ba597f3/core/sql/bin/ex_esp_main.cpp
--
diff --git a/core/sql/bin/ex_esp_main.cpp b/core/sql/bin/ex_esp_main.cpp
index d8bb1eb..b377c63 100644
--- a/core/sql/bin/ex_esp_main.cpp
+++ b/core/sql/bin/ex_esp_main.cpp
@@ -364,10 +364,11 @@ Int32 runESP(Int32 argc, char** argv, GuaReceiveFastStart 
*guaReceiveFastStart)
  // Start the  memory monitor for dynamic memory management
  Lng32 memMonitorWindowSize = 10;
  Lng32 memMonitorSampleInterval = 10;
- MemoryMonitor memMonitor(memMonitorWindowSize,
+ MemoryMonitor *memMonitor = new (espExecutorHeap) 
+   MemoryMonitor(memMonitorWindowSize,
memMonitorSampleInterval,
espExecutorHeap);
- cliGlobals->setMemoryMonitor();
+ cliGlobals->setMemoryMonitor(memMonitor);
   }
   // After CLI globals are initialized but before we begin ESP message
   // processing, have the CLI context set its user identity based on

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8ba597f3/core/sql/cli/memorymonitor.cpp
--
diff --git a/core/sql/cli/memorymonitor.cpp b/core/sql/cli/memorymonitor.cpp
index b53b3d6..33abc00 100644
--- a/core/sql/cli/memorymonitor.cpp
+++ b/core/sql/cli/memorymonitor.cpp
@@ -59,6 +59,7 @@ extern "C" Lng32 __stdcall
 PdhOpenQuery (LPCSTR  szD, DWORD  dw, HQUERY  *phQ );
 #endif
 
+#define FREAD_BUFFER_SIZE 2048
 
 //SQ_LINUX #ifdef NA_WINNT
 
@@ -66,7 +67,7 @@ NABoolean MemoryMonitor::threadIsCreated_ = 0;
 HANDLE MemoryMonitor::updateThread_ = (HANDLE) 0;
 ULng32 MemoryMonitor::threadId_ = 0;
 
-DWORD WINAPI memMonitorUpdateThread(void * param) {
+DWORD WINAPI MemoryMonitor::memMonitorUpdateThread(void * param) {
   // params points to the memory monitor object
   MemoryMonitor *memMonitor = (MemoryMonitor*) param;
   Lng32 sleepTime = memMonitor->getSampleInterval();
@@ -102,18 +103,17 @@ MemoryMonitor::MemoryMonitor(Lng32 windowSize,
 {
   // if the windowSize is 0, we do not need memory monitor.
   assert(windowSize);
-  char buffer[2048];
+  char buffer[FREAD_BUFFER_SIZE+1];
   char *currPtr;
-  size_t bytesRead;
+  size_t bytesRead = 0;
   fd_meminfo_ = fopen("/proc/meminfo", "r");
   if (fd_meminfo_) {
-bytesRead = fread(buffer, 1, 2048, fd_meminfo_);
+bytesRead = fread(buffer, 1, FREAD_BUFFER_SIZE, fd_meminfo_);
 if (ferror(fd_meminfo_))
assert(false); 
 if (feof(fd_meminfo_))
clearerr(fd_meminfo_); 
-else
-   buffer[bytesRead] = '\0';
+buffer[bytesRead] = '\0';
 currPtr = strstr(buffer, "MemTotal");
 if (currPtr) {
   sscanf(currPtr, "%*s " PF64 " kB", _);
@@ -152,7 +152,7 @@ MemoryMonitor::MemoryMonitor(Lng32 windowSize,
 {
   // and finally start the update thread
   updateThread_ = CreateNewThread(
-   , // Thread func
+   ::memMonitorUpdateThread, // Thread 
func
this );   // Argument for thread
   threadIsCreated_ = TRUE;
 }
@@ -222,19 +222,18 @@ void MemoryMonitor::update(float ) {
 Int32 success = fseek(fd_meminfo_, 0, SEEK_SET);
 if (success != 0)
 return;
-   char buffer[4096];
+   char buffer[FREAD_BUFFER_SIZE+1];
Int64 memFree = -1, memCommitAS = 0;
Int64 pgpgout = -1, pgpgin = -1;
-   size_t bytesRead;
+   size_t bytesRead = 0;
char * currPtr;
-bytesRead = fread(buffer, 1, 2048, fd_meminfo_);
+bytesRead = fread(buffer, 1, FREAD_BUFFER_SIZE, fd_meminfo_);
 // Make sure there wasn't an error (next fseek will clear eof)
 if (ferror(fd_meminfo_))
assert(false); 
 if