[ https://issues.apache.org/jira/browse/GEODE-2488?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Kirk Lund updated GEODE-2488: ----------------------------- Labels: MiscellaneousCommands NetstatCommand netstat (was: ) > NetstatDUnitTest fails with OutOfMemoryError > -------------------------------------------- > > Key: GEODE-2488 > URL: https://issues.apache.org/jira/browse/GEODE-2488 > Project: Geode > Issue Type: Bug > Components: gfsh, management > Reporter: Kirk Lund > Labels: MiscellaneousCommands, NetstatCommand, netstat > > The JUnit controller JVM for NetstatDUnitTest fails with something like the > follow stack. The cause appears to be one of the DUnit VMs running out of > memory (see the OOME stack down further in this description. > {noformat} > org.junit.ComparisonFailure: [{"errorCode":505,"message":["Could not process > command due to GemFire error. Error occurred while executing netstat on > [server-1]"]}] expected:<[OK]> but was:<[ERROR]> > at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) > at > sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) > at > sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) > at > org.apache.geode.test.dunit.rules.GfshShellConnectionRule.executeAndVerifyCommand(GfshShellConnectionRule.java:163) > at > org.apache.geode.management.internal.cli.NetstatDUnitTest.testConnectToJmxManagerOne(NetstatDUnitTest.java:81) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57) > at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71) > at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288) > at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) > at > org.apache.geode.test.junit.rules.DescribedExternalResource$1.evaluate(DescribedExternalResource.java:37) > at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48) > at org.junit.rules.RunRules.evaluate(RunRules.java:20) > at org.junit.runners.ParentRunner.run(ParentRunner.java:363) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.runTestClass(JUnitTestClassExecuter.java:114) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecuter.execute(JUnitTestClassExecuter.java:57) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassProcessor.processTestClass(JUnitTestClassProcessor.java:66) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51) > at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:32) > at > org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:93) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:109) > at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:35) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:377) > at > org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:54) > at > org.gradle.internal.concurrent.StoppableExecutorImpl$1.run(StoppableExecutorImpl.java:40) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > {noformat} > The JVM that runs out of memory ends up logging multiple OutOfMemoryError > stacks: > {noformat} > [vm1] 25.584: [GC (System.gc()) [PSYoungGen: 7724K->591K(161792K)] > 242411K->235286K(511488K), 0.0047912 secs] [Times: user=0.02 sys=0.00, > real=0.01 secs] > [vm1] 25.589: [Full GC (System.gc()) [PSYoungGen: 591K->0K(161792K)] > [ParOldGen: 234695K->9294K(349696K)] 235286K->9294K(511488K), [Metaspace: > 34478K->34478K(1081344K)], 0.0891637 secs] [Times: user=0.72 sys=0.01, > real=0.09 secs] > [vm1] [error 2017/02/14 09:36:13.738 UTC <Cache Server Acceptor > 0.0.0.0/0.0.0.0:36823 local port: 36823> tid=0x46] Cache server: Unexpected > IOException from accept > [vm1] [fatal 2017/02/14 09:36:13.739 UTC <unused p2p reader> tid=0x4f] > Uncaught exception in thread Thread[unused p2p reader,10,P2P Reader Threads] > [vm1] java.lang.OutOfMemoryError: Java heap space > [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664) > [vm1] at java.lang.String.<init>(String.java:207) > [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70) > [vm1] at > org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442) > [vm1] at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > [vm1] at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067) > [vm1] at java.lang.Thread.run(Thread.java:745) > [vm1] [fatal 2017/02/14 09:36:13.738 UTC <Function Execution Processor1> > tid=0x48] Uncaught exception in thread Thread[Function Execution > Processor1,10,DistributionManager Threads] > [vm1] java.lang.OutOfMemoryError: Java heap space > [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664) > [vm1] at java.lang.String.<init>(String.java:207) > [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70) > [vm1] at > org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442) > [vm1] at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > [vm1] at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067) > [vm1] at java.lang.Thread.run(Thread.java:745) > [vm1] [fatal 2017/02/14 09:36:13.740 UTC <Cache Server Acceptor > 0.0.0.0/0.0.0.0:36823 local port: 36823> tid=0x46] Uncaught exception in > thread Thread[Cache Server Acceptor 0.0.0.0/0.0.0.0:36823 local port: > 36823,5,Acceptor 0.0.0.0/0.0.0.0:36823] > [vm1] java.lang.OutOfMemoryError: Java heap space > [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664) > [vm1] at java.lang.String.<init>(String.java:207) > [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70) > [vm1] at > org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442) > [vm1] at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > [vm1] at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067) > [vm1] at java.lang.Thread.run(Thread.java:745) > [vm1] [fatal 2017/02/14 09:36:13.744 UTC <unused p2p reader> tid=0x47] > Uncaught exception in thread Thread[unused p2p reader,10,P2P Reader Threads] > [vm1] java.lang.OutOfMemoryError: Java heap space > [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664) > [vm1] at java.lang.String.<init>(String.java:207) > [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70) > [vm1] at > org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442) > [vm1] at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > [vm1] at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067) > [vm1] at java.lang.Thread.run(Thread.java:745) > [vm1] [fatal 2017/02/14 09:36:13.748 UTC <unused p2p reader> tid=0x54] > Uncaught exception in thread Thread[unused p2p reader,10,P2P Reader Threads] > [vm1] java.lang.OutOfMemoryError: Java heap space > [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664) > [vm1] at java.lang.String.<init>(String.java:207) > [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70) > [vm1] at > org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442) > [vm1] at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > [vm1] at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067) > [vm1] at java.lang.Thread.run(Thread.java:745) > [vm1] [fatal 2017/02/14 09:36:14.602 UTC <ClientHealthMonitor Thread> > tid=0x45] Uncaught exception in thread Thread[ClientHealthMonitor > Thread,5,ClientHealthMonitor Thread Group] > [vm1] java.lang.OutOfMemoryError: Java heap space > [vm1] at java.util.Arrays.copyOfRange(Arrays.java:3664) > [vm1] at java.lang.String.<init>(String.java:207) > [vm1] at java.lang.StringBuilder.toString(StringBuilder.java:407) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.executeCommand(NetstatFunction.java:193) > [vm1] at > org.apache.geode.management.internal.cli.functions.NetstatFunction.execute(NetstatFunction.java:70) > [vm1] at > org.apache.geode.internal.cache.MemberFunctionStreamingMessage.process(MemberFunctionStreamingMessage.java:191) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:376) > [vm1] at > org.apache.geode.distributed.internal.DistributionMessage$1.run(DistributionMessage.java:442) > [vm1] at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > [vm1] at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager.runUntilShutdown(DistributionManager.java:621) > [vm1] at > org.apache.geode.distributed.internal.DistributionManager$9$1.run(DistributionManager.java:1067) > [vm1] at java.lang.Thread.run(Thread.java:745) > {noformat} -- This message was sent by Atlassian JIRA (v6.3.15#6346)