Class Path:
\local\maven\repo\org\springframework\boot\spring-boot-starter-web\1.5.12.RELEASE\spring-boot-starter-web-1.5.12.RELEASE.jar
\local\maven\repo\org\springframework\boot\spring-boot-starter\1.5.12.RELEASE\spring-boot-starter-1.5.12.RELEASE.jar
\local\maven\repo\org\springframework\boot\spring-boot\1.5.12.RELEASE\spring-boot-1.5.12.RELEASE.jar
\local\maven\repo\org\springframework\boot\spring-boot-autoconfigure\1.5.12.RELEASE\spring-boot-autoconfigure-1.5.12.RELEASE.jar
\local\maven\repo\org\springframework\boot\spring-boot-starter-logging\1.5.12.RELEASE\spring-boot-starter-logging-1.5.12.RELEASE.jar
\local\maven\repo\ch\qos\logback\logback-classic\1.1.11\logback-classic-1.1.11.jar
\local\maven\repo\ch\qos\logback\logback-core\1.1.11\logback-core-1.1.11.jar
\local\maven\repo\org\slf4j\jul-to-slf4j\1.7.25\jul-to-slf4j-1.7.25.jar
\local\maven\repo\org\slf4j\log4j-over-slf4j\1.7.25\log4j-over-slf4j-1.7.25.jar
\local\maven\repo\org\yaml\snakeyaml\1.17\snakeyaml-1.17.jar
\local\maven\repo\org\hibernate\hibernate-validator\5.3.6.Final\hibernate-validator-5.3.6.Final.jar
\local\maven\repo\javax\validation\validation-api\1.1.0.Final\validation-api-1.1.0.Final.jar
\local\maven\repo\org\jboss\logging\jboss-logging\3.3.2.Final\jboss-logging-3.3.2.Final.jar
\local\maven\repo\com\fasterxml\classmate\1.3.4\classmate-1.3.4.jar
\local\maven\repo\com\fasterxml\jackson\core\jackson-databind\2.9.5\jackson-databind-2.9.5.jar
\local\maven\repo\com\fasterxml\jackson\core\jackson-core\2.9.5\jackson-core-2.9.5.jar
\local\maven\repo\org\springframework\spring-web\4.3.16.RELEASE\spring-web-4.3.16.RELEASE.jar
\local\maven\repo\org\springframework\spring-aop\4.3.16.RELEASE\spring-aop-4.3.16.RELEASE.jar
\local\maven\repo\org\springframework\spring-beans\4.3.16.RELEASE\spring-beans-4.3.16.RELEASE.jar
\local\maven\repo\org\springframework\spring-context\4.3.16.RELEASE\spring-context-4.3.16.RELEASE.jar
\local\maven\repo\org\springframework\spring-webmvc\4.3.16.RELEASE\spring-webmvc-4.3.16.RELEASE.jar
\local\maven\repo\org\springframework\spring-expression\4.3.16.RELEASE\spring-expression-4.3.16.RELEASE.jar
\local\maven\repo\org\springframework\boot\spring-boot-starter-jetty\1.5.12.RELEASE\spring-boot-starter-jetty-1.5.12.RELEASE.jar
\local\maven\repo\org\eclipse\jetty\jetty-servlets\9.4.9.v20180320\jetty-servlets-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-continuation\9.4.9.v20180320\jetty-continuation-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-http\9.4.9.v20180320\jetty-http-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-util\9.4.9.v20180320\jetty-util-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-io\9.4.9.v20180320\jetty-io-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-webapp\9.4.9.v20180320\jetty-webapp-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-xml\9.4.9.v20180320\jetty-xml-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-servlet\9.4.9.v20180320\jetty-servlet-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-security\9.4.9.v20180320\jetty-security-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-server\9.4.9.v20180320\jetty-server-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\websocket\websocket-server\9.4.9.v20180320\websocket-server-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\websocket\websocket-common\9.4.9.v20180320\websocket-common-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\websocket\websocket-api\9.4.9.v20180320\websocket-api-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\websocket\websocket-client\9.4.9.v20180320\websocket-client-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-client\9.4.9.v20180320\jetty-client-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\websocket\websocket-servlet\9.4.9.v20180320\websocket-servlet-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\websocket\javax-websocket-server-impl\9.4.9.v20180320\javax-websocket-server-impl-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-annotations\9.4.9.v20180320\jetty-annotations-9.4.9.v20180320.jar
\local\maven\repo\org\eclipse\jetty\jetty-plus\9.4.9.v20180320\jetty-plus-9.4.9.v20180320.jar
\local\maven\repo\javax\annotation\javax.annotation-api\1.2\javax.annotation-api-1.2.jar
\local\maven\repo\org\ow2\asm\asm\6.0\asm-6.0.jar
\local\maven\repo\org\ow2\asm\asm-commons\6.0\asm-commons-6.0.jar
\local\maven\repo\org\ow2\asm\asm-tree\6.0\asm-tree-6.0.jar
\local\maven\repo\org\eclipse\jetty\websocket\javax-websocket-client-impl\9.4.9.v20180320\javax-websocket-client-impl-9.4.9.v20180320.jar
\local\maven\repo\javax\websocket\javax.websocket-api\1.0\javax.websocket-api-1.0.jar
\local\maven\repo\org\mortbay\jasper\apache-el\8.0.33\apache-el-8.0.33.jar
\local\maven\repo\org\springframework\data\spring-data-geode\1.0.0.INCUBATING-RELEASE\spring-data-geode-1.0.0.INCUBATING-RELEASE.jar
\local\maven\repo\org\springframework\spring-tx\4.3.16.RELEASE\spring-tx-4.3.16.RELEASE.jar
\local\maven\repo\org\springframework\data\spring-data-commons\1.13.11.RELEASE\spring-data-commons-1.13.11.RELEASE.jar
\local\maven\repo\antlr\antlr\2.7.7\antlr-2.7.7.jar
\local\maven\repo\org\aspectj\aspectjweaver\1.8.13\aspectjweaver-1.8.13.jar
\local\maven\repo\org\apache\shiro\shiro-spring\1.3.1\shiro-spring-1.3.1.jar
\local\maven\repo\org\apache\shiro\shiro-web\1.3.1\shiro-web-1.3.1.jar
\local\maven\repo\com\fasterxml\jackson\core\jackson-annotations\2.9.0\jackson-annotations-2.9.0.jar
\local\maven\repo\org\slf4j\jcl-over-slf4j\1.7.25\jcl-over-slf4j-1.7.25.jar
\local\maven\repo\org\slf4j\slf4j-api\1.7.25\slf4j-api-1.7.25.jar
\local\maven\repo\org\apache\geode\geode-core\1.1.1\geode-core-1.1.1.jar
\local\maven\repo\com\github\stephenc\findbugs\findbugs-annotations\1.3.9-1\findbugs-annotations-1.3.9-1.jar
\local\maven\repo\org\jgroups\jgroups\3.6.10.Final\jgroups-3.6.10.Final.jar
\local\maven\repo\commons-lang\commons-lang\2.5\commons-lang-2.5.jar
\local\maven\repo\it\unimi\dsi\fastutil\7.0.2\fastutil-7.0.2.jar
\local\maven\repo\javax\resource\javax.resource-api\1.7\javax.resource-api-1.7.jar
\local\maven\repo\javax\transaction\javax.transaction-api\1.2\javax.transaction-api-1.2.jar
\local\maven\repo\net\java\dev\jna\jna\4.2.2\jna-4.2.2.jar
\local\maven\repo\net\sf\jopt-simple\jopt-simple\5.0.1\jopt-simple-5.0.1.jar
\local\maven\repo\org\apache\logging\log4j\log4j-api\2.7\log4j-api-2.7.jar
\local\maven\repo\org\apache\logging\log4j\log4j-core\2.7\log4j-core-2.7.jar
\local\maven\repo\org\apache\shiro\shiro-core\1.3.1\shiro-core-1.3.1.jar
\local\maven\repo\commons-beanutils\commons-beanutils\1.9.3\commons-beanutils-1.9.3.jar
\local\maven\repo\org\apache\geode\geode-common\1.1.1\geode-common-1.1.1.jar
\local\maven\repo\org\apache\geode\geode-json\1.1.1\geode-json-1.1.1.jar
\local\maven\repo\org\apache\geode\geode-cq\1.1.1\geode-cq-1.1.1.jar
\local\maven\repo\org\apache\geode\geode-wan\1.1.1\geode-wan-1.1.1.jar
\local\maven\repo\org\springframework\boot\spring-boot-starter-data-jpa\1.5.12.RELEASE\spring-boot-starter-data-jpa-1.5.12.RELEASE.jar
\local\maven\repo\org\springframework\boot\spring-boot-starter-aop\1.5.12.RELEASE\spring-boot-starter-aop-1.5.12.RELEASE.jar
\local\maven\repo\org\hibernate\hibernate-entitymanager\5.0.12.Final\hibernate-entitymanager-5.0.12.Final.jar
\local\maven\repo\org\springframework\data\spring-data-jpa\1.11.11.RELEASE\spring-data-jpa-1.11.11.RELEASE.jar
\local\maven\repo\org\springframework\spring-orm\4.3.16.RELEASE\spring-orm-4.3.16.RELEASE.jar
\local\maven\repo\org\springframework\spring-aspects\4.3.16.RELEASE\spring-aspects-4.3.16.RELEASE.jar
\local\maven\repo\org\hibernate\hibernate-envers\5.0.12.Final\hibernate-envers-5.0.12.Final.jar
\local\maven\repo\com\github\ulisesbocchio\jasypt-spring-boot-starter\1.11\jasypt-spring-boot-starter-1.11.jar
\local\maven\repo\com\github\ulisesbocchio\jasypt-spring-boot\1.11\jasypt-spring-boot-1.11.jar
\local\maven\repo\org\jasypt\jasypt\1.9.2\jasypt-1.9.2.jar
\local\maven\repo\org\springframework\boot\spring-boot-starter-jdbc\1.5.12.RELEASE\spring-boot-starter-jdbc-1.5.12.RELEASE.jar
\local\maven\repo\org\apache\tomcat\tomcat-jdbc\8.5.29\tomcat-jdbc-8.5.29.jar
\local\maven\repo\org\apache\tomcat\tomcat-juli\8.5.29\tomcat-juli-8.5.29.jar
\local\maven\repo\org\springframework\spring-jdbc\4.3.16.RELEASE\spring-jdbc-4.3.16.RELEASE.jar
\local\maven\repo\org\hibernate\hibernate-core\5.0.12.Final\hibernate-core-5.0.12.Final.jar
\local\maven\repo\org\hibernate\javax\persistence\hibernate-jpa-2.1-api\1.0.0.Final\hibernate-jpa-2.1-api-1.0.0.Final.jar
\local\maven\repo\org\javassist\javassist\3.21.0-GA\javassist-3.21.0-GA.jar
\local\maven\repo\org\apache\geronimo\specs\geronimo-jta_1.1_spec\1.1.1\geronimo-jta_1.1_spec-1.1.1.jar
\local\maven\repo\org\jboss\jandex\2.0.0.Final\jandex-2.0.0.Final.jar
\local\maven\repo\dom4j\dom4j\1.6.1\dom4j-1.6.1.jar
\local\maven\repo\org\hibernate\common\hibernate-commons-annotations\5.0.1.Final\hibernate-commons-annotations-5.0.1.Final.jar
\local\maven\repo\commons-io\commons-io\2.6\commons-io-2.6.jar
\local\maven\repo\oracle\jdbc\ojdbc7\12.1.0.2.0\ojdbc7-12.1.0.2.0.jar
\local\maven\repo\javax\inject\javax.inject\1\javax.inject-1.jar
\local\maven\repo\org\springframework\spring-oxm\4.3.16.RELEASE\spring-oxm-4.3.16.RELEASE.jar
\local\maven\repo\org\objenesis\objenesis\2.6\objenesis-2.6.jar
\local\maven\repo\org\codehaus\jettison\jettison\1.3.8\jettison-1.3.8.jar
\local\maven\repo\com\thoughtworks\xstream\xstream\1.4.9\xstream-1.4.9.jar
\local\maven\repo\xmlpull\xmlpull\1.1.3.1\xmlpull-1.1.3.1.jar
\local\maven\repo\xpp3\xpp3_min\1.1.4c\xpp3_min-1.1.4c.jar
\local\maven\repo\org\owasp\antisamy\antisamy\1.5.1\antisamy-1.5.1.jar
\local\maven\repo\commons-httpclient\commons-httpclient\3.1\commons-httpclient-3.1.jar
\local\maven\repo\xml-apis\xml-apis\1.4.01\xml-apis-1.4.01.jar
\local\maven\repo\xerces\xercesImpl\2.11.0.SP4\xercesImpl-2.11.0.SP4.jar
\local\maven\repo\net\sourceforge\nekohtml\nekohtml\1.9.22\nekohtml-1.9.22.jar
\local\maven\repo\commons-collections\commons-collections\3.2.2\commons-collections-3.2.2.jar
\local\maven\repo\commons-beanutils\commons-beanutils-core\1.8.2\commons-beanutils-core-1.8.2.jar
\local\maven\repo\commons-logging\commons-logging\1.2\commons-logging-1.2.jar
\local\maven\repo\commons-codec\commons-codec\1.10\commons-codec-1.10.jar
\local\maven\repo\log4j\log4j\1.2.16\log4j-1.2.16.jar
\local\maven\repo\javax\servlet\javax.servlet-api\3.1.0\javax.servlet-api-3.1.0.jar
\local\maven\repo\xalan\xalan\2.7.2\xalan-2.7.2.jar
\local\maven\repo\xalan\serializer\2.7.2\serializer-2.7.2.jar
\local\maven\repo\org\apache\xmlgraphics\batik-css\1.8\batik-css-1.8.jar
\local\maven\repo\xml-apis\xml-apis-ext\1.3.04\xml-apis-ext-1.3.04.jar
\local\maven\repo\org\apache\xmlgraphics\batik-ext\1.8\batik-ext-1.8.jar
\local\maven\repo\org\apache\xmlgraphics\batik-util\1.8\batik-util-1.8.jar
\local\maven\repo\commons-fileupload\commons-fileupload\1.3.3\commons-fileupload-1.3.3.jar
\local\maven\repo\org\owasp\esapi\esapi\2.1.0\esapi-2.1.0.jar
\local\maven\repo\commons-configuration\commons-configuration\1.5\commons-configuration-1.5.jar
\local\maven\repo\commons-digester\commons-digester\2.1\commons-digester-2.1.jar
\local\maven\repo\xom\xom\1.2.5\xom-1.2.5.jar
\local\maven\repo\org\beanshell\bsh-core\2.0b4\bsh-core-2.0b4.jar
\local\maven\repo\org\springframework\spring-core\4.3.16.RELEASE\spring-core-4.3.16.RELEASE.jar
\local\maven\repo\org\springframework\boot\spring-boot-starter-actuator\1.5.12.RELEASE\spring-boot-starter-actuator-1.5.12.RELEASE.jar
\local\maven\repo\org\springframework\boot\spring-boot-actuator\1.5.12.RELEASE\spring-boot-actuator-1.5.12.RELEASE.jar
Hi Anthony,
I have attached classpath portion of log in attachment.
It’s true that I am booting up geode server using spring boot and
spring-data-geode.
But I see all geode-* jars in classpath as expected.
In local, I could regenerate this issue as well.
Option1: I don’t have enough number of tombstones to trigger GC (100,000
default) but server memory dropped below 30% which should have triggered this.
Option2: Simulate force disconnect of member from distributed system and let
server member make an attempt to rejoin distributed system. Once it’s done,
load 100K dummy records and destroy them in sometime with REPLICATED mode, 2
peers.
Thanks,
Dharam
From: Charlie Black [mailto:[email protected]]
Sent: Thursday, July 05, 2018 10:29 PM
To: [email protected]
Subject: Re: Issue with TombstoneService
Are we redeploying the geode jar to the jvm? Or have some kind of interesting
classloader?
On Thu, Jul 5, 2018 at 8:36 AM Anthony Baker
<[email protected]<mailto:[email protected]>> wrote:
Thanks for the error report. Can you share the portion of the log that dumps
the classpath? It looks like you’re starting geode with springboot and perhaps
the classpath is incorrect. If geode-*.jar is on the classpath I don’t see how
you could get this error.
Anthony
On Jul 4, 2018, at 12:16 AM, Thacker, Dharam
<[email protected]<mailto:[email protected]>> wrote:
Hello Team,
Today we encountered below issue with Geode 1.1.1 (Should exists in Geode 1.6.0
as well as per codebase).
[severe 2018/07/03 11:03:53.914 EDT event-server-2 <Replicate/Partition Region
Garbage Collector> tid=0x4d] GemFire garbage collection service encountered an
unexpected exception
java.lang.NoClassDefFoundError:
org/apache/geode/internal/cache/TombstoneService$ReplicateTombstoneSweeper$1
at
org.apache.geode.internal.cache.TombstoneService$ReplicateTombstoneSweeper.expireBatch(TombstoneService.java:566)
at
org.apache.geode.internal.cache.TombstoneService$ReplicateTombstoneSweeper.checkExpiredTombstoneGC(TombstoneService.java:596)
at
org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.run(TombstoneService.java:882)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.ClassNotFoundException:
org.apache.geode.internal.cache.TombstoneService$ReplicateTombstoneSweeper$1
at
java.net<https://secureweb.jpmchase.net/readonly/http:/java.net>.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at
org.springframework.boot.loader.LaunchedURLClassLoader.loadClass(LaunchedURLClassLoader.java:94)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
This makes sense to me as ReplicateTombstoneSweeper is a private static class
within TombstoneService.
Could you also verify at your end?
Another one as below which I think should not come with 1.6.0 as ThreadUtils
has been deprecated/removed from newer versions.
[severe 2018/07/03 11:03:53.916 EDT event-server-2 <Replicate/Partition Region
Garbage Collector> tid=0x4d] Uncaught exception in thread
Thread[Replicate/Partition Region Garbage Collector,5,Destroyed Entries
Processors]
java.lang.NoClassDefFoundError: org/apache/geode/internal/lang/ThreadUtils
at
org.apache.geode.internal.logging.log4j.AlertAppender.append(AlertAppender.java:141)
at
org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
at
org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
at
org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
at
org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
at
org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:447)
at
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:432)
at
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:416)
at
org.apache.logging.log4j.core.config.LoggerConfig.logParent(LoggerConfig.java:438)
at
org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
at
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:416)
at
org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:402)
at
org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
at
org.apache.logging.log4j.spi.ExtendedLoggerWrapper.logMessage(ExtendedLoggerWrapper.java:217)
at
org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2091)
at
org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1813)
at
org.apache.logging.log4j.spi.AbstractLogger.fatal(AbstractLogger.java:1005)
at
org.apache.geode.internal.cache.TombstoneService$TombstoneSweeper.run(TombstoneService.java:895)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.ClassNotFoundException:
org.apache.geode.internal.lang.ThreadUtils
Thanks,
Dharam
This message is confidential and subject to terms at:
http://www.jpmorgan.com/emaildisclaimer<http://www.jpmorgan.com/emaildisclaimer>
including on confidentiality, legal privilege, viruses and monitoring of
electronic messages. If you are not the intended recipient, please delete this
message and notify the sender immediately. Any unauthorized use is strictly
prohibited.
--
[email protected]<mailto:[email protected]> | +1.858.480.9722
Principal Realtime Data Engineer
This message is confidential and subject to terms at:
http://www.jpmorgan.com/emaildisclaimer including on confidentiality, legal
privilege, viruses and monitoring of electronic messages. If you are not the
intended recipient, please delete this message and notify the sender
immediately. Any unauthorized use is strictly prohibited.
