[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15052643#comment-15052643 ] Michael McCandless commented on LUCENE-6629: Again? http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/75411/ {noformat} [junit4] Suite: org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3 [junit4] 2> 12? 11, 2015 2:18:40 ?? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3 [junit4] 2>1) Thread[id=11, name=TEST-TestBlockPostingsFormat3.test-seed#[45106F11E17A4109], state=RUNNABLE, group=TGRP-TestBlockPostingsFormat3] [junit4] 2> at org.apache.lucene.codecs.blocktree.SegmentTermsEnum.next(SegmentTermsEnum.java:894) [junit4] 2> at org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3.assertTermsSeeking(TestBlockPostingsFormat3.java:211) [junit4] 2> at org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3.assertTerms(TestBlockPostingsFormat3.java:180) [junit4] 2> at org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3.verify(TestBlockPostingsFormat3.java:153) [junit4] 2> at org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat3.test(TestBlockPostingsFormat3.java:145) [junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) [junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2> at java.lang.reflect.Method.invoke(Method.java:606) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921) [junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50) [junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46) [junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49) [junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65) [junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827) [junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=15037691#comment-15037691 ] Michael McCandless commented on LUCENE-6629: Maybe another one? http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/74226/ {noformat} [junit4] Suite: org.apache.lucene.codecs.lucene41.TestLucene41StoredFieldsFormat [junit4] 2> gru 03, 2015 12:32:30 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.codecs.lucene41.TestLucene41StoredFieldsFormat [junit4] 2>1) Thread[id=70, name=SUITE-TestLucene41StoredFieldsFormat-seed#[1B32743AE8AA0DDD], state=RUNNABLE, group=TGRP-TestLucene41StoredFieldsFormat] [junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1552) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:688) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:685) [junit4] 2> at java.security.AccessController.doPrivileged(Native Method) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:685) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:701) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:681) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:64) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:414) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:681) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:591) [junit4] 2>2) Thread[id=10, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main] [junit4] 2> at java.lang.Thread.sleep(Native Method) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47) [junit4] 2>3) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2> at java.lang.Object.wait(Native Method) [junit4] 2> at java.lang.Thread.join(Thread.java:1245) [junit4] 2> at java.lang.Thread.join(Thread.java:1319) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:601) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:450) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:244) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:10) [junit4] 2>4) Thread[id=71, name=TEST-TestLucene41StoredFieldsFormat.testMergeLargeDocuments-seed#[1B32743AE8AA0DDD], state=RUNNABLE, group=TGRP-TestLucene41StoredFieldsFormat] [junit4] 2> at org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164) [junit4] 2> at org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsWriter.flush(Lucene41StoredFieldsWriter.java:218) [junit4] 2> at org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsWriter.finishDocument(Lucene41StoredFieldsWriter.java:151) [junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:270) [junit4] 2> at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:311) [junit4] 2> at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) [junit4] 2> at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) [junit4] 2> at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) [junit4] 2> at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1256) [junit4] 2> at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:170) [junit4] 2> at org.apache.lucene.codecs.lucene41.TestLucene41StoredFieldsFormat.testMergeLargeDocuments(TestLucene41StoredFieldsFormat.java:70) [junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2> at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=14734503#comment-14734503 ] Michael McCandless commented on LUCENE-6629: Another one: http://jenkins.thetaphi.de/job/Lucene-Solr-5.x-Linux/13870/ {noformat} Suite: org.apache.lucene.analysis.util.TestCharArrayMap [junit4] 2> сеп 07, 2015 11:56:46 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.analysis.util.TestCharArrayMap [junit4] 2>1) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2> at java.lang.Object.wait(Native Method) [junit4] 2> at java.lang.Thread.join(Thread.java:1245) [junit4] 2> at java.lang.Thread.join(Thread.java:1319) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 2>2) Thread[id=11, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main] [junit4] 2> at java.lang.Thread.sleep(Native Method) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47) [junit4] 2>3) Thread[id=840, name=TEST-TestCharArrayMap.testCharArrayMap-seed#[9E827E74BAD348B7], state=RUNNABLE, group=TGRP-TestCharArrayMap] [junit4] 2> at org.apache.lucene.analysis.util.CharArrayMap.getSlot(CharArrayMap.java:166) [junit4] 2> at org.apache.lucene.analysis.util.CharArrayMap.get(CharArrayMap.java:128) [junit4] 2> at org.apache.lucene.analysis.util.TestCharArrayMap.doRandom(TestCharArrayMap.java:52) [junit4] 2> at org.apache.lucene.analysis.util.TestCharArrayMap.testCharArrayMap(TestCharArrayMap.java:62) [junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2> at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886) [junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50) [junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46) [junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49) [junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65) [junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48) [junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792) [junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46) [junit4] 2> at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=14734549#comment-14734549 ] Michael McCandless commented on LUCENE-6629: Another one: http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/62917 {noformat} [junit4] Suite: org.apache.lucene.search.TestDocValuesRangeQuery [junit4] 2> 9 08, 2015 10:21:29 ?? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2> WARNING: Suite execution timed out: org.apache.lucene.search.TestDocValuesRangeQuery [junit4] 2>1) Thread[id=11, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main] [junit4] 2> at java.lang.Thread.sleep(Native Method) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47) [junit4] 2>2) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2> at java.lang.Object.wait(Native Method) [junit4] 2> at java.lang.Thread.join(Thread.java:1245) [junit4] 2> at java.lang.Thread.join(Thread.java:1319) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2> at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 2>3) Thread[id=14, name=SUITE-TestDocValuesRangeQuery-seed#[57620327D30425D5], state=RUNNABLE, group=TGRP-TestDocValuesRangeQuery] [junit4] 2> at java.lang.Thread.getStackTrace(Thread.java:1552) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:690) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:679) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$900(ThreadLeakControl.java:62) [junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:412) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:651) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:138) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$1.run(RandomizedRunner.java:568) [junit4] 2>4) Thread[id=15, name=TEST-TestDocValuesRangeQuery.testScore-seed#[57620327D30425D5], state=RUNNABLE, group=TGRP-TestDocValuesRangeQuery] [junit4] 2> at org.apache.lucene.search.TopFieldCollector.populateResults(TopFieldCollector.java:537) [junit4] 2> at org.apache.lucene.search.TopDocsCollector.topDocs(TopDocsCollector.java:156) [junit4] 2> at org.apache.lucene.search.TopDocsCollector.topDocs(TopDocsCollector.java:93) [junit4] 2> at org.apache.lucene.search.TopFieldCollector.topDocs(TopFieldCollector.java:561) [junit4] 2> at org.apache.lucene.search.IndexSearcher$4.reduce(IndexSearcher.java:696) [junit4] 2> at org.apache.lucene.search.IndexSearcher$4.reduce(IndexSearcher.java:683) [junit4] 2> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:719) [junit4] 2> at org.apache.lucene.search.IndexSearcher.searchAfter(IndexSearcher.java:703) [junit4] 2> at org.apache.lucene.search.IndexSearcher.searchAfter(IndexSearcher.java:645) [junit4] 2> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:551) [junit4] 2> at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:601) [junit4] 2> at org.apache.lucene.search.TestDocValuesRangeQuery.assertSameMatches(TestDocValuesRangeQuery.java:230) [junit4] 2> at org.apache.lucene.search.TestDocValuesRangeQuery.testScore(TestDocValuesRangeQuery.java:169) [junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2> at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627) [junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836) [junit4] 2>
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14663006#comment-14663006 ] Michael McCandless commented on LUCENE-6629: Another strange timeout: http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/58384/console {noformat} [junit4] Suite: org.apache.lucene.search.suggest.TestInputIterator [junit4] 2 авг 08, 2015 3:57:45 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.search.suggest.TestInputIterator [junit4] 21) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1245) [junit4] 2 at java.lang.Thread.join(Thread.java:1319) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 22) Thread[id=12, name=SUITE-TestInputIterator-seed#[4568F9BCA17391B], state=RUNNABLE, group=TGRP-TestInputIterator] [junit4] 2 at java.lang.Thread.getStackTrace(Thread.java:1552) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:690) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:679) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$900(ThreadLeakControl.java:62) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:412) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:651) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:138) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$1.run(RandomizedRunner.java:568) [junit4] 23) Thread[id=11, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main] [junit4] 2 at java.lang.Thread.sleep(Native Method) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47) [junit4] 24) Thread[id=13, name=TEST-TestInputIterator.testTerms-seed#[4568F9BCA17391B], state=RUNNABLE, group=TGRP-TestInputIterator] [junit4] 2 at org.apache.lucene.search.suggest.TestInputIterator.testTerms(TestInputIterator.java:143) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886) [junit4] 2 at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50) [junit4] 2 at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46) [junit4] 2 at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49) [junit4] 2 at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65) [junit4] 2 at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48) [junit4] 2 at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14648940#comment-14648940 ] Michael McCandless commented on LUCENE-6629: Woops, you're right [~dawid.weiss]! It also stalls for me ... so this is just SimpleText being slow (as expected). Random 7200 seconds build timeouts / infinite loops in Lucene tests? Key: LUCENE-6629 URL: https://issues.apache.org/jira/browse/LUCENE-6629 Project: Lucene - Core Issue Type: Bug Reporter: Michael McCandless Attachments: 54457_consoleText.txt I'm not sure what's going on here, but every so often a Jenkins run will fail with a build timeout (7200 seconds) with stack traces that do not look like deadlock. They never reproduce for me. We really need to improve test infra here, so that each HEARTBEAT we also got 1) full thread stacks and 2) total CPU usage of the process as reported by the ManagementBean APIs ... this would shed more light on whether the JVM is somehow hung vs our bug (infinite loop). But infinite loop seems most likely ... the stacks always seem to be somewhere spooky. We should try to gather recent Jenkins runs where this is happening, here, to see if there are patterns / we can get to the root cause. Anyway, this happened to me on my old beast box, which runs the nightly ant test times graphs: http://people.apache.org/~mikemccand/lucenebench/antcleantest.html The 2015/06/27 data point is missing because it failed with this timeout: {noformat} [junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 2 ??? 28, 2015 7:01:29 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 21) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1245) [junit4] 2 at java.lang.Thread.join(Thread.java:1319) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 22) Thread[id=213, name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8], state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod] [junit4] 2 at org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900) [junit4] 2 at org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389) [junit4] 2 at org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14648919#comment-14648919 ] Dawid Weiss commented on LUCENE-6629: - To me it looks like it's just churning a lot of data and using simple text codec? I tried running on of the repros locally and it also stalls. Random 7200 seconds build timeouts / infinite loops in Lucene tests? Key: LUCENE-6629 URL: https://issues.apache.org/jira/browse/LUCENE-6629 Project: Lucene - Core Issue Type: Bug Reporter: Michael McCandless Attachments: 54457_consoleText.txt I'm not sure what's going on here, but every so often a Jenkins run will fail with a build timeout (7200 seconds) with stack traces that do not look like deadlock. They never reproduce for me. We really need to improve test infra here, so that each HEARTBEAT we also got 1) full thread stacks and 2) total CPU usage of the process as reported by the ManagementBean APIs ... this would shed more light on whether the JVM is somehow hung vs our bug (infinite loop). But infinite loop seems most likely ... the stacks always seem to be somewhere spooky. We should try to gather recent Jenkins runs where this is happening, here, to see if there are patterns / we can get to the root cause. Anyway, this happened to me on my old beast box, which runs the nightly ant test times graphs: http://people.apache.org/~mikemccand/lucenebench/antcleantest.html The 2015/06/27 data point is missing because it failed with this timeout: {noformat} [junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 2 ??? 28, 2015 7:01:29 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 21) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1245) [junit4] 2 at java.lang.Thread.join(Thread.java:1319) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 22) Thread[id=213, name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8], state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod] [junit4] 2 at org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900) [junit4] 2 at org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389) [junit4] 2 at org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14648195#comment-14648195 ] Michael McCandless commented on LUCENE-6629: Maybe another example: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/915/console {noformat} Suite: org.apache.lucene.codecs.simpletext.TestSimpleTextTermVectorsFormat [junit4] 2 jul 30, 2015 2:46:47 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.codecs.simpletext.TestSimpleTextTermVectorsFormat [junit4] 21) Thread[id=147, name=TEST-TestSimpleTextTermVectorsFormat.testRamBytesUsed-seed#[4AF0D6F4E5896BB7], state=RUNNABLE, group=TGRP-TestSimpleTextTermVectorsFormat] [junit4] 2 at org.apache.lucene.store.MockIndexInputWrapper.readByte(MockIndexInputWrapper.java:132) [junit4] 2 at org.apache.lucene.codecs.simpletext.SimpleTextUtil.readLine(SimpleTextUtil.java:60) [junit4] 2 at org.apache.lucene.codecs.simpletext.SimpleTextTermVectorsReader.readLine(SimpleTextTermVectorsReader.java:234) [junit4] 2 at org.apache.lucene.codecs.simpletext.SimpleTextTermVectorsReader.get(SimpleTextTermVectorsReader.java:141) [junit4] 2 at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:194) [junit4] 2 at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:187) [junit4] 2 at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:127) [junit4] 2 at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4089) [junit4] 2 at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3664) [junit4] 2 at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40) [junit4] 2 at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1929) [junit4] 2 at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4731) [junit4] 2 at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:695) [junit4] 2 at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4757) [junit4] 2 at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4748) [junit4] 2 at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1476) [junit4] 2 at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1254) [junit4] 2 at org.apache.lucene.index.BaseIndexFileFormatTestCase.testRamBytesUsed(BaseIndexFileFormatTestCase.java:260) [junit4] 2 at org.apache.lucene.index.BaseTermVectorsFormatTestCase.testRamBytesUsed(BaseTermVectorsFormatTestCase.java:74) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:606) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886) [junit4] 2 at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50) [junit4] 2 at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46) [junit4] 2 at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49) [junit4] 2 at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65) [junit4] 2 at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48) [junit4] 2 at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14621227#comment-14621227 ] Michael McCandless commented on LUCENE-6629: bq. Can we have the 1.9ea JVM upgraded to b71 on that machine? Maybe it's a JVM issue that's been solved already. The hangs seem to happen with 1.8.0_40 and 1.9ea... Random 7200 seconds build timeouts / infinite loops in Lucene tests? Key: LUCENE-6629 URL: https://issues.apache.org/jira/browse/LUCENE-6629 Project: Lucene - Core Issue Type: Bug Reporter: Michael McCandless Attachments: 54457_consoleText.txt I'm not sure what's going on here, but every so often a Jenkins run will fail with a build timeout (7200 seconds) with stack traces that do not look like deadlock. They never reproduce for me. We really need to improve test infra here, so that each HEARTBEAT we also got 1) full thread stacks and 2) total CPU usage of the process as reported by the ManagementBean APIs ... this would shed more light on whether the JVM is somehow hung vs our bug (infinite loop). But infinite loop seems most likely ... the stacks always seem to be somewhere spooky. We should try to gather recent Jenkins runs where this is happening, here, to see if there are patterns / we can get to the root cause. Anyway, this happened to me on my old beast box, which runs the nightly ant test times graphs: http://people.apache.org/~mikemccand/lucenebench/antcleantest.html The 2015/06/27 data point is missing because it failed with this timeout: {noformat} [junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 2 ??? 28, 2015 7:01:29 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 21) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1245) [junit4] 2 at java.lang.Thread.join(Thread.java:1319) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 22) Thread[id=213, name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8], state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod] [junit4] 2 at org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900) [junit4] 2 at org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389) [junit4] 2 at org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14621225#comment-14621225 ] Michael McCandless commented on LUCENE-6629: Another confusing build hang: http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/55047/ {noformat} [junit4] 2 NOTE: reproduce with: ant test -Dtestcase=TestJapaneseAnalyzer -Dtests.method=test5thCuriousString -Dtests.seed=4C20E5ADD6D736D1 -Dtests.slow=true -Dtests.locale=pl_PL -Dtests.timezone=Indian/Mahe -Dtests.asserts=true -Dtests.file.encoding=US-ASCII [junit4] ERROR 7197s J2 | TestJapaneseAnalyzer.test5thCuriousString [junit4] Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached. [junit4]at __randomizedtesting.SeedInfo.seed([4C20E5ADD6D736D1]:0) [junit4] 2 lip 09, 2015 5:49:51 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2 WARNING: Will linger awaiting termination of 1 leaked thread(s). [junit4] 2 lip 09, 2015 5:50:11 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks [junit4] 2 SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.analysis.ja.TestJapaneseAnalyzer: [junit4] 21) Thread[id=13, name=TEST-TestJapaneseAnalyzer.test5thCuriousString-seed#[4C20E5ADD6D736D1], state=RUNNABLE, group=TGRP-TestJapaneseAnalyzer] [junit4] 2 at org.apache.lucene.analysis.ja.JapaneseTokenizer.backtrace(JapaneseTokenizer.java:1056) [junit4] 2 at org.apache.lucene.analysis.ja.JapaneseTokenizer.parse(JapaneseTokenizer.java:647) [junit4] 2 at org.apache.lucene.analysis.ja.JapaneseTokenizer.incrementToken(JapaneseTokenizer.java:474) [junit4] 2 at org.apache.lucene.analysis.ja.JapaneseBaseFormFilter.incrementToken(JapaneseBaseFormFilter.java:50) [junit4] 2 at org.apache.lucene.analysis.util.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:51) [junit4] 2 at org.apache.lucene.analysis.cjk.CJKWidthFilter.incrementToken(CJKWidthFilter.java:63) [junit4] 2 at org.apache.lucene.analysis.util.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:51) [junit4] 2 at org.apache.lucene.analysis.ja.JapaneseKatakanaStemFilter.incrementToken(JapaneseKatakanaStemFilter.java:63) [junit4] 2 at org.apache.lucene.analysis.core.LowerCaseFilter.incrementToken(LowerCaseFilter.java:45) [junit4] 2 at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:706) [junit4] 2 at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:680) [junit4] 2 at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:676) [junit4] 2 at org.apache.lucene.analysis.ja.TestJapaneseAnalyzer.test5thCuriousString(TestJapaneseAnalyzer.java:217) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886) [junit4] 2 at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50) [junit4] 2 at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46) [junit4] 2 at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49) [junit4] 2 at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65) [junit4] 2 at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48) [junit4] 2 at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14618825#comment-14618825 ] Dawid Weiss commented on LUCENE-6629: - Can we have the 1.9ea JVM upgraded to b71 on that machine? Maybe it's a JVM issue that's been solved already. Random 7200 seconds build timeouts / infinite loops in Lucene tests? Key: LUCENE-6629 URL: https://issues.apache.org/jira/browse/LUCENE-6629 Project: Lucene - Core Issue Type: Bug Reporter: Michael McCandless Attachments: 54457_consoleText.txt I'm not sure what's going on here, but every so often a Jenkins run will fail with a build timeout (7200 seconds) with stack traces that do not look like deadlock. They never reproduce for me. We really need to improve test infra here, so that each HEARTBEAT we also got 1) full thread stacks and 2) total CPU usage of the process as reported by the ManagementBean APIs ... this would shed more light on whether the JVM is somehow hung vs our bug (infinite loop). But infinite loop seems most likely ... the stacks always seem to be somewhere spooky. We should try to gather recent Jenkins runs where this is happening, here, to see if there are patterns / we can get to the root cause. Anyway, this happened to me on my old beast box, which runs the nightly ant test times graphs: http://people.apache.org/~mikemccand/lucenebench/antcleantest.html The 2015/06/27 data point is missing because it failed with this timeout: {noformat} [junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 2 ??? 28, 2015 7:01:29 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 21) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1245) [junit4] 2 at java.lang.Thread.join(Thread.java:1319) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 22) Thread[id=213, name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8], state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod] [junit4] 2 at org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900) [junit4] 2 at org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389) [junit4] 2 at org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14618922#comment-14618922 ] Uwe Schindler commented on LUCENE-6629: --- Policeman Jenkins has Java 9 EA b71 already. Random 7200 seconds build timeouts / infinite loops in Lucene tests? Key: LUCENE-6629 URL: https://issues.apache.org/jira/browse/LUCENE-6629 Project: Lucene - Core Issue Type: Bug Reporter: Michael McCandless Attachments: 54457_consoleText.txt I'm not sure what's going on here, but every so often a Jenkins run will fail with a build timeout (7200 seconds) with stack traces that do not look like deadlock. They never reproduce for me. We really need to improve test infra here, so that each HEARTBEAT we also got 1) full thread stacks and 2) total CPU usage of the process as reported by the ManagementBean APIs ... this would shed more light on whether the JVM is somehow hung vs our bug (infinite loop). But infinite loop seems most likely ... the stacks always seem to be somewhere spooky. We should try to gather recent Jenkins runs where this is happening, here, to see if there are patterns / we can get to the root cause. Anyway, this happened to me on my old beast box, which runs the nightly ant test times graphs: http://people.apache.org/~mikemccand/lucenebench/antcleantest.html The 2015/06/27 data point is missing because it failed with this timeout: {noformat} [junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 2 ??? 28, 2015 7:01:29 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 21) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1245) [junit4] 2 at java.lang.Thread.join(Thread.java:1319) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 22) Thread[id=213, name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8], state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod] [junit4] 2 at org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900) [junit4] 2 at org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389) [junit4] 2 at org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14606016#comment-14606016 ] Robert Muir commented on LUCENE-6629: - I saw that, but its on a 1.9EA build and we have encountered some arraycopy issues there? I don't think we should worry too much on those until we have a more recent build in jenkins. Random 7200 seconds build timeouts / infinite loops in Lucene tests? Key: LUCENE-6629 URL: https://issues.apache.org/jira/browse/LUCENE-6629 Project: Lucene - Core Issue Type: Bug Reporter: Michael McCandless I'm not sure what's going on here, but every so often a Jenkins run will fail with a build timeout (7200 seconds) with stack traces that do not look like deadlock. They never reproduce for me. We really need to improve test infra here, so that each HEARTBEAT we also got 1) full thread stacks and 2) total CPU usage of the process as reported by the ManagementBean APIs ... this would shed more light on whether the JVM is somehow hung vs our bug (infinite loop). But infinite loop seems most likely ... the stacks always seem to be somewhere spooky. We should try to gather recent Jenkins runs where this is happening, here, to see if there are patterns / we can get to the root cause. Anyway, this happened to me on my old beast box, which runs the nightly ant test times graphs: http://people.apache.org/~mikemccand/lucenebench/antcleantest.html The 2015/06/27 data point is missing because it failed with this timeout: {noformat} [junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 2 ??? 28, 2015 7:01:29 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 21) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1245) [junit4] 2 at java.lang.Thread.join(Thread.java:1319) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 22) Thread[id=213, name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8], state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod] [junit4] 2 at org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900) [junit4] 2 at org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389) [junit4] 2 at org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14605324#comment-14605324 ] Michael McCandless commented on LUCENE-6629: Here's another one (just happened last night): http://build-eu-00.elastic.co/job/lucene_linux_java8_64_test_only/53453/console {noformat} [junit4] HEARTBEAT J2 PID(14791@localhost): 2015-06-29T01:59:23, stalled for 7144s at: TestIndexWriterCommit.testCommitUserData [junit4] Suite: org.apache.lucene.index.TestIndexWriterCommit [junit4] 2 6? 28, 2015 6:00:17 ?? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.index.TestIndexWriterCommit [junit4] 21) Thread[id=12, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main] [junit4] 2 at java.lang.Thread.sleep(Native Method) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47) [junit4] 22) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1249) [junit4] 2 at java.lang.Thread.join(Thread.java:1323) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 23) Thread[id=18, name=SUITE-TestIndexWriterCommit-seed#[9DCD05E439A5112E], state=RUNNABLE, group=TGRP-TestIndexWriterCommit] [junit4] 2 at java.lang.Thread.getStackTrace(Thread.java:1556) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:690) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:679) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$900(ThreadLeakControl.java:62) [junit4] 2 at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:412) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:651) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:138) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner$1.run(RandomizedRunner.java:568) [junit4] 24) Thread[id=19, name=TEST-TestIndexWriterCommit.testCommitUserData-seed#[9DCD05E439A5112E], state=RUNNABLE, group=TGRP-TestIndexWriterCommit] [junit4] 2 at org.apache.lucene.store.DataOutput.writeString(DataOutput.java:259) [junit4] 2 at org.apache.lucene.codecs.CodecUtil.writeHeader(CodecUtil.java:92) [junit4] 2 at org.apache.lucene.codecs.CodecUtil.writeIndexHeader(CodecUtil.java:134) [junit4] 2 at org.apache.lucene.codecs.lucene50.Lucene50SegmentInfoFormat.write(Lucene50SegmentInfoFormat.java:129) [junit4] 2 at org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:499) [junit4] 2 at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:457) [junit4] 2 at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:514) [junit4] 2 at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:384) [junit4] 2 at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:477) [junit4] 2 at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1470) [junit4] 2 at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1249) [junit4] 2 at org.apache.lucene.index.TestIndexWriter.addDoc(TestIndexWriter.java:148) [junit4] 2 at org.apache.lucene.index.TestIndexWriterCommit.testCommitUserData(TestIndexWriterCommit.java:664) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:502) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14604651#comment-14604651 ] Michael McCandless commented on LUCENE-6629: Here's another possible case, from http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/12746 (but now deleted): {noformat} [junit4] 21) Thread[id=149, name=TEST-TestSnowballVocab.testStemmers-seed#[7FBAD445ED608074], state=RUNNABLE, group=TGRP-TestSnowballVocab] [junit4] 2 at org.tartarus.snowball.ext.FrenchStemmer.stem(FrenchStemmer.java:1379) [junit4] 2 at org.apache.lucene.analysis.snowball.SnowballFilter.incrementToken(SnowballFilter.java:95) [junit4] 2 at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkResetException(BaseTokenStreamTestCase.java:400) [junit4] 2 at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo(BaseTokenStreamTestCase.java:352) [junit4] 2 at org.apache.lucene.analysis.BaseTokenStreamTestCase.assertAnalyzesTo(BaseTokenStreamTestCase.java:362) [junit4] 2 at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkOneTerm(BaseTokenStreamTestCase.java:423) [junit4] 2 at org.apache.lucene.analysis.VocabularyAssert.assertVocabulary(VocabularyAssert.java:47) [junit4] 2 at org.apache.lucene.analysis.VocabularyAssert.assertVocabulary(VocabularyAssert.java:73) [junit4] 2 at org.apache.lucene.analysis.snowball.TestSnowballVocab.assertCorrectOutput(TestSnowballVocab.java:80) [junit4] 2 at org.apache.lucene.analysis.snowball.TestSnowballVocab.testStemmers(TestSnowballVocab.java:45) {noformat} Random 7200 seconds build timeouts / infinite loops in Lucene tests? Key: LUCENE-6629 URL: https://issues.apache.org/jira/browse/LUCENE-6629 Project: Lucene - Core Issue Type: Bug Reporter: Michael McCandless I'm not sure what's going on here, but every so often a Jenkins run will fail with a build timeout (7200 seconds) with stack traces that do not look like deadlock. They never reproduce for me. We really need to improve test infra here, so that each HEARTBEAT we also got 1) full thread stacks and 2) total CPU usage of the process as reported by the ManagementBean APIs ... this would shed more light on whether the JVM is somehow hung vs our bug (infinite loop). But infinite loop seems most likely ... the stacks always seem to be somewhere spooky. We should try to gather recent Jenkins runs where this is happening, here, to see if there are patterns / we can get to the root cause. Anyway, this happened to me on my old beast box, which runs the nightly ant test times graphs: http://people.apache.org/~mikemccand/lucenebench/antcleantest.html The 2015/06/27 data point is missing because it failed with this timeout: {noformat} [junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 2 ??? 28, 2015 7:01:29 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 21) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1245) [junit4] 2 at java.lang.Thread.join(Thread.java:1319) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 22) Thread[id=213, name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8], state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod] [junit4] 2 at org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900) [junit4] 2 at org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389) [junit4] 2 at org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at
[jira] [Commented] (LUCENE-6629) Random 7200 seconds build timeouts / infinite loops in Lucene tests?
[ https://issues.apache.org/jira/browse/LUCENE-6629?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanelfocusedCommentId=14604653#comment-14604653 ] Robert Muir commented on LUCENE-6629: - I don't think that one is us. Those vocabulary asserts just stem the same .zip files always. its deterministic and a rather small (maybe 100k) amount of strings, same ones tested every time. Random 7200 seconds build timeouts / infinite loops in Lucene tests? Key: LUCENE-6629 URL: https://issues.apache.org/jira/browse/LUCENE-6629 Project: Lucene - Core Issue Type: Bug Reporter: Michael McCandless I'm not sure what's going on here, but every so often a Jenkins run will fail with a build timeout (7200 seconds) with stack traces that do not look like deadlock. They never reproduce for me. We really need to improve test infra here, so that each HEARTBEAT we also got 1) full thread stacks and 2) total CPU usage of the process as reported by the ManagementBean APIs ... this would shed more light on whether the JVM is somehow hung vs our bug (infinite loop). But infinite loop seems most likely ... the stacks always seem to be somewhere spooky. We should try to gather recent Jenkins runs where this is happening, here, to see if there are patterns / we can get to the root cause. Anyway, this happened to me on my old beast box, which runs the nightly ant test times graphs: http://people.apache.org/~mikemccand/lucenebench/antcleantest.html The 2015/06/27 data point is missing because it failed with this timeout: {noformat} [junit4] Suite: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 2 ??? 28, 2015 7:01:29 ? com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate [junit4] 2 WARNING: Suite execution timed out: org.apache.lucene.search.TestDocValuesRewriteMethod [junit4] 21) Thread[id=1, name=main, state=WAITING, group=main] [junit4] 2 at java.lang.Object.wait(Native Method) [junit4] 2 at java.lang.Thread.join(Thread.java:1245) [junit4] 2 at java.lang.Thread.join(Thread.java:1319) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578) [junit4] 2 at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310) [junit4] 2 at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12) [junit4] 22) Thread[id=213, name=TEST-TestDocValuesRewriteMethod.testRegexps-seed#[C2DDF486BB909D8], state=RUNNABLE, group=TGRP-TestDocValuesRewriteMethod] [junit4] 2 at org.apache.lucene.util.automaton.Operations.getLiveStates(Operations.java:900) [junit4] 2 at org.apache.lucene.util.automaton.Operations.hasDeadStates(Operations.java:389) [junit4] 2 at org.apache.lucene.util.automaton.Automata.makeString(Automata.java:517) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:579) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:519) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.findLeaves(RegExp.java:617) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomatonInternal(RegExp.java:510) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:495) [junit4] 2 at org.apache.lucene.util.automaton.RegExp.toAutomaton(RegExp.java:466) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:109) [junit4] 2 at org.apache.lucene.search.RegexpQuery.init(RegexpQuery.java:79) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.assertSame(TestDocValuesRewriteMethod.java:117) [junit4] 2 at org.apache.lucene.search.TestDocValuesRewriteMethod.testRegexps(TestDocValuesRewriteMethod.java:109) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [junit4] 2 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) [junit4] 2 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [junit4] 2 at java.lang.reflect.Method.invoke(Method.java:497) [junit4] 2 at