[JENKINS] Lucene-Solr-7.6-Linux (64bit/jdk-10.0.1) - Build # 118 - Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.6-Linux/118/
Java: 64bit/jdk-10.0.1 -XX:+UseCompressedOops -XX:+UseSerialGC

4 tests failed.
FAILED:  
org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
Expected new active leader null Live Nodes: [127.0.0.1:35213_solr, 
127.0.0.1:38721_solr, 127.0.0.1:43807_solr] Last available state: 
DocCollection(raceDeleteReplica_true//collections/raceDeleteReplica_true/state.json/11)={
   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{"shard1":{   
"range":"8000-7fff",   "state":"active",   "replicas":{ 
"core_node3":{   "core":"raceDeleteReplica_true_shard1_replica_n1", 
  "base_url":"https://127.0.0.1:46857/solr;,   
"node_name":"127.0.0.1:46857_solr",   "state":"down",   
"type":"NRT",   "leader":"true"}, "core_node6":{   
"core":"raceDeleteReplica_true_shard1_replica_n5",   
"base_url":"https://127.0.0.1:46857/solr;,   
"node_name":"127.0.0.1:46857_solr",   "state":"down",   
"type":"NRT"}, "core_node4":{   
"core":"raceDeleteReplica_true_shard1_replica_n2",   
"base_url":"https://127.0.0.1:43807/solr;,   
"node_name":"127.0.0.1:43807_solr",   "state":"down",   
"type":"NRT",   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",  
 "autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected new active leader
null
Live Nodes: [127.0.0.1:35213_solr, 127.0.0.1:38721_solr, 127.0.0.1:43807_solr]
Last available state: 
DocCollection(raceDeleteReplica_true//collections/raceDeleteReplica_true/state.json/11)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
  "range":"8000-7fff",
  "state":"active",
  "replicas":{
"core_node3":{
  "core":"raceDeleteReplica_true_shard1_replica_n1",
  "base_url":"https://127.0.0.1:46857/solr;,
  "node_name":"127.0.0.1:46857_solr",
  "state":"down",
  "type":"NRT",
  "leader":"true"},
"core_node6":{
  "core":"raceDeleteReplica_true_shard1_replica_n5",
  "base_url":"https://127.0.0.1:46857/solr;,
  "node_name":"127.0.0.1:46857_solr",
  "state":"down",
  "type":"NRT"},
"core_node4":{
  "core":"raceDeleteReplica_true_shard1_replica_n2",
  "base_url":"https://127.0.0.1:43807/solr;,
  "node_name":"127.0.0.1:43807_solr",
  "state":"down",
  "type":"NRT",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
at 
__randomizedtesting.SeedInfo.seed([8F3B9DFAC1CA0EF7:E52DFC2AA938443D]:0)
at org.junit.Assert.fail(Assert.java:93)
at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:280)
at 
org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:334)
at 
org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:229)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 

[JENKINS] Lucene-Solr-repro - Build # 2520 - Still Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-repro/2520/

[...truncated 57 lines...]
[repro] Jenkins log URL: 
https://builds.apache.org/job/Lucene-Solr-Tests-master/3073/consoleText

[repro] Revision: 3be5b59907acc3049121e980bee71df01185d40e

[repro] Repro line:  ant test  
-Dtestcase=ChaosMonkeySafeLeaderWithPullReplicasTest 
-Dtests.seed=56890122A4C878A1 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=el -Dtests.timezone=America/Anchorage -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1

[repro] Repro line:  ant test  -Dtestcase=TestSimTriggerIntegration 
-Dtests.method=testNodeAddedTriggerRestoreState -Dtests.seed=56890122A4C878A1 
-Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-GT 
-Dtests.timezone=Africa/Nouakchott -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1

[repro] git rev-parse --abbrev-ref HEAD
[repro] git rev-parse HEAD
[repro] Initial local git branch/revision: 
71f024ac8f1be7f74dceb84d91ea9d39a705172d
[repro] git fetch
[repro] git checkout 3be5b59907acc3049121e980bee71df01185d40e

[...truncated 2 lines...]
[repro] git merge --ff-only

[...truncated 1 lines...]
[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]solr/core
[repro]   TestSimTriggerIntegration
[repro]   ChaosMonkeySafeLeaderWithPullReplicasTest
[repro] ant compile-test

[...truncated 3592 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=10 
-Dtests.class="*.TestSimTriggerIntegration|*.ChaosMonkeySafeLeaderWithPullReplicasTest"
 -Dtests.showOutput=onerror  -Dtests.seed=56890122A4C878A1 -Dtests.multiplier=2 
-Dtests.slow=true -Dtests.locale=es-GT -Dtests.timezone=Africa/Nouakchott 
-Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1

[...truncated 5559 lines...]
[repro] Setting last failure code to 256

[repro] Failures:
[repro]   0/5 failed: 
org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
[repro]   4/5 failed: 
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[repro] git checkout 71f024ac8f1be7f74dceb84d91ea9d39a705172d

[...truncated 2 lines...]
[repro] Exiting with code 256

[...truncated 5 lines...]

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: Lucene/Solr 8.0

[S G]

It would be nice to see Solr 8 in January soon as there is an enhancement
on nested-documents we are waiting to get our hands on.
Any idea when Solr 8 would be out ?

Thx
SG

On Mon, Dec 17, 2018 at 1:34 PM David Smiley 
wrote:

> I see 10 JIRA issues matching this filter:   project in (SOLR, LUCENE) AND
> priority = Blocker and status = open and fixVersion = "master (8.0)"
>click here:
>
> https://issues.apache.org/jira/issues/?jql=project%20in%20(SOLR%2C%20LUCENE)%20AND%20priority%20%3D%20Blocker%20and%20status%20%3D%20open%20and%20fixVersion%20%3D%20%22master%20(8.0)%22%20
>
> Thru the end of the month, I intend to work on those issues not yet
> assigned.
>
> On Mon, Dec 17, 2018 at 4:51 AM Adrien Grand  wrote:
>
>> +1
>>
>> On Mon, Dec 17, 2018 at 10:38 AM Alan Woodward 
>> wrote:
>> >
>> > Hi all,
>> >
>> > Now that 7.6 is out of the door (thanks Nick!) we should think about
>> cutting the 8.0 branch and moving master to 9.0.  I’ll volunteer to create
>> the branch this week - say Wednesday?  Then we should have some time to
>> clean up the master branch and uncover anything that still needs to be done
>> on 8.0 before we start the release process next year.
>> >
>> > On 22 Oct 2018, at 18:12, Cassandra Targett 
>> wrote:
>> >
>> > I'm a bit delayed, but +1 on the 7.6 and 8.0 plan from me too.
>> >
>> > On Fri, Oct 19, 2018 at 7:18 AM Erick Erickson 
>> wrote:
>> >>
>> >> +1, this gives us all a chance to prioritize getting the blockers out
>> >> of the way in a careful manner.
>> >> On Fri, Oct 19, 2018 at 7:56 AM jim ferenczi 
>> wrote:
>> >> >
>> >> > +1 too. With this new perspective we could create the branch just
>> after the 7.6 release and target the 8.0 release for January 2019 which
>> gives almost 3 month to finish the blockers ?
>> >> >
>> >> > Le jeu. 18 oct. 2018 à 23:56, David Smiley 
>> a écrit :
>> >> >>
>> >> >> +1 to a 7.6 —lots of stuff in there
>> >> >> On Thu, Oct 18, 2018 at 4:47 PM Nicholas Knize 
>> wrote:
>> >> >>>
>> >> >>> If we're planning to postpone cutting an 8.0 branch until a few
>> weeks from now then I'd like to propose (and volunteer to RM) a 7.6 release
>> targeted for late November or early December (following the typical 2 month
>> release pattern). It feels like this might give a little breathing room for
>> finishing up 8.0 blockers? And looking at the change log there appear to be
>> a healthy list of features, bug fixes, and improvements to both Solr and
>> Lucene that warrant a 7.6 release? Personally I wouldn't mind releasing the
>> LatLonShape encoding changes in LUCENE-8521 and selective indexing work
>> done in LUCENE-8496. Any objections or thoughts?
>> >> >>>
>> >> >>> - Nick
>> >> >>>
>> >> >>>
>> >> >>> On Thu, Oct 18, 2018 at 5:32 AM Đạt Cao Mạnh <
>> caomanhdat...@gmail.com> wrote:
>> >> 
>> >>  Thanks Cassandra and Jim,
>> >> 
>> >>  I created a blocker issue for Solr 8.0 SOLR-12883, currently in
>> jira/http2 branch there are a draft-unmature implementation of SPNEGO
>> authentication which enough to makes the test pass, this implementation
>> will be removed when SOLR-12883 gets resolved . Therefore I don't see any
>> problem on merging jira/http2 to master branch in the next week.
>> >> 
>> >>  On Thu, Oct 18, 2018 at 2:33 AM jim ferenczi <
>> jim.feren...@gmail.com> wrote:
>> >> >
>> >> > > But if you're working with a different assumption - that just
>> the existence of the branch does not stop Dat from still merging his work
>> and the work being included in 8.0 - then I agree, waiting for him to merge
>> doesn't need to stop the creation of the branch.
>> >> >
>> >> > Yes that's my reasoning. This issue is a blocker so we won't
>> release without it but we can work on the branch in the meantime and let
>> other people work on new features that are not targeted to 8.
>> >> >
>> >> > Le mer. 17 oct. 2018 à 20:51, Cassandra Targett <
>> casstarg...@gmail.com> a écrit :
>> >> >>
>> >> >> OK - I was making an assumption that the timeline for the first
>> 8.0 RC would be ASAP after the branch is created.
>> >> >>
>> >> >> It's a common perception that making a branch freezes adding
>> new features to the release, perhaps in an unofficial way (more of a
>> courtesy rather than a rule). But if you're working with a different
>> assumption - that just the existence of the branch does not stop Dat from
>> still merging his work and the work being included in 8.0 - then I agree,
>> waiting for him to merge doesn't need to stop the creation of the branch.
>> >> >>
>> >> >> If, however, once the branch is there people object to Dat
>> merging his work because it's "too late", then the branch shouldn't be
>> created yet because we want to really try to clear that blocker for 8.0.
>> >> >>
>> >> >> Cassandra
>> >> >>
>> >> >> On Wed, Oct 17, 2018 at 12:13 PM jim ferenczi <
>> jim.feren...@gmail.com> wrote:
>> >> >>>
>> >> >>> Ok thanks for answering.
>> 

[JENKINS] Lucene-Solr-repro - Build # 2519 - Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-repro/2519/

[...truncated 28 lines...]
[repro] Jenkins log URL: 
https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.x/407/consoleText

[repro] Revision: 462adbc7f639839f2628202c6281ecdab223c9c8

[repro] Ant options: -Dtests.multiplier=2 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt
[repro] Repro line:  ant test  -Dtestcase=LIROnShardRestartTest 
-Dtests.method=testAllReplicasInLIR -Dtests.seed=3672748A295765FD 
-Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt
 -Dtests.locale=it-IT -Dtests.timezone=Africa/Blantyre -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8

[repro] Repro line:  ant test  -Dtestcase=TestSimTriggerIntegration 
-Dtests.method=testNodeAddedTriggerRestoreState -Dtests.seed=3672748A295765FD 
-Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt
 -Dtests.locale=fi -Dtests.timezone=Pacific/Port_Moresby -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8

[repro] Repro line:  ant test  -Dtestcase=RestartWhileUpdatingTest 
-Dtests.seed=3672748A295765FD -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt
 -Dtests.locale=de-LU -Dtests.timezone=Etc/GMT-0 -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8

[repro] git rev-parse --abbrev-ref HEAD
[repro] git rev-parse HEAD
[repro] Initial local git branch/revision: 
71f024ac8f1be7f74dceb84d91ea9d39a705172d
[repro] git fetch
[repro] git checkout 462adbc7f639839f2628202c6281ecdab223c9c8

[...truncated 2 lines...]
[repro] git merge --ff-only

[...truncated 1 lines...]
[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]solr/core
[repro]   LIROnShardRestartTest
[repro]   TestSimTriggerIntegration
[repro]   RestartWhileUpdatingTest
[repro] ant compile-test

[...truncated 3605 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=15 
-Dtests.class="*.LIROnShardRestartTest|*.TestSimTriggerIntegration|*.RestartWhileUpdatingTest"
 -Dtests.showOutput=onerror -Dtests.multiplier=2 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt
 -Dtests.seed=3672748A295765FD -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.x/test-data/enwiki.random.lines.txt
 -Dtests.locale=it-IT -Dtests.timezone=Africa/Blantyre -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8

[...truncated 86027 lines...]
[repro] Setting last failure code to 256

[repro] Failures:
[repro]   0/5 failed: 
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[repro]   3/5 failed: org.apache.solr.cloud.LIROnShardRestartTest
[repro]   4/5 failed: org.apache.solr.cloud.RestartWhileUpdatingTest
[repro] git checkout 71f024ac8f1be7f74dceb84d91ea9d39a705172d

[...truncated 2 lines...]
[repro] Exiting with code 256

[...truncated 6 lines...]

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12999) Index replication could delete segments first

[Erick Erickson (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12999?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724643#comment-16724643
 ] 

Erick Erickson commented on SOLR-12999:
---

Well, there's a third possibility. Use whatever remains of your index, possibly 
with checkindex and the -fix option. Admittedly you'd only have partial index, 
but if that somehow was the only replica left at least you'd have something.

I don't think that's a valid reason to hold up this JIRA, but let's be explicit 
about the design decision. That said, the case where recovering the partial 
index from your one remaining replica is outweighed by the utility of avoiding 
disk full situations IMO.

> Index replication could delete segments first
> -
>
> Key: SOLR-12999
> URL: https://issues.apache.org/jira/browse/SOLR-12999
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: replication (java)
>Reporter: David Smiley
>Priority: Major
>
> Index replication could optionally delete files that it knows will not be 
> needed _first_.  This would reduce disk capacity requirements of Solr, and it 
> would reduce some disk fragmentation when space get tight.
> Solr (IndexFetcher) already grabs the remote file list, and it could see 
> which files it has locally, then delete the others.  Today it asks Lucene to 
> {{deleteUnusedFiles}} at the end.  This new mode would probably only be 
> useful if there is no SolrIndexSearcher open, since it would prevent the 
> removal of files.
> The motivating scenario is a SolrCloud replica that is going into full 
> recovery.  It ought to not be fielding searches.  The code changes would not 
> depend on SolrCloud though.
> This option would have some danger the user should be aware of.  If the 
> replication fails, leaving the local files incomplete/corrupt, the only 
> recourse is to try full replication again.  You can't just give up and field 
> queries.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-BadApples-Tests-master - Build # 239 - Still unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/239/

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderTragicEventTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! 
[MockDirectoryWrapper] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.MockDirectoryWrapper  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:95)  at 
org.apache.solr.core.SolrCore.initIndex(SolrCore.java:770)  at 
org.apache.solr.core.SolrCore.(SolrCore.java:967)  at 
org.apache.solr.core.SolrCore.(SolrCore.java:874)  at 
org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191)
  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101)  at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
  at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
  at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
  at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
  at 
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)  
at 
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)  
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:394)
  at 
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:340)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
  at 
org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:164)
  at 
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540) 
 at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
  at 
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)  
at 
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
  at 
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
  at 
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)  
at 
org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:703)  
at 
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) 
 at org.eclipse.jetty.server.Server.handle(Server.java:502)  at 
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)  at 
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)  at 
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at 
org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)
  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)  
at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)  
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)  at 
org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)  at 
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
  at 
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) 
 at java.lang.Thread.run(Thread.java:748)   expected null, but 
was:(SolrCore.java:967)  at 
org.apache.solr.core.SolrCore.(SolrCore.java:874)  at 
org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191)
  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101)  at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
  at 
org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:360)
  at 
org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:395)
  at 
org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:180)
  at 

[jira] [Updated] (LUCENE-8614) ArrayIndexOutOfBoundsException in ByteBlockPool

[Igor Motov (JIRA)]

 [ 
https://issues.apache.org/jira/browse/LUCENE-8614?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Igor Motov updated LUCENE-8614:
---
Attachment: LUCENE-8614.patch

> ArrayIndexOutOfBoundsException in ByteBlockPool
> ---
>
> Key: LUCENE-8614
> URL: https://issues.apache.org/jira/browse/LUCENE-8614
> Project: Lucene - Core
>  Issue Type: Bug
>  Components: core/index
>Affects Versions: 7.5
>Reporter: Igor Motov
>Priority: Major
> Attachments: LUCENE-8614.patch
>
>
> A field with a very large number of small tokens can cause 
> ArrayIndexOutOfBoundsException in ByteBlockPool due to an arithmetic overflow 
> in ByteBlockPool.
> The issue was originally reported in 
> [https://github.com/elastic/elasticsearch/issues/23670] where due to the 
> indexing settings the geo_shape generated a very large number of tokens and 
> caused the indexing operation to fail with the following exception: 
> {noformat}
> Caused by: java.lang.ArrayIndexOutOfBoundsException: -65531
>   at 
> org.apache.lucene.util.ByteBlockPool.setBytesRef(ByteBlockPool.java:308) 
> ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at org.apache.lucene.util.BytesRefHash.equals(BytesRefHash.java:183) 
> ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at org.apache.lucene.util.BytesRefHash.findHash(BytesRefHash.java:337) 
> ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at org.apache.lucene.util.BytesRefHash.add(BytesRefHash.java:255) 
> ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at 
> org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:149) 
> ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at 
> org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:766)
>  ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at 
> org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:417)
>  ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at 
> org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:373)
>  ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at 
> org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>  ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at 
> org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:478)
>  ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at 
> org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1575) 
> ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
>   at 
> org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1320) 
> ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim 
> - 2017-01-17 15:57:29]
> {noformat}
> I was able to reproduce the issue and somewhat reduce the test that 
> reproduces it (see enclosed patch) but unfortunately it still requires 12G of 
> heap to run.
> The issue seems to be caused by arithmetic overflow in the {{byteOffset}} 
> calculation when {{BytesBlockPool}} advances to the next buffer on the last 
> line of the 
> [nextBuffer()|https://github.com/apache/lucene-solr/blob/e386ec973b8a4ec2de2bfc43f51df511a365d60f/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java#L207]
>  method, but it doesn't manifest itself until much later when this offset is 
> used to calculate the 
> [bytesStart|https://github.com/apache/lucene-solr/blob/e386ec973b8a4ec2de2bfc43f51df511a365d60f/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java#L277]
>  in {{BytesRefHash}}, which in turn causes AIOB back in the {{ByteBlockPool}} 
> [setBytesRef()|https://github.com/apache/lucene-solr/blob/e386ec973b8a4ec2de2bfc43f51df511a365d60f/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java#L308]
>  method where it is used to find the term's buffer.
> I realize that it's unreasonable to expect lucene to index such fields, but I 
> wonder if an overflow check should be added to {{BytesBlockPool.nextBuffer}} 
> in order to handle such condition more gracefully. 
>   
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Created] (LUCENE-8614) ArrayIndexOutOfBoundsException in ByteBlockPool

[Igor Motov (JIRA)]

Igor Motov created LUCENE-8614:
--

 Summary: ArrayIndexOutOfBoundsException in ByteBlockPool
 Key: LUCENE-8614
 URL: https://issues.apache.org/jira/browse/LUCENE-8614
 Project: Lucene - Core
  Issue Type: Bug
  Components: core/index
Affects Versions: 7.5
Reporter: Igor Motov


A field with a very large number of small tokens can cause 
ArrayIndexOutOfBoundsException in ByteBlockPool due to an arithmetic overflow 
in ByteBlockPool.

The issue was originally reported in 
[https://github.com/elastic/elasticsearch/issues/23670] where due to the 
indexing settings the geo_shape generated a very large number of tokens and 
caused the indexing operation to fail with the following exception: 
{noformat}
Caused by: java.lang.ArrayIndexOutOfBoundsException: -65531
at 
org.apache.lucene.util.ByteBlockPool.setBytesRef(ByteBlockPool.java:308) 
~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at org.apache.lucene.util.BytesRefHash.equals(BytesRefHash.java:183) 
~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at org.apache.lucene.util.BytesRefHash.findHash(BytesRefHash.java:337) 
~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at org.apache.lucene.util.BytesRefHash.add(BytesRefHash.java:255) 
~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at 
org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:149) 
~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at 
org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:766)
 ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at 
org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:417)
 ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at 
org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:373)
 ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at 
org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
 ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at 
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:478)
 ~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at 
org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1575) 
~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
at 
org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1320) 
~[lucene-core-6.4.0.jar:6.4.0 bbe4b08cc1fb673d0c3eb4b8455f23ddc1364124 - jim - 
2017-01-17 15:57:29]
{noformat}
I was able to reproduce the issue and somewhat reduce the test that reproduces 
it (see enclosed patch) but unfortunately it still requires 12G of heap to run.

The issue seems to be caused by arithmetic overflow in the {{byteOffset}} 
calculation when {{BytesBlockPool}} advances to the next buffer on the last 
line of the 
[nextBuffer()|https://github.com/apache/lucene-solr/blob/e386ec973b8a4ec2de2bfc43f51df511a365d60f/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java#L207]
 method, but it doesn't manifest itself until much later when this offset is 
used to calculate the 
[bytesStart|https://github.com/apache/lucene-solr/blob/e386ec973b8a4ec2de2bfc43f51df511a365d60f/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java#L277]
 in {{BytesRefHash}}, which in turn causes AIOB back in the {{ByteBlockPool}} 
[setBytesRef()|https://github.com/apache/lucene-solr/blob/e386ec973b8a4ec2de2bfc43f51df511a365d60f/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java#L308]
 method where it is used to find the term's buffer.

I realize that it's unreasonable to expect lucene to index such fields, but I 
wonder if an overflow check should be added to {{BytesBlockPool.nextBuffer}} in 
order to handle such condition more gracefully. 
  

 



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12999) Index replication could delete segments first

[Noble Paul (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12999?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724573#comment-16724573
 ] 

Noble Paul commented on SOLR-12999:
---

Let's look at a usecase where the disk space left in the replica is less than 
the space required to download the segments from the leader. So the only 
choices left are , 
# fail replication altogether
# or delete the local segments and fetch the segments from the leader  

I think #2 is definitely a better option

> Index replication could delete segments first
> -
>
> Key: SOLR-12999
> URL: https://issues.apache.org/jira/browse/SOLR-12999
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: replication (java)
>Reporter: David Smiley
>Priority: Major
>
> Index replication could optionally delete files that it knows will not be 
> needed _first_.  This would reduce disk capacity requirements of Solr, and it 
> would reduce some disk fragmentation when space get tight.
> Solr (IndexFetcher) already grabs the remote file list, and it could see 
> which files it has locally, then delete the others.  Today it asks Lucene to 
> {{deleteUnusedFiles}} at the end.  This new mode would probably only be 
> useful if there is no SolrIndexSearcher open, since it would prevent the 
> removal of files.
> The motivating scenario is a SolrCloud replica that is going into full 
> recovery.  It ought to not be fielding searches.  The code changes would not 
> depend on SolrCloud though.
> This option would have some danger the user should be aware of.  If the 
> replication fails, leaving the local files incomplete/corrupt, the only 
> recourse is to try full replication again.  You can't just give up and field 
> queries.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 964 - Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/964/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud.testCreateCollectionSplitShard

Error Message:
Expected exactly one replica of collection on node with port: 10029 
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: Expected exactly one replica of collection on node 
with port: 10029 expected:<1> but was:<0>
at 
__randomizedtesting.SeedInfo.seed([AD6F5F726E4A1850:78E12C89DE0099EE]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotEquals(Assert.java:834)
at org.junit.Assert.assertEquals(Assert.java:645)
at 
org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud.testCreateCollectionSplitShard(TestSimPolicyCloud.java:171)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at 
org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:239)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk-10.0.1) - Build # 23352 - Failure!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/23352/
Java: 64bit/jdk-10.0.1 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

All tests passed

Build Log:
[...truncated 2064 lines...]
   [junit4] JVM J0: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/core/test/temp/junit4-J0-20181218_213130_6747708599016798351355.syserr
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J0: EOF 

[...truncated 3 lines...]
   [junit4] JVM J2: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/core/test/temp/junit4-J2-20181218_213130_6746119389051736584945.syserr
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J2: EOF 

[...truncated 5 lines...]
   [junit4] JVM J1: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/core/test/temp/junit4-J1-20181218_213130_6742513111646685291449.syserr
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J1: EOF 

[...truncated 307 lines...]
   [junit4] JVM J1: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/test-framework/test/temp/junit4-J1-20181218_214245_37010835170708056815045.syserr
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J1: EOF 

   [junit4] JVM J2: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/test-framework/test/temp/junit4-J2-20181218_214245_37014345105120058747701.syserr
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J2: EOF 

[...truncated 3 lines...]
   [junit4] JVM J0: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/test-framework/test/temp/junit4-J0-20181218_214245_37015415881966369766000.syserr
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J0: EOF 

[...truncated 1083 lines...]
   [junit4] JVM J2: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/analysis/common/test/temp/junit4-J2-20181218_214432_865896182183882615.syserr
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J2: EOF 

   [junit4] JVM J1: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/analysis/common/test/temp/junit4-J1-20181218_214432_86415932851075735916813.syserr
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J1: EOF 

[...truncated 3 lines...]
   [junit4] JVM J0: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/analysis/common/test/temp/junit4-J0-20181218_214432_86413814102953478613272.syserr
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J0: EOF 

[...truncated 255 lines...]
   [junit4] JVM J1: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/analysis/icu/test/temp/junit4-J1-20181218_214630_4171924240463300637535.syserr
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) 
   [junit4] OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was 
deprecated in version 9.0 and will likely be removed in a future release.
   [junit4] <<< JVM J1: EOF 

[...truncated 6 lines...]
   [junit4] JVM J0: stderr was not empty, see: 
/home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/build/analysis/icu/test/temp/junit4-J0-20181218_214630_4165253257694235578279.syserr
   

[jira] [Commented] (SOLR-13074) MoveReplicaHDFSTest leaks threads, falls into an endless loop, logging like crazy

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13074?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724512#comment-16724512
 ] 

ASF subversion and git services commented on SOLR-13074:


Commit 71f024ac8f1be7f74dceb84d91ea9d39a705172d in lucene-solr's branch 
refs/heads/master from Dawid Weiss
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=71f024a ]

SOLR-13074: clean up static variables properly, avoiding NPEs.


> MoveReplicaHDFSTest leaks threads, falls into an endless loop, logging like 
> crazy
> -
>
> Key: SOLR-13074
> URL: https://issues.apache.org/jira/browse/SOLR-13074
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Major
>
> This reproduces for me, always (Linux box):
> {code}
> ant test  -Dtestcase=MoveReplicaHDFSTest -Dtests.seed=DC1CE772C445A55D 
> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.locale=fr 
> -Dtests.timezone=Australia/Tasmania -Dtests.asserts=true 
> -Dtests.file.encoding=UTF-8
> {code}
> It's the bug in Hadoop I discusse in SOLR-13060 -- one of the threads falls 
> into an endless loop when terminated (interrupted). Perhaps we should close 
> something cleanly and don't.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: Proposed Additional Hooks in SolrEventListener

[Tomás Fernández Löbbe]

Hi Kevin,
Isn't UpdateRequestProcessor enough for what you need?

On Tue, Dec 18, 2018 at 1:29 PM Kevin Jia 
wrote:

> bounce
> --
> *From:* Kevin Jia
> *Sent:* December 12, 2018 6:36:33 PM
> *To:* dev@lucene.apache.org
> *Subject:* Proposed Additional Hooks in SolrEventListener
>
>
> Hi Everyone,
>
>
> I'm looking to add Prospective Search functionality to Solr, similar to
> what Luwak (https://github.com/flaxsearch/luwak) does - see existing JIRA
> ticket : https://issues.apache.org/jira/browse/SOLR-4587
> .
> To do this I need to maintain an in-memory cache that is in sync with the
> document index (custom cache that is not a straightforwards field cache).
>
> To maintain my in-memory cache, I wanted to add functionality after
> updates (in DirectUpdateHandler2) and SolrCore instantiation. Instead of
> changing the code directly I wanted to add more hooks to SolrEventListener,
> namely these:
>
>
> public void postCoreConstruct(SolrCore core);
> public void preAddDoc(AddUpdateCommand cmd);
> public void postAddDoc(AddUpdateCommand cmd);
> public void preDelete(DeleteUpdateCommand cmd);
> public void postDelete(DeleteUpdateCommand cmd);
>
>
> I also made a ticket: https://issues.apache.org/jira/browse/SOLR-4587. If
> anyone ever needs similar custom behavior, they would be able to use
> these hooks as well.
>
> Does anyone have any thoughts or suggestions on my proposed changes? Is
> there a better way to do this? If not, I can submit a patch soon.
>
>
> Best,
>
> Kevin
>
>
>

[jira] [Commented] (SOLR-13074) MoveReplicaHDFSTest leaks threads, falls into an endless loop, logging like crazy

[Dawid Weiss (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13074?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724511#comment-16724511
 ] 

Dawid Weiss commented on SOLR-13074:


So the problem here is that a NPE happens in BeforeClass hook in 
MoveReplicaHDFSTest:
{code}
  @BeforeClass
  public static void setupClass() throws Exception {
System.setProperty("solr.hdfs.blockcache.enabled", "false");
dfsCluster = 
HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());

ZkConfigManager configManager = new ZkConfigManager(zkClient());
{code}

when zkClient() is called from SolrCloudTestCase, the 'cluster' variable is 
null, causing an NPE. Then things get out of hand because we already 
initialized dfsCluster, but the AfterClass hook fails with an NPE before it can 
clean it up:
{code}
  @AfterClass
  public static void teardownClass() throws Exception {
cluster.shutdown(); // need to close before the MiniDFSCluster
HdfsTestUtil.teardownClass(dfsCluster);
dfsCluster = null;
  }
{code}

That's the reason of all those thread leaks from Hdfs. Now, I have no idea how 
to initialize this cluster properly (I know nothing about cloud infra). I've 
committed some code to master to clean up this test properly: this now displays 
the actual cause of the problem. The cleanup code begs for some kind of 
higher-level "closer" which could close all these objects in order, taking into 
account nulls and their specific close methods. I didn't deal with it.

[~markrmil...@gmail.com] -- would you take a look at how to initialize the 
cluster properly in this test? Or maybe [~ab] would know how to fix it (I see 
you're the original author of this test, Andrzej, hence the question).

> MoveReplicaHDFSTest leaks threads, falls into an endless loop, logging like 
> crazy
> -
>
> Key: SOLR-13074
> URL: https://issues.apache.org/jira/browse/SOLR-13074
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Major
>
> This reproduces for me, always (Linux box):
> {code}
> ant test  -Dtestcase=MoveReplicaHDFSTest -Dtests.seed=DC1CE772C445A55D 
> -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.locale=fr 
> -Dtests.timezone=Australia/Tasmania -Dtests.asserts=true 
> -Dtests.file.encoding=UTF-8
> {code}
> It's the bug in Hadoop I discusse in SOLR-13060 -- one of the threads falls 
> into an endless loop when terminated (interrupted). Perhaps we should close 
> something cleanly and don't.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1726 - Failure

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1726/

5 tests failed.
FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest

Error Message:
ObjectTracker found 5 object(s) that were not released!!! [InternalHttpClient, 
MMapDirectory, MMapDirectory, SolrCore, MMapDirectory] 
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.http.impl.client.InternalHttpClient  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:321)
  at 
org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:330)
  at 
org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:225)  
at org.apache.solr.handler.IndexFetcher.(IndexFetcher.java:267)  at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420) 
 at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:237) 
 at 
org.apache.solr.cloud.RecoveryStrategy.doReplicateOnlyRecovery(RecoveryStrategy.java:382)
  at 
org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:328)  
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:307)  at 
com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  
at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at 
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
 at java.lang.Thread.run(Thread.java:748)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.MMapDirectory  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1184)
  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)  at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
  at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
 at java.lang.Thread.run(Thread.java:748)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.lucene.store.MMapDirectory  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at 
org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:348)
  at 
org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:503)  
at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:346) 
 at 
org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:424) 
 at 
org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1184)
  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)  
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)  at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
  at 
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
  at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 
 at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 
 at java.lang.Thread.run(Thread.java:748)  
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: 
org.apache.solr.core.SolrCore  at 
org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
  at org.apache.solr.core.SolrCore.(SolrCore.java:1054)  at 
org.apache.solr.core.SolrCore.(SolrCore.java:874)  at 
org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1191)
  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1101)  at 
org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
  at 

Re: Proposed Additional Hooks in SolrEventListener

[Kevin Jia]

bounce


From: Kevin Jia
Sent: December 12, 2018 6:36:33 PM
To: dev@lucene.apache.org
Subject: Proposed Additional Hooks in SolrEventListener


Hi Everyone,


I'm looking to add Prospective Search functionality to Solr, similar to what 
Luwak (https://github.com/flaxsearch/luwak) does - see existing JIRA ticket : 
https://issues.apache.org/jira/browse/SOLR-4587.
 To do this I need to maintain an in-memory cache that is in sync with the 
document index (custom cache that is not a straightforwards field cache).

To maintain my in-memory cache, I wanted to add functionality after updates (in 
DirectUpdateHandler2) and SolrCore instantiation. Instead of changing the code 
directly I wanted to add more hooks to SolrEventListener, namely these:


public void postCoreConstruct(SolrCore core);
public void preAddDoc(AddUpdateCommand cmd);
public void postAddDoc(AddUpdateCommand cmd);
public void preDelete(DeleteUpdateCommand cmd);
public void postDelete(DeleteUpdateCommand cmd);


I also made a ticket: https://issues.apache.org/jira/browse/SOLR-4587. If 
anyone ever needs similar custom behavior, they would be able to use these 
hooks as well.

Does anyone have any thoughts or suggestions on my proposed changes? Is there a 
better way to do this? If not, I can submit a patch soon.


Best,

Kevin

[jira] [Resolved] (LUCENE-8604) Add TestRuleLimitSysouts tests, do some cleanup and add hard limit

[Dawid Weiss (JIRA)]

 [ 
https://issues.apache.org/jira/browse/LUCENE-8604?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Dawid Weiss resolved LUCENE-8604.
-
Resolution: Fixed

> Add TestRuleLimitSysouts tests, do some cleanup and add hard limit
> --
>
> Key: LUCENE-8604
> URL: https://issues.apache.org/jira/browse/LUCENE-8604
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Trivial
> Fix For: 7.7
>
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> PR from github here. 
> https://github.com/apache/lucene-solr/pull/524



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8604) Add TestRuleLimitSysouts tests, do some cleanup and add hard limit

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8604?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724471#comment-16724471
 ] 

ASF subversion and git services commented on LUCENE-8604:
-

Commit f28c5bec9bbd06a1f70f0ddbc2d71e12a1961f65 in lucene-solr's branch 
refs/heads/master from Dawid Weiss
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=f28c5be ]

LUCENE-8604: TestRuleLimitSysouts now has an optional "hard limit" of bytes 
that can be written to stderr and stdout (anything beyond the hard limit is 
ignored). The default hard limit is 2 GB of logs per test class.


> Add TestRuleLimitSysouts tests, do some cleanup and add hard limit
> --
>
> Key: LUCENE-8604
> URL: https://issues.apache.org/jira/browse/LUCENE-8604
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Trivial
> Fix For: 7.7
>
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> PR from github here. 
> https://github.com/apache/lucene-solr/pull/524



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (LUCENE-8604) Add TestRuleLimitSysouts tests, do some cleanup and add hard limit

[Dawid Weiss (JIRA)]

 [ 
https://issues.apache.org/jira/browse/LUCENE-8604?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Dawid Weiss updated LUCENE-8604:

Fix Version/s: 7.7

> Add TestRuleLimitSysouts tests, do some cleanup and add hard limit
> --
>
> Key: LUCENE-8604
> URL: https://issues.apache.org/jira/browse/LUCENE-8604
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Trivial
> Fix For: 7.7
>
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> PR from github here. 
> https://github.com/apache/lucene-solr/pull/524



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-7.6-Linux (32bit/jdk1.8.0_172) - Build # 116 - Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.6-Linux/116/
Java: 32bit/jdk1.8.0_172 -client -XX:+UseParallelGC

4 tests failed.
FAILED:  
org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
Expected new active leader null Live Nodes: [127.0.0.1:38699_solr, 
127.0.0.1:38827_solr, 127.0.0.1:40259_solr] Last available state: 
DocCollection(raceDeleteReplica_false//collections/raceDeleteReplica_false/state.json/13)={
   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{"shard1":{   
"range":"8000-7fff",   "state":"active",   "replicas":{ 
"core_node4":{   "core":"raceDeleteReplica_false_shard1_replica_n2",
   "base_url":"http://127.0.0.1:43843/solr;,   
"node_name":"127.0.0.1:43843_solr",   "state":"down",   
"type":"NRT",   "force_set_state":"false",   "leader":"true"},  
   "core_node6":{   
"core":"raceDeleteReplica_false_shard1_replica_n5",   
"base_url":"http://127.0.0.1:43843/solr;,   
"node_name":"127.0.0.1:43843_solr",   "state":"down",   
"type":"NRT",   "force_set_state":"false",   
"router":{"name":"compositeId"},   "maxShardsPerNode":"1",   
"autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected new active leader
null
Live Nodes: [127.0.0.1:38699_solr, 127.0.0.1:38827_solr, 127.0.0.1:40259_solr]
Last available state: 
DocCollection(raceDeleteReplica_false//collections/raceDeleteReplica_false/state.json/13)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
  "range":"8000-7fff",
  "state":"active",
  "replicas":{
"core_node4":{
  "core":"raceDeleteReplica_false_shard1_replica_n2",
  "base_url":"http://127.0.0.1:43843/solr;,
  "node_name":"127.0.0.1:43843_solr",
  "state":"down",
  "type":"NRT",
  "force_set_state":"false",
  "leader":"true"},
"core_node6":{
  "core":"raceDeleteReplica_false_shard1_replica_n5",
  "base_url":"http://127.0.0.1:43843/solr;,
  "node_name":"127.0.0.1:43843_solr",
  "state":"down",
  "type":"NRT",
  "force_set_state":"false",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
at 
__randomizedtesting.SeedInfo.seed([994479D7B8667A57:F3521807D094309D]:0)
at org.junit.Assert.fail(Assert.java:93)
at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:280)
at 
org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:334)
at 
org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:230)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at 

[jira] [Commented] (SOLR-13080) TermsQParserPlugin automaton method fails to sort input

[Daniel Lowe (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13080?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724441#comment-16724441
 ] 

Daniel Lowe commented on SOLR-13080:


I attempted to come up with a failing test before posting this and failed to do 
so. With unsorted input makeStringUnion seems to produce a non-deterministic, 
non-minimal automaton (rather than a deterministic minimal automaton)... but it 
still matches the intended terms. Additionally the time to build the automaton 
was significantly greater than the time required to perform a sort then build 
the automaton (obviously this wouldn't be true in the case where the input 
terms were provided in sorted order). I guess strictly speaking that makes this 
only a performance optimization.

The closest I can really get to a unit test is that with assertions enabled, 
unsorted input and the method set to automaton, DaciukMihovAutomatonBuilder 
should throw an assertion error!

> TermsQParserPlugin automaton method fails to sort input
> ---
>
> Key: SOLR-13080
> URL: https://issues.apache.org/jira/browse/SOLR-13080
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: query parsers
>Affects Versions: 7.5
>Reporter: Daniel Lowe
>Priority: Minor
>
> The contract for Automata.makeStringUnion is that the input is sorted. As 
> BytesRef implements Comparable. The simplest fix would probably to make
> Arrays.sort(bytesRefs);
> The first line of automaton's makeFilter method in TermsQParserPlugin.
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: SOLR-12259: online index schema modification - adding docValues to existing indexed data?

[Andrzej Białecki]

Right - the code on this branch is a port from another branch with other 
changes, too - among others a modified UninvertingReader that discards 
docValues even when FieldInfo claims the field has them (ie it always wraps 
when the mapping says so). Even with that change the results are the same. 

> On 18 Dec 2018, at 20:36, Adrien Grand  wrote:
> 
> UninvertingReader#wrap seems to skip uninverting if the field to
> uninvert already has doc values of the expected type?
> 
> On Tue, Dec 18, 2018 at 8:24 PM Andrzej Białecki
>  wrote:
>> 
>> The unexpected part is that I would have expected the code to handle 
>> franken-segments as well, because at some point we finally resorted to 
>> always forcing the wrapping even for segments that don’t need it (ie. they 
>> claim the field contains DVs) but the test is still failing.
>> 
>> On 18 Dec 2018, at 19:05, Adrien Grand  wrote:
>> 
>> I had a quick look and couldn't find anything to prevent what you
>> called “franken-segments” in the Lucene test?
>> 
>> On Tue, Dec 18, 2018 at 5:59 PM Erick Erickson  
>> wrote:
>> 
>> 
>> A couple of additions:
>> 
>> AddDVMPLuceneTest2 does not use Solr constructs at all, so is the test
>> we think is most interesting at this point, it won't lead anyone down
>> the path of "what's all this Solr stuff and is it right" kinds of
>> questions (believe me, we've spent some time on that path!). Please
>> feel free to look at all the rest of it of course, but the place we're
>> stuck is why this test fails.
>> 
>> AddDvStress is intended as an integration-level test, it requires some
>> special setup (in particular providing a particular configset), we put
>> that together to reliably make the problem visible. We thought the new
>> code was the issue at first and needed something to narrow down the
>> possibilities...
>> 
>> The reason we're obsessing about this is that it calls into question
>> how segments are merged when "things change". We don't understand why
>> this is happening at the Lucene level so don't know how to insure that
>> things like the schema API in Solr aren't affected.
>> 
>> Andrzej isn't the only one running out of ideas ;).
>> 
>> On Tue, Dec 18, 2018 at 4:46 AM Andrzej Białecki  wrote:
>> 
>> 
>> Hi,
>> 
>> I'm working on a use case where an existing Solr setup needs to migrate to a 
>> schema that uses docValues for faceting, instead of uninversion. This case 
>> fits into a broader subject of SOLR-12259 (Robustly upgrade indexes). 
>> However, in this case there are two major requirements for this migration 
>> process:
>> 
>> * data cannot be reindexed from scratch - I need to work with the already 
>> indexed documents (which do contain the values needed for faceting, but 
>> these values are simply indexed and not stored as doc values)
>> 
>> * indexing can’t be stopped while the schema is being changed (the 
>> conversion process needs to work on-the-fly while the collection is online, 
>> both for searching and for updates). Collection reloads / reopening is ok 
>> but it’s not ok to take the collection offline for several minutes (or 
>> hours).
>> 
>> Together with Erick Erickson we implemented a solution that uses MergePolicy 
>> (actually MergePolicyFactory in Solr) to enforce re-writing of segments that 
>> no longer match the schema, ie. don’t contain docValues in a field where the 
>> new schema requires it. This merge policy determines what segments need this 
>> conversion and then forces the “merging” (actually re-writing) of these 
>> segments by first wrapping them into UninvertingReader to supply docValues 
>> where they are required by the new schema but actually are missing in the 
>> segment’s data. This “AddDocValuesMergePolicy” (ADVMP for short) is supposed 
>> to deal with the following types of segments:
>> 
>> * old segments created before the schema change - these don’t contain any 
>> docValues in the target fields and so they are wrapped in UninvertingReader 
>> for merging (and for searching) according to the new schema.
>> 
>> * new segments created after the schema change - if FieldInfo-s for these 
>> fields claim that the segment already contains docValues where it should 
>> then the segment is passed as-is to merging, otherwise it’s wrapped again. 
>> An optimisation was also put here to “mark” the already converted segments 
>> using a marker in SegmentInfo diagnostics map so that we can avoid 
>> re-checking and re-converting already converted data.
>> 
>> So, long story short, this process works very well when there’s no 
>> concurrent indexing activity - all old segments are properly wrapped and 
>> re-written and merging with new segments works as intended. However, in a 
>> situation with concurrent indexing it works well but only for a short while. 
>> At some point this conversion process seems to lose large percentage of the 
>> docValues, even though it seems that at all points the source segments are 
>> properly wrapped - the ADVMP merge policy adds a lot 

[jira] [Commented] (LUCENE-8585) Create jump-tables for DocValues at index-time

[Adrien Grand (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8585?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724413#comment-16724413
 ] 

Adrien Grand commented on LUCENE-8585:
--

My understanding of [~romseygeek]'s proposal to cut a branch is to start 
cleaning up master rather than freezing 8.0. I'd expect another call for a 
feature freeze when the time comes.

I'm also supportive of having it in 8.0 so that we won't have to keep the 7.x 
codec until 9.x.

Please ping me when the PR is ready for another round of review.

> Create jump-tables for DocValues at index-time
> --
>
> Key: LUCENE-8585
> URL: https://issues.apache.org/jira/browse/LUCENE-8585
> Project: Lucene - Core
>  Issue Type: Improvement
>  Components: core/codecs
>Affects Versions: master (8.0)
>Reporter: Toke Eskildsen
>Priority: Minor
>  Labels: performance
> Attachments: LUCENE-8585.patch, LUCENE-8585.patch, 
> make_patch_lucene8585.sh
>
>  Time Spent: 4h 50m
>  Remaining Estimate: 0h
>
> As noted in LUCENE-7589, lookup of DocValues should use jump-tables to avoid 
> long iterative walks. This is implemented in LUCENE-8374 at search-time 
> (first request for DocValues from a field in a segment), with the benefit of 
> working without changes to existing Lucene 7 indexes and the downside of 
> introducing a startup time penalty and a memory overhead.
> As discussed in LUCENE-8374, the codec should be updated to create these 
> jump-tables at index time. This eliminates the segment-open time & memory 
> penalties, with the potential downside of increasing index-time for DocValues.
> The three elements of LUCENE-8374 should be transferable to index-time 
> without much alteration of the core structures:
>  * {{IndexedDISI}} block offset and index skips: A {{long}} (64 bits) for 
> every 65536 documents, containing the offset of the block in 33 bits and the 
> index (number of set bits) up to the block in 31 bits.
>  It can be build sequentially and should be stored as a simple sequence of 
> consecutive longs for caching of lookups.
>  As it is fairly small, relative to document count, it might be better to 
> simply memory cache it?
>  * {{IndexedDISI}} DENSE (> 4095, < 65536 set bits) blocks: A {{short}} (16 
> bits) for every 8 {{longs}} (512 bits) for a total of 256 bytes/DENSE_block. 
> Each {{short}} represents the number of set bits up to right before the 
> corresponding sub-block of 512 docIDs.
>  The \{{shorts}} can be computed sequentially or when the DENSE block is 
> flushed (probably the easiest). They should be stored as a simple sequence of 
> consecutive shorts for caching of lookups, one logically independent sequence 
> for each DENSE block. The logical position would be one sequence at the start 
> of every DENSE block.
>  Whether it is best to read all the 16 {{shorts}} up front when a DENSE block 
> is accessed or whether it is best to only read any individual {{short}} when 
> needed is not clear at this point.
>  * Variable Bits Per Value: A {{long}} (64 bits) for every 16384 numeric 
> values. Each {{long}} holds the offset to the corresponding block of values.
>  The offsets can be computed sequentially and should be stored as a simple 
> sequence of consecutive {{longs}} for caching of lookups.
>  The vBPV-offsets has the largest space overhead og the 3 jump-tables and a 
> lot of the 64 bits in each long are not used for most indexes. They could be 
> represented as a simple {{PackedInts}} sequence or {{MonotonicLongValues}}, 
> with the downsides of a potential lookup-time overhead and the need for doing 
> the compression after all offsets has been determined.
> I have no experience with the codec-parts responsible for creating 
> index-structures. I'm quite willing to take a stab at this, although I 
> probably won't do much about it before January 2019. Should anyone else wish 
> to adopt this JIRA-issue or co-work on it, I'll be happy to share.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: SOLR-12259: online index schema modification - adding docValues to existing indexed data?

[Adrien Grand]

UninvertingReader#wrap seems to skip uninverting if the field to
uninvert already has doc values of the expected type?

On Tue, Dec 18, 2018 at 8:24 PM Andrzej Białecki
 wrote:
>
> The unexpected part is that I would have expected the code to handle 
> franken-segments as well, because at some point we finally resorted to always 
> forcing the wrapping even for segments that don’t need it (ie. they claim the 
> field contains DVs) but the test is still failing.
>
> On 18 Dec 2018, at 19:05, Adrien Grand  wrote:
>
> I had a quick look and couldn't find anything to prevent what you
> called “franken-segments” in the Lucene test?
>
> On Tue, Dec 18, 2018 at 5:59 PM Erick Erickson  
> wrote:
>
>
> A couple of additions:
>
> AddDVMPLuceneTest2 does not use Solr constructs at all, so is the test
> we think is most interesting at this point, it won't lead anyone down
> the path of "what's all this Solr stuff and is it right" kinds of
> questions (believe me, we've spent some time on that path!). Please
> feel free to look at all the rest of it of course, but the place we're
> stuck is why this test fails.
>
> AddDvStress is intended as an integration-level test, it requires some
> special setup (in particular providing a particular configset), we put
> that together to reliably make the problem visible. We thought the new
> code was the issue at first and needed something to narrow down the
> possibilities...
>
> The reason we're obsessing about this is that it calls into question
> how segments are merged when "things change". We don't understand why
> this is happening at the Lucene level so don't know how to insure that
> things like the schema API in Solr aren't affected.
>
> Andrzej isn't the only one running out of ideas ;).
>
> On Tue, Dec 18, 2018 at 4:46 AM Andrzej Białecki  wrote:
>
>
> Hi,
>
> I'm working on a use case where an existing Solr setup needs to migrate to a 
> schema that uses docValues for faceting, instead of uninversion. This case 
> fits into a broader subject of SOLR-12259 (Robustly upgrade indexes). 
> However, in this case there are two major requirements for this migration 
> process:
>
> * data cannot be reindexed from scratch - I need to work with the already 
> indexed documents (which do contain the values needed for faceting, but these 
> values are simply indexed and not stored as doc values)
>
> * indexing can’t be stopped while the schema is being changed (the conversion 
> process needs to work on-the-fly while the collection is online, both for 
> searching and for updates). Collection reloads / reopening is ok but it’s not 
> ok to take the collection offline for several minutes (or hours).
>
> Together with Erick Erickson we implemented a solution that uses MergePolicy 
> (actually MergePolicyFactory in Solr) to enforce re-writing of segments that 
> no longer match the schema, ie. don’t contain docValues in a field where the 
> new schema requires it. This merge policy determines what segments need this 
> conversion and then forces the “merging” (actually re-writing) of these 
> segments by first wrapping them into UninvertingReader to supply docValues 
> where they are required by the new schema but actually are missing in the 
> segment’s data. This “AddDocValuesMergePolicy” (ADVMP for short) is supposed 
> to deal with the following types of segments:
>
> * old segments created before the schema change - these don’t contain any 
> docValues in the target fields and so they are wrapped in UninvertingReader 
> for merging (and for searching) according to the new schema.
>
> * new segments created after the schema change - if FieldInfo-s for these 
> fields claim that the segment already contains docValues where it should then 
> the segment is passed as-is to merging, otherwise it’s wrapped again. An 
> optimisation was also put here to “mark” the already converted segments using 
> a marker in SegmentInfo diagnostics map so that we can avoid re-checking and 
> re-converting already converted data.
>
> So, long story short, this process works very well when there’s no concurrent 
> indexing activity - all old segments are properly wrapped and re-written and 
> merging with new segments works as intended. However, in a situation with 
> concurrent indexing it works well but only for a short while. At some point 
> this conversion process seems to lose large percentage of the docValues, even 
> though it seems that at all points the source segments are properly wrapped - 
> the ADVMP merge policy adds a lot of debugging information to track the 
> source and type of segments across many levels of merging and whether they 
> were wrapped or not.
>
> My working theory is that somehow this schema change produces 
> “franken-segments” (while they still haven’t been flushed) where only some of 
> the most recent docs have the docValues and earlier ones don’t. As I 
> understand it, this should not happen in Solr because a schema change results 
> in a core reload. The 

[jira] [Commented] (LUCENE-8585) Create jump-tables for DocValues at index-time

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8585?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724407#comment-16724407
 ] 

David Smiley commented on LUCENE-8585:
--

+1 for this to be incorporated into 8.0.  For some people, this is going to be 
a huge optimization.

> Create jump-tables for DocValues at index-time
> --
>
> Key: LUCENE-8585
> URL: https://issues.apache.org/jira/browse/LUCENE-8585
> Project: Lucene - Core
>  Issue Type: Improvement
>  Components: core/codecs
>Affects Versions: master (8.0)
>Reporter: Toke Eskildsen
>Priority: Minor
>  Labels: performance
> Attachments: LUCENE-8585.patch, LUCENE-8585.patch, 
> make_patch_lucene8585.sh
>
>  Time Spent: 4h 50m
>  Remaining Estimate: 0h
>
> As noted in LUCENE-7589, lookup of DocValues should use jump-tables to avoid 
> long iterative walks. This is implemented in LUCENE-8374 at search-time 
> (first request for DocValues from a field in a segment), with the benefit of 
> working without changes to existing Lucene 7 indexes and the downside of 
> introducing a startup time penalty and a memory overhead.
> As discussed in LUCENE-8374, the codec should be updated to create these 
> jump-tables at index time. This eliminates the segment-open time & memory 
> penalties, with the potential downside of increasing index-time for DocValues.
> The three elements of LUCENE-8374 should be transferable to index-time 
> without much alteration of the core structures:
>  * {{IndexedDISI}} block offset and index skips: A {{long}} (64 bits) for 
> every 65536 documents, containing the offset of the block in 33 bits and the 
> index (number of set bits) up to the block in 31 bits.
>  It can be build sequentially and should be stored as a simple sequence of 
> consecutive longs for caching of lookups.
>  As it is fairly small, relative to document count, it might be better to 
> simply memory cache it?
>  * {{IndexedDISI}} DENSE (> 4095, < 65536 set bits) blocks: A {{short}} (16 
> bits) for every 8 {{longs}} (512 bits) for a total of 256 bytes/DENSE_block. 
> Each {{short}} represents the number of set bits up to right before the 
> corresponding sub-block of 512 docIDs.
>  The \{{shorts}} can be computed sequentially or when the DENSE block is 
> flushed (probably the easiest). They should be stored as a simple sequence of 
> consecutive shorts for caching of lookups, one logically independent sequence 
> for each DENSE block. The logical position would be one sequence at the start 
> of every DENSE block.
>  Whether it is best to read all the 16 {{shorts}} up front when a DENSE block 
> is accessed or whether it is best to only read any individual {{short}} when 
> needed is not clear at this point.
>  * Variable Bits Per Value: A {{long}} (64 bits) for every 16384 numeric 
> values. Each {{long}} holds the offset to the corresponding block of values.
>  The offsets can be computed sequentially and should be stored as a simple 
> sequence of consecutive {{longs}} for caching of lookups.
>  The vBPV-offsets has the largest space overhead og the 3 jump-tables and a 
> lot of the 64 bits in each long are not used for most indexes. They could be 
> represented as a simple {{PackedInts}} sequence or {{MonotonicLongValues}}, 
> with the downsides of a potential lookup-time overhead and the need for doing 
> the compression after all offsets has been determined.
> I have no experience with the codec-parts responsible for creating 
> index-structures. I'm quite willing to take a stab at this, although I 
> probably won't do much about it before January 2019. Should anyone else wish 
> to adopt this JIRA-issue or co-work on it, I'll be happy to share.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: SOLR-12259: online index schema modification - adding docValues to existing indexed data?

[Andrzej Białecki]

The unexpected part is that I would have expected the code to handle 
franken-segments as well, because at some point we finally resorted to always 
forcing the wrapping even for segments that don’t need it (ie. they claim the 
field contains DVs) but the test is still failing.

> On 18 Dec 2018, at 19:05, Adrien Grand  wrote:
> 
> I had a quick look and couldn't find anything to prevent what you
> called “franken-segments” in the Lucene test?
> 
> On Tue, Dec 18, 2018 at 5:59 PM Erick Erickson  
> wrote:
>> 
>> A couple of additions:
>> 
>> AddDVMPLuceneTest2 does not use Solr constructs at all, so is the test
>> we think is most interesting at this point, it won't lead anyone down
>> the path of "what's all this Solr stuff and is it right" kinds of
>> questions (believe me, we've spent some time on that path!). Please
>> feel free to look at all the rest of it of course, but the place we're
>> stuck is why this test fails.
>> 
>> AddDvStress is intended as an integration-level test, it requires some
>> special setup (in particular providing a particular configset), we put
>> that together to reliably make the problem visible. We thought the new
>> code was the issue at first and needed something to narrow down the
>> possibilities...
>> 
>> The reason we're obsessing about this is that it calls into question
>> how segments are merged when "things change". We don't understand why
>> this is happening at the Lucene level so don't know how to insure that
>> things like the schema API in Solr aren't affected.
>> 
>> Andrzej isn't the only one running out of ideas ;).
>> 
>> On Tue, Dec 18, 2018 at 4:46 AM Andrzej Białecki  wrote:
>>> 
>>> Hi,
>>> 
>>> I'm working on a use case where an existing Solr setup needs to migrate to 
>>> a schema that uses docValues for faceting, instead of uninversion. This 
>>> case fits into a broader subject of SOLR-12259 (Robustly upgrade indexes). 
>>> However, in this case there are two major requirements for this migration 
>>> process:
>>> 
>>> * data cannot be reindexed from scratch - I need to work with the already 
>>> indexed documents (which do contain the values needed for faceting, but 
>>> these values are simply indexed and not stored as doc values)
>>> 
>>> * indexing can’t be stopped while the schema is being changed (the 
>>> conversion process needs to work on-the-fly while the collection is online, 
>>> both for searching and for updates). Collection reloads / reopening is ok 
>>> but it’s not ok to take the collection offline for several minutes (or 
>>> hours).
>>> 
>>> Together with Erick Erickson we implemented a solution that uses 
>>> MergePolicy (actually MergePolicyFactory in Solr) to enforce re-writing of 
>>> segments that no longer match the schema, ie. don’t contain docValues in a 
>>> field where the new schema requires it. This merge policy determines what 
>>> segments need this conversion and then forces the “merging” (actually 
>>> re-writing) of these segments by first wrapping them into UninvertingReader 
>>> to supply docValues where they are required by the new schema but actually 
>>> are missing in the segment’s data. This “AddDocValuesMergePolicy” (ADVMP 
>>> for short) is supposed to deal with the following types of segments:
>>> 
>>> * old segments created before the schema change - these don’t contain any 
>>> docValues in the target fields and so they are wrapped in UninvertingReader 
>>> for merging (and for searching) according to the new schema.
>>> 
>>> * new segments created after the schema change - if FieldInfo-s for these 
>>> fields claim that the segment already contains docValues where it should 
>>> then the segment is passed as-is to merging, otherwise it’s wrapped again. 
>>> An optimisation was also put here to “mark” the already converted segments 
>>> using a marker in SegmentInfo diagnostics map so that we can avoid 
>>> re-checking and re-converting already converted data.
>>> 
>>> So, long story short, this process works very well when there’s no 
>>> concurrent indexing activity - all old segments are properly wrapped and 
>>> re-written and merging with new segments works as intended. However, in a 
>>> situation with concurrent indexing it works well but only for a short 
>>> while. At some point this conversion process seems to lose large percentage 
>>> of the docValues, even though it seems that at all points the source 
>>> segments are properly wrapped - the ADVMP merge policy adds a lot of 
>>> debugging information to track the source and type of segments across many 
>>> levels of merging and whether they were wrapped or not.
>>> 
>>> My working theory is that somehow this schema change produces 
>>> “franken-segments” (while they still haven’t been flushed) where only some 
>>> of the most recent docs have the docValues and earlier ones don’t. As I 
>>> understand it, this should not happen in Solr because a schema change 
>>> results in a core reload. The tracking information from ADVMP  seems to 

[jira] [Commented] (SOLR-13080) TermsQParserPlugin automaton method fails to sort input

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13080?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724385#comment-16724385
 ] 

David Smiley commented on SOLR-13080:
-

patch/PR with failing test is welcome :)

> TermsQParserPlugin automaton method fails to sort input
> ---
>
> Key: SOLR-13080
> URL: https://issues.apache.org/jira/browse/SOLR-13080
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: query parsers
>Affects Versions: 7.5
>Reporter: Daniel Lowe
>Priority: Minor
>
> The contract for Automata.makeStringUnion is that the input is sorted. As 
> BytesRef implements Comparable. The simplest fix would probably to make
> Arrays.sort(bytesRefs);
> The first line of automaton's makeFilter method in TermsQParserPlugin.
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-13079) refactor and harden common 'suspend-trigger' patern in autoscaling test setup

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13079?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724366#comment-16724366
 ] 

ASF subversion and git services commented on SOLR-13079:


Commit 28eaf354fb46ba3d5593551c690f4ae17b89ce23 in lucene-solr's branch 
refs/heads/branch_7x from Chris M. Hostetter
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=28eaf35 ]

SOLR-13079: refactor and harden common 'suspend-trigger' patern in autoscaling 
test setup

(cherry picked from commit 73299f0f220c44f9696e7deeb720440f9a2b0cdd)


> refactor and harden common 'suspend-trigger' patern in autoscaling test setup
> -
>
> Key: SOLR-13079
> URL: https://issues.apache.org/jira/browse/SOLR-13079
> Project: Solr
>  Issue Type: Sub-task
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Hoss Man
>Assignee: Hoss Man
>Priority: Major
> Attachments: SOLR-13079.patch
>
>
> A semi-common pattern in autoscaling tests is that immediately after 
> configuring the cluster, they (ab)use 
> {{AutoScalingHandlerTest.createAutoScalingRequest(...)}} to exec 
> {{suspend-trigger}} on {{".scheduled_maintenance"}}.
> The problem that can occasionally pops up in jenkins failures is that it's 
> possible the OverseerThread hasn't gotten a chance to add this implicit 
> trigger to the schedule by the time the test code tries to do this...
> {noformat}
>[junit4] ERROR   0.03s J1 | TestSimTriggerIntegration.testCooldown <<<
>[junit4]> Throwable #1: java.io.IOException: 
> java.util.concurrent.ExecutionException: java.io.IOException: 
> org.apache.solr.api.ApiBag$ExceptionWithErrObject: Error in command payload, 
> errors: [{suspend-trigger={name=.scheduled_maintenance}, errorMessages=[No 
> trigger exists with name: .scheduled_maintenance]}], 
>[junit4]>at 
> __randomizedtesting.SeedInfo.seed([51029DC2D3857924:60BCF026AD2F0CD6]:0)
>[junit4]>at 
> org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:654)
>[junit4]>at 
> org.apache.solr.cloud.autoscaling.sim.SimCloudManager$1.request(SimCloudManager.java:224)
>[junit4]>at 
> org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1260)
>[junit4]>at 
> org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.setupTest(TestSimTriggerIntegration.java:136)
> {noformat}
> I think we should:
> # refactor the {{AutoScalingHandlerTest.AutoScalingRequest}} static inner 
> class into {{CloudTestUtils}}
> # add a helper method to {{CloudTestUtils}} to make it easy to wait for a 
> trigger to be scheduled
> # add a helper method to {{CloudTestUtils}} to make it easy to susspend an 
> existing trigger



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-13079) refactor and harden common 'suspend-trigger' patern in autoscaling test setup

[ASF subversion and git services (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13079?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724367#comment-16724367
 ] 

ASF subversion and git services commented on SOLR-13079:


Commit 73299f0f220c44f9696e7deeb720440f9a2b0cdd in lucene-solr's branch 
refs/heads/master from Chris M. Hostetter
[ https://git-wip-us.apache.org/repos/asf?p=lucene-solr.git;h=73299f0 ]

SOLR-13079: refactor and harden common 'suspend-trigger' patern in autoscaling 
test setup


> refactor and harden common 'suspend-trigger' patern in autoscaling test setup
> -
>
> Key: SOLR-13079
> URL: https://issues.apache.org/jira/browse/SOLR-13079
> Project: Solr
>  Issue Type: Sub-task
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Hoss Man
>Assignee: Hoss Man
>Priority: Major
> Attachments: SOLR-13079.patch
>
>
> A semi-common pattern in autoscaling tests is that immediately after 
> configuring the cluster, they (ab)use 
> {{AutoScalingHandlerTest.createAutoScalingRequest(...)}} to exec 
> {{suspend-trigger}} on {{".scheduled_maintenance"}}.
> The problem that can occasionally pops up in jenkins failures is that it's 
> possible the OverseerThread hasn't gotten a chance to add this implicit 
> trigger to the schedule by the time the test code tries to do this...
> {noformat}
>[junit4] ERROR   0.03s J1 | TestSimTriggerIntegration.testCooldown <<<
>[junit4]> Throwable #1: java.io.IOException: 
> java.util.concurrent.ExecutionException: java.io.IOException: 
> org.apache.solr.api.ApiBag$ExceptionWithErrObject: Error in command payload, 
> errors: [{suspend-trigger={name=.scheduled_maintenance}, errorMessages=[No 
> trigger exists with name: .scheduled_maintenance]}], 
>[junit4]>at 
> __randomizedtesting.SeedInfo.seed([51029DC2D3857924:60BCF026AD2F0CD6]:0)
>[junit4]>at 
> org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:654)
>[junit4]>at 
> org.apache.solr.cloud.autoscaling.sim.SimCloudManager$1.request(SimCloudManager.java:224)
>[junit4]>at 
> org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1260)
>[junit4]>at 
> org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.setupTest(TestSimTriggerIntegration.java:136)
> {noformat}
> I think we should:
> # refactor the {{AutoScalingHandlerTest.AutoScalingRequest}} static inner 
> class into {{CloudTestUtils}}
> # add a helper method to {{CloudTestUtils}} to make it easy to wait for a 
> trigger to be scheduled
> # add a helper method to {{CloudTestUtils}} to make it easy to susspend an 
> existing trigger



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12304) Interesting Terms parameter is ignored by MLT Component

[Lucene/Solr QA (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12304?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724342#comment-16724342
 ] 

Lucene/Solr QA commented on SOLR-12304:
---

| (/) *{color:green}+1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green}  0m 
 0s{color} | {color:green} The patch appears to include 1 new or modified test 
files. {color} |
|| || || || {color:brown} master Compile Tests {color} ||
| {color:green}+1{color} | {color:green} compile {color} | {color:green}  1m 
51s{color} | {color:green} master passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} compile {color} | {color:green}  1m 
36s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green}  1m 
36s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Release audit (RAT) {color} | 
{color:green}  1m 36s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Check forbidden APIs {color} | 
{color:green}  1m 36s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Validate source patterns {color} | 
{color:green}  1m 36s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 38m 
40s{color} | {color:green} core in the patch passed. {color} |
| {color:black}{color} | {color:black} {color} | {color:black} 44m 53s{color} | 
{color:black} {color} |
\\
\\
|| Subsystem || Report/Notes ||
| JIRA Issue | SOLR-12304 |
| JIRA Patch URL | 
https://issues.apache.org/jira/secure/attachment/12921774/SOLR-12304.patch |
| Optional Tests |  compile  javac  unit  ratsources  checkforbiddenapis  
validatesourcepatterns  |
| uname | Linux lucene1-us-west 4.4.0-137-generic #163~14.04.1-Ubuntu SMP Mon 
Sep 24 17:14:57 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | ant |
| Personality | 
/home/jenkins/jenkins-slave/workspace/PreCommit-SOLR-Build/sourcedir/dev-tools/test-patch/lucene-solr-yetus-personality.sh
 |
| git revision | master / dcd4a28 |
| ant | version: Apache Ant(TM) version 1.9.3 compiled on July 24 2018 |
| Default Java | 1.8.0_191 |
|  Test Results | 
https://builds.apache.org/job/PreCommit-SOLR-Build/252/testReport/ |
| modules | C: solr/core U: solr/core |
| Console output | 
https://builds.apache.org/job/PreCommit-SOLR-Build/252/console |
| Powered by | Apache Yetus 0.7.0   http://yetus.apache.org |


This message was automatically generated.



> Interesting Terms parameter is ignored by MLT Component
> ---
>
> Key: SOLR-12304
> URL: https://issues.apache.org/jira/browse/SOLR-12304
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: MoreLikeThis
>Affects Versions: 7.2
>Reporter: Alessandro Benedetti
>Priority: Major
> Attachments: SOLR-12304.patch, SOLR-12304.patch
>
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> Currently the More Like This component just ignores the mlt.InterestingTerms 
> parameter ( which is usable by the MoreLikeThisHandler).
> Scope of this issue is to fix the bug and add related tests ( which will 
> succeed after the fix )
> *N.B.* MoreLikeThisComponent and MoreLikeThisHandler are very coupled and the 
> tests for the MoreLikeThisHandler are intersecting the MoreLikeThisComponent 
> ones .
>  It is out of scope for this issue any consideration or refactor of that.
>  Other issues will follow.
> *N.B.* out of scope for this issue is the distributed case, which is much 
> more complicated and requires much deeper investigations



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-13035) Utilize solr.data.home / solrDataHome in solr.xml to set all writable files in single directory

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13035?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724318#comment-16724318
 ] 

David Smiley commented on SOLR-13035:
-

Nice plan Jan.  Thanks for working on this everyone; it's important.

> Utilize solr.data.home / solrDataHome in solr.xml to set all writable files 
> in single directory
> ---
>
> Key: SOLR-13035
> URL: https://issues.apache.org/jira/browse/SOLR-13035
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Amrit Sarkar
>Priority: Major
> Attachments: SOLR-13035.patch, SOLR-13035.patch
>
>
> {{solr.data.home}} system property or {{solrDataHome}} in _solr.xml_ is 
> already available as per SOLR-6671.
> The writable content in Solr are index files, core properties, and ZK data if 
> embedded zookeeper is started in SolrCloud mode. It would be great if all 
> writable content can come under the same directory to have separate READ-ONLY 
> and WRITE-ONLY directories.
> It can then also solve official docker Solr image issues:
> https://github.com/docker-solr/docker-solr/issues/74
> https://github.com/docker-solr/docker-solr/issues/133



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

Re: SOLR-12259: online index schema modification - adding docValues to existing indexed data?

[Adrien Grand]

I had a quick look and couldn't find anything to prevent what you
called “franken-segments” in the Lucene test?

On Tue, Dec 18, 2018 at 5:59 PM Erick Erickson  wrote:
>
> A couple of additions:
>
> AddDVMPLuceneTest2 does not use Solr constructs at all, so is the test
> we think is most interesting at this point, it won't lead anyone down
> the path of "what's all this Solr stuff and is it right" kinds of
> questions (believe me, we've spent some time on that path!). Please
> feel free to look at all the rest of it of course, but the place we're
> stuck is why this test fails.
>
> AddDvStress is intended as an integration-level test, it requires some
> special setup (in particular providing a particular configset), we put
> that together to reliably make the problem visible. We thought the new
> code was the issue at first and needed something to narrow down the
> possibilities...
>
> The reason we're obsessing about this is that it calls into question
> how segments are merged when "things change". We don't understand why
> this is happening at the Lucene level so don't know how to insure that
> things like the schema API in Solr aren't affected.
>
> Andrzej isn't the only one running out of ideas ;).
>
> On Tue, Dec 18, 2018 at 4:46 AM Andrzej Białecki  wrote:
> >
> > Hi,
> >
> > I'm working on a use case where an existing Solr setup needs to migrate to 
> > a schema that uses docValues for faceting, instead of uninversion. This 
> > case fits into a broader subject of SOLR-12259 (Robustly upgrade indexes). 
> > However, in this case there are two major requirements for this migration 
> > process:
> >
> > * data cannot be reindexed from scratch - I need to work with the already 
> > indexed documents (which do contain the values needed for faceting, but 
> > these values are simply indexed and not stored as doc values)
> >
> > * indexing can’t be stopped while the schema is being changed (the 
> > conversion process needs to work on-the-fly while the collection is online, 
> > both for searching and for updates). Collection reloads / reopening is ok 
> > but it’s not ok to take the collection offline for several minutes (or 
> > hours).
> >
> > Together with Erick Erickson we implemented a solution that uses 
> > MergePolicy (actually MergePolicyFactory in Solr) to enforce re-writing of 
> > segments that no longer match the schema, ie. don’t contain docValues in a 
> > field where the new schema requires it. This merge policy determines what 
> > segments need this conversion and then forces the “merging” (actually 
> > re-writing) of these segments by first wrapping them into UninvertingReader 
> > to supply docValues where they are required by the new schema but actually 
> > are missing in the segment’s data. This “AddDocValuesMergePolicy” (ADVMP 
> > for short) is supposed to deal with the following types of segments:
> >
> > * old segments created before the schema change - these don’t contain any 
> > docValues in the target fields and so they are wrapped in UninvertingReader 
> > for merging (and for searching) according to the new schema.
> >
> > * new segments created after the schema change - if FieldInfo-s for these 
> > fields claim that the segment already contains docValues where it should 
> > then the segment is passed as-is to merging, otherwise it’s wrapped again. 
> > An optimisation was also put here to “mark” the already converted segments 
> > using a marker in SegmentInfo diagnostics map so that we can avoid 
> > re-checking and re-converting already converted data.
> >
> > So, long story short, this process works very well when there’s no 
> > concurrent indexing activity - all old segments are properly wrapped and 
> > re-written and merging with new segments works as intended. However, in a 
> > situation with concurrent indexing it works well but only for a short 
> > while. At some point this conversion process seems to lose large percentage 
> > of the docValues, even though it seems that at all points the source 
> > segments are properly wrapped - the ADVMP merge policy adds a lot of 
> > debugging information to track the source and type of segments across many 
> > levels of merging and whether they were wrapped or not.
> >
> > My working theory is that somehow this schema change produces 
> > “franken-segments” (while they still haven’t been flushed) where only some 
> > of the most recent docs have the docValues and earlier ones don’t. As I 
> > understand it, this should not happen in Solr because a schema change 
> > results in a core reload. The tracking information from ADVMP  seems to 
> > indicate that all generations of segments, both those that were flushed and 
> > merged earlier, have been properly wrapped.
> >
> > My alternate theory is that there’s some bug in the doc values merging 
> > process when UninvertingReader is involved, because this problem occurs 
> > also when we modify ADVMP to always force the wrapping of all segments in 
> > 

[JENKINS] Lucene-Solr-BadApples-Tests-7.x - Build # 241 - Still Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/241/

1 tests failed.
FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testNodeMarkersRegistration

Error Message:
java.util.concurrent.ExecutionException: java.io.IOException: 
org.apache.solr.api.ApiBag$ExceptionWithErrObject: Error in command payload, 
errors: [{suspend-trigger={name=.scheduled_maintenance}, errorMessages=[No 
trigger exists with name: .scheduled_maintenance]}], 

Stack Trace:
java.io.IOException: java.util.concurrent.ExecutionException: 
java.io.IOException: org.apache.solr.api.ApiBag$ExceptionWithErrObject: Error 
in command payload, errors: [{suspend-trigger={name=.scheduled_maintenance}, 
errorMessages=[No trigger exists with name: .scheduled_maintenance]}], 
at 
__randomizedtesting.SeedInfo.seed([763E2A4E1F78566E:6E84A242114D9B81]:0)
at 
org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:654)
at 
org.apache.solr.cloud.autoscaling.sim.SimCloudManager$1.request(SimCloudManager.java:224)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1260)
at 
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.setupTest(TestSimTriggerIntegration.java:136)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:972)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 

[jira] [Commented] (SOLR-13075) Harden SaslZkACLProviderTest.

[Erick Erickson (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13075?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724271#comment-16724271
 ] 

Erick Erickson commented on SOLR-13075:
---

I have a Jenkins log for this if anyone else is interested.

> Harden SaslZkACLProviderTest.
> -
>
> Key: SOLR-13075
> URL: https://issues.apache.org/jira/browse/SOLR-13075
> Project: Solr
>  Issue Type: Sub-task
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Mark Miller
>Assignee: Erick Erickson
>Priority: Major
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-12768) Determine how _nest_path_ should be analyzed to support various use-cases

[David Smiley (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-12768?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724260#comment-16724260
 ] 

David Smiley commented on SOLR-12768:
-

Simple proposal:
* Use a new FieldType subclass to a simplify upgrades and enable ease of use
* Use one index token instead of path tokenizing at this stage.  This is 
lighter-weight when a user might not even need/want to query on it.  Instead, 
queries would use wildcards on it to express relationships.  Some day in the 
future, someone could make an easy to use query parser and/or query language 
that would build the appropriate wildcard patterns.

The index analyzer would simply be the indexed equivalent of:
{code:xml}
  
  
  
{code}
Notice the last pattern is simplified and fixes a bug in the current test that 
will match all digits instead of only those after a pound.  I wrote a unit test 
for that fix.

CC [~moshebla]


> Determine how _nest_path_ should be analyzed to support various use-cases
> -
>
> Key: SOLR-12768
> URL: https://issues.apache.org/jira/browse/SOLR-12768
> Project: Solr
>  Issue Type: Sub-task
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: David Smiley
>Priority: Blocker
> Fix For: master (8.0)
>
>
> We know we need {{\_nest\_path\_}} in the schema for the new nested documents 
> support, and we loosely know what goes in it.  From a DocValues perspective, 
> we've got it down; though we might tweak it.  From an indexing (text 
> analysis) perspective, we're not quite sure yet, though we've got a test 
> schema, {{schema-nest.xml}} with a decent shot at it.  Ultimately, how we 
> index it will depend on the query/filter use-cases we need to support.  So 
> we'll review some of them here.
> TBD: Not sure if the outcome of this task is just a "decide" or wether we 
> also potentially add a few tests for some of these cases, and/or if we also 
> add a FieldType to make declaring it as easy as a one-liner.  A FieldType 
> would have other benefits too once we're ready to make querying on the path 
> easier.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-7.6-Windows (64bit/jdk1.8.0_172) - Build # 25 - Still Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.6-Windows/25/
Java: 64bit/jdk1.8.0_172 -XX:-UseCompressedOops -XX:+UseG1GC

9 tests failed.
FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestSimGenericDistributedQueue.testDistributedQueueBlocking

Error Message:


Stack Trace:
java.util.concurrent.TimeoutException
at 
__randomizedtesting.SeedInfo.seed([DC1326BC5698C384:99B954C512CA7FF0]:0)
at java.util.concurrent.FutureTask.get(FutureTask.java:205)
at 
org.apache.solr.cloud.autoscaling.sim.TestSimDistributedQueue.testDistributedQueueBlocking(TestSimDistributedQueue.java:117)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)


FAILED:  
junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.sim.TestSimGenericDistributedQueue

Error Message:
Suite timeout exceeded (>= 720 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 720 msec).
at __randomizedtesting.SeedInfo.seed([DC1326BC5698C384]:0)

Re: SOLR-12259: online index schema modification - adding docValues to existing indexed data?

[Erick Erickson]

A couple of additions:

AddDVMPLuceneTest2 does not use Solr constructs at all, so is the test
we think is most interesting at this point, it won't lead anyone down
the path of "what's all this Solr stuff and is it right" kinds of
questions (believe me, we've spent some time on that path!). Please
feel free to look at all the rest of it of course, but the place we're
stuck is why this test fails.

AddDvStress is intended as an integration-level test, it requires some
special setup (in particular providing a particular configset), we put
that together to reliably make the problem visible. We thought the new
code was the issue at first and needed something to narrow down the
possibilities...

The reason we're obsessing about this is that it calls into question
how segments are merged when "things change". We don't understand why
this is happening at the Lucene level so don't know how to insure that
things like the schema API in Solr aren't affected.

Andrzej isn't the only one running out of ideas ;).

On Tue, Dec 18, 2018 at 4:46 AM Andrzej Białecki  wrote:
>
> Hi,
>
> I'm working on a use case where an existing Solr setup needs to migrate to a 
> schema that uses docValues for faceting, instead of uninversion. This case 
> fits into a broader subject of SOLR-12259 (Robustly upgrade indexes). 
> However, in this case there are two major requirements for this migration 
> process:
>
> * data cannot be reindexed from scratch - I need to work with the already 
> indexed documents (which do contain the values needed for faceting, but these 
> values are simply indexed and not stored as doc values)
>
> * indexing can’t be stopped while the schema is being changed (the conversion 
> process needs to work on-the-fly while the collection is online, both for 
> searching and for updates). Collection reloads / reopening is ok but it’s not 
> ok to take the collection offline for several minutes (or hours).
>
> Together with Erick Erickson we implemented a solution that uses MergePolicy 
> (actually MergePolicyFactory in Solr) to enforce re-writing of segments that 
> no longer match the schema, ie. don’t contain docValues in a field where the 
> new schema requires it. This merge policy determines what segments need this 
> conversion and then forces the “merging” (actually re-writing) of these 
> segments by first wrapping them into UninvertingReader to supply docValues 
> where they are required by the new schema but actually are missing in the 
> segment’s data. This “AddDocValuesMergePolicy” (ADVMP for short) is supposed 
> to deal with the following types of segments:
>
> * old segments created before the schema change - these don’t contain any 
> docValues in the target fields and so they are wrapped in UninvertingReader 
> for merging (and for searching) according to the new schema.
>
> * new segments created after the schema change - if FieldInfo-s for these 
> fields claim that the segment already contains docValues where it should then 
> the segment is passed as-is to merging, otherwise it’s wrapped again. An 
> optimisation was also put here to “mark” the already converted segments using 
> a marker in SegmentInfo diagnostics map so that we can avoid re-checking and 
> re-converting already converted data.
>
> So, long story short, this process works very well when there’s no concurrent 
> indexing activity - all old segments are properly wrapped and re-written and 
> merging with new segments works as intended. However, in a situation with 
> concurrent indexing it works well but only for a short while. At some point 
> this conversion process seems to lose large percentage of the docValues, even 
> though it seems that at all points the source segments are properly wrapped - 
> the ADVMP merge policy adds a lot of debugging information to track the 
> source and type of segments across many levels of merging and whether they 
> were wrapped or not.
>
> My working theory is that somehow this schema change produces 
> “franken-segments” (while they still haven’t been flushed) where only some of 
> the most recent docs have the docValues and earlier ones don’t. As I 
> understand it, this should not happen in Solr because a schema change results 
> in a core reload. The tracking information from ADVMP  seems to indicate that 
> all generations of segments, both those that were flushed and merged earlier, 
> have been properly wrapped.
>
> My alternate theory is that there’s some bug in the doc values merging 
> process when UninvertingReader is involved, because this problem occurs also 
> when we modify ADVMP to always force the wrapping of all segments in 
> UninvertingReader-s. The percentage of lost doc values is sometimes quite 
> large, up to 50%, perhaps it’s a bug somewhere where the code accounts for 
> the presence of doc values in FieldCacheImpl?
>
> Together with Erick we implemented a bunch of tests that illustrate this 
> issue - both the tests and the code can be found on branch 

[jira] [Commented] (SOLR-13081) In-Place Update doesn't work when route.field is defined

[Mikhail Khludnev (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13081?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724198#comment-16724198
 ] 

Mikhail Khludnev commented on SOLR-13081:
-

It make sense to me. I worry where we can commit public method changes. I 
appreciate a review by other pair of eyes, I'll take care about commit. 

> In-Place Update doesn't work when route.field is defined
> 
>
> Key: SOLR-13081
> URL: https://issues.apache.org/jira/browse/SOLR-13081
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: update
>Reporter: Dr Oleg Savrasov
>Priority: Major
> Attachments: SOLR-13081.patch
>
>
> As soon as cloud collection is configured with route.field property, In-Place 
> Updates are not applied anymore. This happens because 
> AtomicUpdateDocumentMerger skips only id and version fields and doesn't 
> verify configured route.field.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Comment Edited] (SOLR-13081) In-Place Update doesn't work when route.field is defined

[Mikhail Khludnev (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13081?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724198#comment-16724198
 ] 

Mikhail Khludnev edited comment on SOLR-13081 at 12/18/18 4:04 PM:
---

It makes sense to me. I worry where we can commit public method changes. I 
appreciate a review by other pair of eyes, I'll take care about commit. 


was (Author: mkhludnev):
It make sense to me. I worry where we can commit public method changes. I 
appreciate a review by other pair of eyes, I'll take care about commit. 

> In-Place Update doesn't work when route.field is defined
> 
>
> Key: SOLR-13081
> URL: https://issues.apache.org/jira/browse/SOLR-13081
> Project: Solr
>  Issue Type: Bug
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: update
>Reporter: Dr Oleg Savrasov
>Priority: Major
> Attachments: SOLR-13081.patch
>
>
> As soon as cloud collection is configured with route.field property, In-Place 
> Updates are not applied anymore. This happens because 
> AtomicUpdateDocumentMerger skips only id and version fields and doesn't 
> verify configured route.field.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Comment Edited] (LUCENE-8581) Change LatLonShape encoding to use 4 BYTES Per Dimension

[Ignacio Vera (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8581?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724194#comment-16724194
 ] 

Ignacio Vera edited comment on LUCENE-8581 at 12/18/18 4:03 PM:


For some reason commits were not linked to the issue:

master:

d185ba99de0828bd31c2f6bcd74dc96fe8c6a452

 

branch_7x:

84fcfae3dc501e9fb1c777565aab0f9ca30cba

 


was (Author: ivera):
For some reason commits were not liken to the issue:

master:

d185ba99de0828bd31c2f6bcd74dc96fe8c6a452

 

branch_7x:

84fcfae3dc501e9fb1c777565aab0f9ca30cba

 

> Change LatLonShape encoding to use 4 BYTES Per Dimension
> 
>
> Key: LUCENE-8581
> URL: https://issues.apache.org/jira/browse/LUCENE-8581
> Project: Lucene - Core
>  Issue Type: New Feature
>Reporter: Nicholas Knize
>Assignee: Ignacio Vera
>Priority: Major
> Fix For: master (8.0), 7.7
>
> Attachments: LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch
>
>
> {{LatLonShape}} tessellated triangles currently use a relatively naive 
> encoding with the first four dimensions as the bounding box of the triangle 
> and the last three dimensions as the vertices of the triangle. To encode the 
> {{x,y}} vertices in the last three dimensions requires {{bytesPerDim}} to be 
> set to 8, with 4 bytes for the x & y axis, respectively. We can reduce 
> {{bytesPerDim}} to 4 by encoding the index(es) of the vertices shared by the 
> bounding box along with the orientation of the triangle. This also opens the 
> door for supporting {{CONTAINS}} queries.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8581) Change LatLonShape encoding to use 4 BYTES Per Dimension

[Adrien Grand (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8581?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724186#comment-16724186
 ] 

Adrien Grand commented on LUCENE-8581:
--

+1

> Change LatLonShape encoding to use 4 BYTES Per Dimension
> 
>
> Key: LUCENE-8581
> URL: https://issues.apache.org/jira/browse/LUCENE-8581
> Project: Lucene - Core
>  Issue Type: New Feature
>Reporter: Nicholas Knize
>Assignee: Ignacio Vera
>Priority: Major
> Attachments: LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch
>
>
> {{LatLonShape}} tessellated triangles currently use a relatively naive 
> encoding with the first four dimensions as the bounding box of the triangle 
> and the last three dimensions as the vertices of the triangle. To encode the 
> {{x,y}} vertices in the last three dimensions requires {{bytesPerDim}} to be 
> set to 8, with 4 bytes for the x & y axis, respectively. We can reduce 
> {{bytesPerDim}} to 4 by encoding the index(es) of the vertices shared by the 
> bounding box along with the orientation of the triangle. This also opens the 
> door for supporting {{CONTAINS}} queries.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS-MAVEN] Lucene-Solr-Maven-master #2439: POMs out of sync

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-Maven-master/2439/

No tests ran.

Build Log:
[...truncated 29378 lines...]
  [mvn] [INFO] -
  [mvn] [INFO] -
  [mvn] [ERROR] COMPILATION ERROR : 
  [mvn] [INFO] -

[...truncated 1172 lines...]
  [mvn] [INFO] -
  [mvn] [INFO] -
  [mvn] [ERROR] COMPILATION ERROR : 
  [mvn] [INFO] -

[...truncated 354 lines...]
  [mvn] [INFO] Compiling 13 source files to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Maven-master/maven-build/lucene/classification/target/test-classes
  [mvn] [INFO] -
  [mvn] [ERROR] COMPILATION ERROR : 
  [mvn] [INFO] -

[...truncated 272 lines...]
  [mvn] [INFO] -
  [mvn] [INFO] -
  [mvn] [ERROR] COMPILATION ERROR : 
  [mvn] [INFO] -

[...truncated 601 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Maven-master/build.xml:679: The 
following error occurred while executing this line:
: Java returned: 1

Total time: 17 minutes 46 seconds
Build step 'Invoke Ant' marked build as failure
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Assigned] (LUCENE-8581) Change LatLonShape encoding to use 4 BYTES Per Dimension

[Nicholas Knize (JIRA)]

 [ 
https://issues.apache.org/jira/browse/LUCENE-8581?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Nicholas Knize reassigned LUCENE-8581:
--

Assignee: Ignacio Vera  (was: Nicholas Knize)

> Change LatLonShape encoding to use 4 BYTES Per Dimension
> 
>
> Key: LUCENE-8581
> URL: https://issues.apache.org/jira/browse/LUCENE-8581
> Project: Lucene - Core
>  Issue Type: New Feature
>Reporter: Nicholas Knize
>Assignee: Ignacio Vera
>Priority: Major
> Attachments: LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch
>
>
> {{LatLonShape}} tessellated triangles currently use a relatively naive 
> encoding with the first four dimensions as the bounding box of the triangle 
> and the last three dimensions as the vertices of the triangle. To encode the 
> {{x,y}} vertices in the last three dimensions requires {{bytesPerDim}} to be 
> set to 8, with 4 bytes for the x & y axis, respectively. We can reduce 
> {{bytesPerDim}} to 4 by encoding the index(es) of the vertices shared by the 
> bounding box along with the orientation of the triangle. This also opens the 
> door for supporting {{CONTAINS}} queries.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8581) Change LatLonShape encoding to use 4 BYTES Per Dimension

[Nicholas Knize (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8581?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724178#comment-16724178
 ] 

Nicholas Knize commented on LUCENE-8581:


Thanks for all of your work on this [~ivera]. This LGTM!

> Change LatLonShape encoding to use 4 BYTES Per Dimension
> 
>
> Key: LUCENE-8581
> URL: https://issues.apache.org/jira/browse/LUCENE-8581
> Project: Lucene - Core
>  Issue Type: New Feature
>Reporter: Nicholas Knize
>Assignee: Nicholas Knize
>Priority: Major
> Attachments: LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch
>
>
> {{LatLonShape}} tessellated triangles currently use a relatively naive 
> encoding with the first four dimensions as the bounding box of the triangle 
> and the last three dimensions as the vertices of the triangle. To encode the 
> {{x,y}} vertices in the last three dimensions requires {{bytesPerDim}} to be 
> set to 8, with 4 bytes for the x & y axis, respectively. We can reduce 
> {{bytesPerDim}} to 4 by encoding the index(es) of the vertices shared by the 
> bounding box along with the orientation of the triangle. This also opens the 
> door for supporting {{CONTAINS}} queries.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Assigned] (LUCENE-8581) Change LatLonShape encoding to use 4 BYTES Per Dimension

[Nicholas Knize (JIRA)]

 [ 
https://issues.apache.org/jira/browse/LUCENE-8581?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Nicholas Knize reassigned LUCENE-8581:
--

Assignee: Nicholas Knize  (was: Ignacio Vera)

> Change LatLonShape encoding to use 4 BYTES Per Dimension
> 
>
> Key: LUCENE-8581
> URL: https://issues.apache.org/jira/browse/LUCENE-8581
> Project: Lucene - Core
>  Issue Type: New Feature
>Reporter: Nicholas Knize
>Assignee: Nicholas Knize
>Priority: Major
> Attachments: LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, LUCENE-8581.patch, 
> LUCENE-8581.patch
>
>
> {{LatLonShape}} tessellated triangles currently use a relatively naive 
> encoding with the first four dimensions as the bounding box of the triangle 
> and the last three dimensions as the vertices of the triangle. To encode the 
> {{x,y}} vertices in the last three dimensions requires {{bytesPerDim}} to be 
> set to 8, with 4 bytes for the x & y axis, respectively. We can reduce 
> {{bytesPerDim}} to 4 by encoding the index(es) of the vertices shared by the 
> bounding box along with the orientation of the triangle. This also opens the 
> door for supporting {{CONTAINS}} queries.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-repro - Build # 2516 - Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-repro/2516/

[...truncated 28 lines...]
[repro] Jenkins log URL: 
https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-master/238/consoleText

[repro] Revision: 5e305e2f00fa79f8fe9abb6f949a196c911d1377

[repro] Repro line:  ant test  
-Dtestcase=ChaosMonkeyNothingIsSafeWithPullReplicasTest -Dtests.method=test 
-Dtests.seed=B94CBB37570DFA37 -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.badapples=true -Dtests.locale=zh-HK -Dtests.timezone=Asia/Aqtau 
-Dtests.asserts=true -Dtests.file.encoding=US-ASCII

[repro] Repro line:  ant test  -Dtestcase=HdfsRecoveryZkTest 
-Dtests.method=test -Dtests.seed=B94CBB37570DFA37 -Dtests.multiplier=2 
-Dtests.slow=true -Dtests.badapples=true -Dtests.locale=es-BO 
-Dtests.timezone=Australia/Currie -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[repro] Repro line:  ant test  -Dtestcase=TestSimTriggerIntegration 
-Dtests.method=testNodeAddedTriggerRestoreState -Dtests.seed=B94CBB37570DFA37 
-Dtests.multiplier=2 -Dtests.slow=true -Dtests.badapples=true 
-Dtests.locale=nn-NO -Dtests.timezone=America/Anchorage -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[repro] git rev-parse --abbrev-ref HEAD
[repro] git rev-parse HEAD
[repro] Initial local git branch/revision: 
3be5b59907acc3049121e980bee71df01185d40e
[repro] git fetch
[repro] git checkout 5e305e2f00fa79f8fe9abb6f949a196c911d1377

[...truncated 2 lines...]
[repro] git merge --ff-only

[...truncated 1 lines...]
[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]solr/core
[repro]   HdfsRecoveryZkTest
[repro]   TestSimTriggerIntegration
[repro]   ChaosMonkeyNothingIsSafeWithPullReplicasTest
[repro] ant compile-test

[...truncated 3593 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=15 
-Dtests.class="*.HdfsRecoveryZkTest|*.TestSimTriggerIntegration|*.ChaosMonkeyNothingIsSafeWithPullReplicasTest"
 -Dtests.showOutput=onerror  -Dtests.seed=B94CBB37570DFA37 -Dtests.multiplier=2 
-Dtests.slow=true -Dtests.badapples=true -Dtests.locale=es-BO 
-Dtests.timezone=Australia/Currie -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[...truncated 3129 lines...]
[repro] Setting last failure code to 256

[repro] Failures:
[repro]   0/5 failed: 
org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
[repro]   0/5 failed: org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest
[repro]   2/5 failed: 
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[repro] git checkout 3be5b59907acc3049121e980bee71df01185d40e

[...truncated 2 lines...]
[repro] Exiting with code 256

[...truncated 6 lines...]

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-13035) Utilize solr.data.home / solrDataHome in solr.xml to set all writable files in single directory

[JIRA]

[ 
https://issues.apache.org/jira/browse/SOLR-13035?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724152#comment-16724152
 ] 

Jan Høydahl commented on SOLR-13035:


I think we are all confused, and I think the problem is that the current 
situation is not result of a careful design, but rather random addition of 
various paths since Solr 1.x :)

So if the goal is to rectify this and create a clearer story where you can 
install binaries R/O in SOLR_TIP and mount another partition for all R/W data, 
including index, logs, config (solr-home), PID etc, then I don't see how 
defaulting to SOLR_TIP will get us closer to that goal.

The Linux installer puts everything below /var/solr but it does not assign any 
environment variable to that path. Instead, the various SOLR_HOME, 
SOLR_DATA_DIR, SOLR_LOGS_DIR, PID_DIR, LOG4J_PROPS are explicitly placed in 
various locations below that path.

So one way could be to let SOLR_VAR_ROOT, if not set, default to 
{{SOLR_TIP/var}}, and thus if you simply unzip and start, this would be the 
picture:
{noformat}
SOLR_TIP  = /path/to/solr
SOLR_VAR_ROOT = $SOLR_TIP/var --> /path/to/solr/var
SOLR_HOME = $SOLR_VAR_ROOT/home   --> /path/to/solr/var/home
SOLR_DATA_HOME= $SOLR_VAR_ROOT/data   --> /path/to/solr/var/data
SOLR_LOGS_DIR = $SOLR_VAR_ROOT/logs   --> /path/to/solr/var/logs
PID_DIR   = $SOLR_VAR_ROOT--> /path/to/solr/var{noformat}
Then you could install a big disk and move SOLR_VAR_ROOT to 
{{/mnt/bigdisk/solr}} and you're done. Likewise you could pass an 
{{--vardir=/var/solr}} argument to the install script to tell it where R/W data 
lives.

> Utilize solr.data.home / solrDataHome in solr.xml to set all writable files 
> in single directory
> ---
>
> Key: SOLR-13035
> URL: https://issues.apache.org/jira/browse/SOLR-13035
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Amrit Sarkar
>Priority: Major
> Attachments: SOLR-13035.patch, SOLR-13035.patch
>
>
> {{solr.data.home}} system property or {{solrDataHome}} in _solr.xml_ is 
> already available as per SOLR-6671.
> The writable content in Solr are index files, core properties, and ZK data if 
> embedded zookeeper is started in SolrCloud mode. It would be great if all 
> writable content can come under the same directory to have separate READ-ONLY 
> and WRITE-ONLY directories.
> It can then also solve official docker Solr image issues:
> https://github.com/docker-solr/docker-solr/issues/74
> https://github.com/docker-solr/docker-solr/issues/133



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS-MAVEN] Lucene-Solr-Maven-7.x #388: POMs out of sync

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-Maven-7.x/388/

No tests ran.

Build Log:
[...truncated 29388 lines...]
  [mvn] [INFO] -
  [mvn] [INFO] -
  [mvn] [ERROR] COMPILATION ERROR : 
  [mvn] [INFO] -

[...truncated 1133 lines...]
  [mvn] [INFO] -
  [mvn] [INFO] -
  [mvn] [ERROR] COMPILATION ERROR : 
  [mvn] [INFO] -

[...truncated 292 lines...]
  [mvn] [INFO] Compiling 13 source files to 
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Maven-7.x/maven-build/lucene/classification/target/test-classes
  [mvn] [INFO] -
  [mvn] [ERROR] COMPILATION ERROR : 
  [mvn] [INFO] -

[...truncated 272 lines...]
  [mvn] [INFO] -
  [mvn] [INFO] -
  [mvn] [ERROR] COMPILATION ERROR : 
  [mvn] [INFO] -

[...truncated 494 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Maven-7.x/build.xml:679: The 
following error occurred while executing this line:
: Java returned: 1

Total time: 16 minutes 29 seconds
Build step 'Invoke Ant' marked build as failure
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8601) Adding attributes to IndexFieldType

[Lucene/Solr QA (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8601?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724142#comment-16724142
 ] 

Lucene/Solr QA commented on LUCENE-8601:


| (/) *{color:green}+1 overall{color}* |
\\
\\
|| Vote || Subsystem || Runtime || Comment ||
|| || || || {color:brown} Prechecks {color} ||
| {color:green}+1{color} | {color:green} test4tests {color} | {color:green}  0m 
 0s{color} | {color:green} The patch appears to include 2 new or modified test 
files. {color} |
|| || || || {color:brown} master Compile Tests {color} ||
| {color:green}+1{color} | {color:green} compile {color} | {color:green}  2m 
22s{color} | {color:green} master passed {color} |
|| || || || {color:brown} Patch Compile Tests {color} ||
| {color:green}+1{color} | {color:green} compile {color} | {color:green}  2m  
9s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} javac {color} | {color:green}  2m  
9s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Release audit (RAT) {color} | 
{color:green}  0m 40s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Check forbidden APIs {color} | 
{color:green}  0m 34s{color} | {color:green} the patch passed {color} |
| {color:green}+1{color} | {color:green} Validate source patterns {color} | 
{color:green}  0m 34s{color} | {color:green} the patch passed {color} |
|| || || || {color:brown} Other Tests {color} ||
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 14m 
55s{color} | {color:green} core in the patch passed. {color} |
| {color:green}+1{color} | {color:green} unit {color} | {color:green} 37m 
57s{color} | {color:green} core in the patch passed. {color} |
| {color:black}{color} | {color:black} {color} | {color:black} 61m  5s{color} | 
{color:black} {color} |
\\
\\
|| Subsystem || Report/Notes ||
| JIRA Issue | LUCENE-8601 |
| JIRA Patch URL | 
https://issues.apache.org/jira/secure/attachment/12952123/LUCENE-8601.04.patch |
| Optional Tests |  compile  javac  unit  ratsources  checkforbiddenapis  
validatesourcepatterns  |
| uname | Linux lucene1-us-west 4.4.0-137-generic #163~14.04.1-Ubuntu SMP Mon 
Sep 24 17:14:57 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | ant |
| Personality | 
/home/jenkins/jenkins-slave/workspace/PreCommit-LUCENE-Build/sourcedir/dev-tools/test-patch/lucene-solr-yetus-personality.sh
 |
| git revision | master / 3be5b59 |
| ant | version: Apache Ant(TM) version 1.9.3 compiled on July 24 2018 |
| Default Java | 1.8.0_191 |
|  Test Results | 
https://builds.apache.org/job/PreCommit-LUCENE-Build/140/testReport/ |
| modules | C: lucene/core solr/core U: . |
| Console output | 
https://builds.apache.org/job/PreCommit-LUCENE-Build/140/console |
| Powered by | Apache Yetus 0.7.0   http://yetus.apache.org |


This message was automatically generated.



> Adding attributes to IndexFieldType
> ---
>
> Key: LUCENE-8601
> URL: https://issues.apache.org/jira/browse/LUCENE-8601
> Project: Lucene - Core
>  Issue Type: Improvement
>  Components: core/index
>Affects Versions: 7.5
>Reporter: Murali Krishna P
>Priority: Major
> Attachments: LUCENE-8601.01.patch, LUCENE-8601.02.patch, 
> LUCENE-8601.03.patch, LUCENE-8601.04.patch, LUCENE-8601.patch
>
>
> Today, we can write a custom Field using custom IndexFieldType, but when the 
> DefaultIndexingChain converts [IndexFieldType to 
> FieldInfo|https://github.com/apache/lucene-solr/blob/master/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java#L662],
>  only few key informations such as indexing options and doc value type are 
> retained. The [Codec gets the 
> FieldInfo|https://github.com/apache/lucene-solr/blob/master/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java#L90],
>  but not the type details.
>   
>  FieldInfo has support for ['attributes'| 
> https://github.com/apache/lucene-solr/blob/master/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java#L47]
>  and it would be great if we can add 'attributes' to IndexFieldType also and 
> copy it to FieldInfo's 'attribute'.
>   
>  This would allow someone to write a custom codec (extending docvalueformat 
> for example) for only the 'special field' that he wants and delegate the rest 
> of the fields to the default codec.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-13035) Utilize solr.data.home / solrDataHome in solr.xml to set all writable files in single directory

[Amrit Sarkar (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13035?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724134#comment-16724134
 ] 

Amrit Sarkar commented on SOLR-13035:
-

Hi [~janhoy],

SOLR_TIP is the system property defined in Solr startup script which defines 
the parent path of Solr installation directory. If we unzip solr-7.5.0.tgz, 
{{[PATH-TO]/solr-7.5.0}} would be the SOLR_.TIP. So yes, a default Linux script 
install will point SOLR_TIP to {{/opt/solr}}.

SOLR_VAR_ROOT will be a great addition operation wise, but do we need another 
variable amidst various available, it is already quite confusing with 
{{instancePath}}, {{dataDir}} and others etc w.r.t cores. 
One can define absolute paths for DIR properties (SOLR_LOGS_DIR, SOLR_PID_DIR 
etc.) if desired and supporting relative paths will still work w.r.t current 
directory / SOLR_HOME / SOLR_TIP etc.

> Utilize solr.data.home / solrDataHome in solr.xml to set all writable files 
> in single directory
> ---
>
> Key: SOLR-13035
> URL: https://issues.apache.org/jira/browse/SOLR-13035
> Project: Solr
>  Issue Type: Improvement
>  Security Level: Public(Default Security Level. Issues are Public) 
>Reporter: Amrit Sarkar
>Priority: Major
> Attachments: SOLR-13035.patch, SOLR-13035.patch
>
>
> {{solr.data.home}} system property or {{solrDataHome}} in _solr.xml_ is 
> already available as per SOLR-6671.
> The writable content in Solr are index files, core properties, and ZK data if 
> embedded zookeeper is started in SolrCloud mode. It would be great if all 
> writable content can come under the same directory to have separate READ-ONLY 
> and WRITE-ONLY directories.
> It can then also solve official docker Solr image issues:
> https://github.com/docker-solr/docker-solr/issues/74
> https://github.com/docker-solr/docker-solr/issues/133



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (SOLR-13045) Harden TestSimPolicyCloud

[Jason Gerlowski (JIRA)]

[ 
https://issues.apache.org/jira/browse/SOLR-13045?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724116#comment-16724116
 ] 

Jason Gerlowski commented on SOLR-13045:


Checking back in a week later.  The work above has cut down the failure rate 
from 5% to maybe 1-2%, but there's still issues with this test.  Attaching a 
jenkins log containing a current failure from 2 days ago.  (Don't want to lose 
the log when it cycles out of fucit).

At first glance, the failure looks like it happens because a replica is created 
on the wrong node (contrary to a specified policy).  Starting to look into 
things now.

> Harden TestSimPolicyCloud
> -
>
> Key: SOLR-13045
> URL: https://issues.apache.org/jira/browse/SOLR-13045
> Project: Solr
>  Issue Type: Test
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: AutoScaling
>Affects Versions: master (8.0)
>Reporter: Jason Gerlowski
>Assignee: Jason Gerlowski
>Priority: Major
> Attachments: SOLR-13045.patch, SOLR-13045.patch
>
>
> Several tests in TestSimPolicyCloud, but especially 
> {{testCreateCollectionAddReplica}}, have some flaky behavior, even after 
> Mark's recent test-fix commit.  This JIRA covers looking into and (hopefully) 
> fixing this test failure.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (SOLR-13045) Harden TestSimPolicyCloud

[Jason Gerlowski (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SOLR-13045?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Jason Gerlowski updated SOLR-13045:
---
Attachment: jenkins.log.txt.gz

> Harden TestSimPolicyCloud
> -
>
> Key: SOLR-13045
> URL: https://issues.apache.org/jira/browse/SOLR-13045
> Project: Solr
>  Issue Type: Test
>  Security Level: Public(Default Security Level. Issues are Public) 
>  Components: AutoScaling
>Affects Versions: master (8.0)
>Reporter: Jason Gerlowski
>Assignee: Jason Gerlowski
>Priority: Major
> Attachments: SOLR-13045.patch, SOLR-13045.patch, jenkins.log.txt.gz
>
>
> Several tests in TestSimPolicyCloud, but especially 
> {{testCreateCollectionAddReplica}}, have some flaky behavior, even after 
> Mark's recent test-fix commit.  This JIRA covers looking into and (hopefully) 
> fixing this test failure.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-repro - Build # 2514 - Still Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-repro/2514/

[...truncated 28 lines...]
[repro] Jenkins log URL: 
https://builds.apache.org/job/Lucene-Solr-BadApples-Tests-7.x/240/consoleText

[repro] Revision: 811451152114938fead571e1dab9de88968d05a8

[repro] Repro line:  ant test  -Dtestcase=TestSimTriggerIntegration 
-Dtests.method=testListeners -Dtests.seed=CBAFC01709ECE15D -Dtests.multiplier=2 
-Dtests.slow=true -Dtests.badapples=true -Dtests.locale=ar-JO 
-Dtests.timezone=Europe/Uzhgorod -Dtests.asserts=true 
-Dtests.file.encoding=UTF-8

[repro] git rev-parse --abbrev-ref HEAD
[repro] git rev-parse HEAD
[repro] Initial local git branch/revision: 
3be5b59907acc3049121e980bee71df01185d40e
[repro] git fetch
[repro] git checkout 811451152114938fead571e1dab9de88968d05a8

[...truncated 2 lines...]
[repro] git merge --ff-only

[...truncated 1 lines...]
[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]solr/core
[repro]   TestSimTriggerIntegration
[repro] ant compile-test

[...truncated 3605 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 
-Dtests.class="*.TestSimTriggerIntegration" -Dtests.showOutput=onerror  
-Dtests.seed=CBAFC01709ECE15D -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.badapples=true -Dtests.locale=ar-JO -Dtests.timezone=Europe/Uzhgorod 
-Dtests.asserts=true -Dtests.file.encoding=UTF-8

[...truncated 2604 lines...]
[repro] Setting last failure code to 256

[repro] Failures:
[repro]   2/5 failed: 
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[repro] git checkout 3be5b59907acc3049121e980bee71df01185d40e

[...truncated 2 lines...]
[repro] Exiting with code 256

[...truncated 5 lines...]

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-repro - Build # 2509 - Still Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-repro/2509/

[...truncated 28 lines...]
[repro] Jenkins log URL: 
https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.6/27/consoleText

[repro] Revision: e1d5761f7b976aa4ab83969f9a699597c0855b3e

[repro] Ant options: -Dtests.multiplier=2 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.6/test-data/enwiki.random.lines.txt
[repro] Repro line:  ant test  -Dtestcase=FullSolrCloudDistribCmdsTest 
-Dtests.method=test -Dtests.seed=176FDA4AD2D42496 -Dtests.multiplier=2 
-Dtests.nightly=true -Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.6/test-data/enwiki.random.lines.txt
 -Dtests.locale=el -Dtests.timezone=Asia/Kabul -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1

[repro] git rev-parse --abbrev-ref HEAD
[repro] git rev-parse HEAD
[repro] Initial local git branch/revision: 
5e305e2f00fa79f8fe9abb6f949a196c911d1377
[repro] git fetch
[repro] git checkout e1d5761f7b976aa4ab83969f9a699597c0855b3e

[...truncated 2 lines...]
[repro] git merge --ff-only

[...truncated 1 lines...]
[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]solr/core
[repro]   FullSolrCloudDistribCmdsTest
[repro] ant compile-test

[...truncated 3580 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 
-Dtests.class="*.FullSolrCloudDistribCmdsTest" -Dtests.showOutput=onerror 
-Dtests.multiplier=2 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.6/test-data/enwiki.random.lines.txt
 -Dtests.seed=176FDA4AD2D42496 -Dtests.multiplier=2 -Dtests.nightly=true 
-Dtests.slow=true 
-Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.6/test-data/enwiki.random.lines.txt
 -Dtests.locale=el -Dtests.timezone=Asia/Kabul -Dtests.asserts=true 
-Dtests.file.encoding=ISO-8859-1

[...truncated 73124 lines...]
[repro] Setting last failure code to 256

[repro] Failures:
[repro]   4/5 failed: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
[repro] git checkout 5e305e2f00fa79f8fe9abb6f949a196c911d1377

[...truncated 2 lines...]
[repro] Exiting with code 256

[...truncated 5 lines...]

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8613) CoveringQuery to use multi-valued field length as minimum match

[Alan Woodward (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8613?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724061#comment-16724061
 ] 

Alan Woodward commented on LUCENE-8613:
---

If you're happy treating multi-valued fields as a set, then you can write a 
LongValuesSource implementation that counts the number of values in the set 
when advanceExact() is called - having a look at how 
LongValuesSource.FieldValuesSource works should give you a good start.

> CoveringQuery to use multi-valued field length as minimum match
> ---
>
> Key: LUCENE-8613
> URL: https://issues.apache.org/jira/browse/LUCENE-8613
> Project: Lucene - Core
>  Issue Type: Improvement
>Affects Versions: 7.5
>Reporter: Tristan Stevens
>Priority: Minor
>
> Currently CoveringQuery uses a long field's value for the minimumMatch for 
> each document.
> There are use cases where this value may be derived from the length of a 
> multi-valued field (including docValues).
> This request is to support an alternative constructor and implementation for 
> CoveringQuery to take a field name that can be used for the per-document 
> minimum match value.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8613) CoveringQuery to use multi-valued field length as minimum match

[Tristan Stevens (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8613?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724039#comment-16724039
 ] 

Tristan Stevens commented on LUCENE-8613:
-

So I'm trying to avoid having to have a separate field as it's cumbersome and 
error-prone to maintain. I think de-duplicated is fine, right? - if it's a set 
it's a set. if it's a list it's a list. One would expect a set to be 
de-duplicated.

I sort of made a start here: 
https://github.com/apache/lucene-solr/compare/master...tmgstevens:LUCENE-8613?expand=1
 but at the moment the tests don't work and I'm running out of Lucene knowledge 
with which to debug, hence I'd welcome any support.


> CoveringQuery to use multi-valued field length as minimum match
> ---
>
> Key: LUCENE-8613
> URL: https://issues.apache.org/jira/browse/LUCENE-8613
> Project: Lucene - Core
>  Issue Type: Improvement
>Affects Versions: 7.5
>Reporter: Tristan Stevens
>Priority: Minor
>
> Currently CoveringQuery uses a long field's value for the minimumMatch for 
> each document.
> There are use cases where this value may be derived from the length of a 
> multi-valued field (including docValues).
> This request is to support an alternative constructor and implementation for 
> CoveringQuery to take a field name that can be used for the per-document 
> minimum match value.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Commented] (LUCENE-8613) CoveringQuery to use multi-valued field length as minimum match

[Alan Woodward (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8613?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724026#comment-16724026
 ] 

Alan Woodward commented on LUCENE-8613:
---

The major hurdle is that at query time, we don't know how many entries there 
are in a multi-valued field (docvalues de-duplicate, so you can't just count 
the number of entries in a sorted set).  So the general way to do this is to 
add another field at index time containing the number of values, and then pass 
LongValuesSource.fromLongField(fieldname) to the CoveringQuery to retrieve this 
value.

> CoveringQuery to use multi-valued field length as minimum match
> ---
>
> Key: LUCENE-8613
> URL: https://issues.apache.org/jira/browse/LUCENE-8613
> Project: Lucene - Core
>  Issue Type: Improvement
>Affects Versions: 7.5
>Reporter: Tristan Stevens
>Priority: Minor
>
> Currently CoveringQuery uses a long field's value for the minimumMatch for 
> each document.
> There are use cases where this value may be derived from the length of a 
> multi-valued field (including docValues).
> This request is to support an alternative constructor and implementation for 
> CoveringQuery to take a field name that can be used for the per-document 
> minimum match value.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (SOLR-13081) In-Place Update doesn't work when route.field is defined

[Dr Oleg Savrasov (JIRA)]

Dr Oleg Savrasov created SOLR-13081:
---

 Summary: In-Place Update doesn't work when route.field is defined
 Key: SOLR-13081
 URL: https://issues.apache.org/jira/browse/SOLR-13081
 Project: Solr
  Issue Type: Bug
  Security Level: Public (Default Security Level. Issues are Public)
  Components: update
Reporter: Dr Oleg Savrasov


As soon as cloud collection is configured with route.field property, In-Place 
Updates are not applied anymore. This happens because 
AtomicUpdateDocumentMerger skips only id and version fields and doesn't verify 
configured route.field.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

SOLR-12259: online index schema modification - adding docValues to existing indexed data?

[Andrzej Białecki]

Hi,

I'm working on a use case where an existing Solr setup needs to migrate to a 
schema that uses docValues for faceting, instead of uninversion. This case fits 
into a broader subject of SOLR-12259 (Robustly upgrade indexes). However, in 
this case there are two major requirements for this migration process:

* data cannot be reindexed from scratch - I need to work with the already 
indexed documents (which do contain the values needed for faceting, but these 
values are simply indexed and not stored as doc values)

* indexing can’t be stopped while the schema is being changed (the conversion 
process needs to work on-the-fly while the collection is online, both for 
searching and for updates). Collection reloads / reopening is ok but it’s not 
ok to take the collection offline for several minutes (or hours).

Together with Erick Erickson we implemented a solution that uses MergePolicy 
(actually MergePolicyFactory in Solr) to enforce re-writing of segments that no 
longer match the schema, ie. don’t contain docValues in a field where the new 
schema requires it. This merge policy determines what segments need this 
conversion and then forces the “merging” (actually re-writing) of these 
segments by first wrapping them into UninvertingReader to supply docValues 
where they are required by the new schema but actually are missing in the 
segment’s data. This “AddDocValuesMergePolicy” (ADVMP for short) is supposed to 
deal with the following types of segments:

* old segments created before the schema change - these don’t contain any 
docValues in the target fields and so they are wrapped in UninvertingReader for 
merging (and for searching) according to the new schema.

* new segments created after the schema change - if FieldInfo-s for these 
fields claim that the segment already contains docValues where it should then 
the segment is passed as-is to merging, otherwise it’s wrapped again. An 
optimisation was also put here to “mark” the already converted segments using a 
marker in SegmentInfo diagnostics map so that we can avoid re-checking and 
re-converting already converted data.

So, long story short, this process works very well when there’s no concurrent 
indexing activity - all old segments are properly wrapped and re-written and 
merging with new segments works as intended. However, in a situation with 
concurrent indexing it works well but only for a short while. At some point 
this conversion process seems to lose large percentage of the docValues, even 
though it seems that at all points the source segments are properly wrapped - 
the ADVMP merge policy adds a lot of debugging information to track the source 
and type of segments across many levels of merging and whether they were 
wrapped or not.

My working theory is that somehow this schema change produces 
“franken-segments” (while they still haven’t been flushed) where only some of 
the most recent docs have the docValues and earlier ones don’t. As I understand 
it, this should not happen in Solr because a schema change results in a core 
reload. The tracking information from ADVMP  seems to indicate that all 
generations of segments, both those that were flushed and merged earlier, have 
been properly wrapped.

My alternate theory is that there’s some bug in the doc values merging process 
when UninvertingReader is involved, because this problem occurs also when we 
modify ADVMP to always force the wrapping of all segments in 
UninvertingReader-s. The percentage of lost doc values is sometimes quite 
large, up to 50%, perhaps it’s a bug somewhere where the code accounts for the 
presence of doc values in FieldCacheImpl?

Together with Erick we implemented a bunch of tests that illustrate this issue 
- both the tests and the code can be found on branch "jira/solr-12259":

* code.tests.AddDVMPLuceneTest2 - this is a pure Lucene test that shows how doc 
values are lost after several rounds of merging while concurrent indexing is 
going on. This failure is reproducible 100%.

* code.tests.AddDvStress - this is a Solr test that repeatedly creates a 
collection without doc values, starts the indexing, changes the config to use 
ADVMP, makes the schema change to turn doc values on, and verifies the number 
of facets on the target field. This test also fails after a while with the same 
symptoms as the Lucene one, so I think that solving the Lucene test failure 
should solve this failure too.

Any suggestions or insights are very much appreciated - I'm running out of 
ideas to try...

—

Andrzej Białecki

[jira] [Commented] (LUCENE-8613) CoveringQuery to use multi-valued field length as minimum match

[Tristan Stevens (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8613?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16724009#comment-16724009
 ] 

Tristan Stevens commented on LUCENE-8613:
-

[~romseygeek] I've spent a fair amount of time pondering how one might do this. 
If you're able to provide any pointers I'd be immensely grateful.

> CoveringQuery to use multi-valued field length as minimum match
> ---
>
> Key: LUCENE-8613
> URL: https://issues.apache.org/jira/browse/LUCENE-8613
> Project: Lucene - Core
>  Issue Type: Improvement
>Affects Versions: 7.5
>Reporter: Tristan Stevens
>Priority: Minor
>
> Currently CoveringQuery uses a long field's value for the minimumMatch for 
> each document.
> There are use cases where this value may be derived from the length of a 
> multi-valued field (including docValues).
> This request is to support an alternative constructor and implementation for 
> CoveringQuery to take a field name that can be used for the per-document 
> minimum match value.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS-EA] Lucene-Solr-7.6-Linux (64bit/jdk-12-ea+23) - Build # 114 - Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.6-Linux/114/
Java: 64bit/jdk-12-ea+23 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

4 tests failed.
FAILED:  
org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
Expected new active leader null Live Nodes: [127.0.0.1:34179_solr, 
127.0.0.1:44507_solr, 127.0.0.1:45405_solr] Last available state: 
DocCollection(raceDeleteReplica_true//collections/raceDeleteReplica_true/state.json/14)={
   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{"shard1":{   
"range":"8000-7fff",   "state":"active",   "replicas":{ 
"core_node3":{   "core":"raceDeleteReplica_true_shard1_replica_n1", 
  "base_url":"https://127.0.0.1:46485/solr;,   
"node_name":"127.0.0.1:46485_solr",   "state":"down",   
"type":"NRT",   "leader":"true"}, "core_node6":{   
"core":"raceDeleteReplica_true_shard1_replica_n5",   
"base_url":"https://127.0.0.1:46485/solr;,   
"node_name":"127.0.0.1:46485_solr",   "state":"down",   
"type":"NRT"}, "core_node4":{   
"core":"raceDeleteReplica_true_shard1_replica_n2",   
"base_url":"https://127.0.0.1:45405/solr;,   
"node_name":"127.0.0.1:45405_solr",   "state":"down",   
"type":"NRT",   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",  
 "autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected new active leader
null
Live Nodes: [127.0.0.1:34179_solr, 127.0.0.1:44507_solr, 127.0.0.1:45405_solr]
Last available state: 
DocCollection(raceDeleteReplica_true//collections/raceDeleteReplica_true/state.json/14)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
  "range":"8000-7fff",
  "state":"active",
  "replicas":{
"core_node3":{
  "core":"raceDeleteReplica_true_shard1_replica_n1",
  "base_url":"https://127.0.0.1:46485/solr;,
  "node_name":"127.0.0.1:46485_solr",
  "state":"down",
  "type":"NRT",
  "leader":"true"},
"core_node6":{
  "core":"raceDeleteReplica_true_shard1_replica_n5",
  "base_url":"https://127.0.0.1:46485/solr;,
  "node_name":"127.0.0.1:46485_solr",
  "state":"down",
  "type":"NRT"},
"core_node4":{
  "core":"raceDeleteReplica_true_shard1_replica_n2",
  "base_url":"https://127.0.0.1:45405/solr;,
  "node_name":"127.0.0.1:45405_solr",
  "state":"down",
  "type":"NRT",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
at 
__randomizedtesting.SeedInfo.seed([2EA8A399F4E647D6:44BEC2499C140D1C]:0)
at org.junit.Assert.fail(Assert.java:93)
at 
org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:280)
at 
org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:334)
at 
org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:229)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 

[jira] [Commented] (LUCENE-8613) CoveringQuery to use multi-valued field length as minimum match

[Alan Woodward (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8613?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16723993#comment-16723993
 ] 

Alan Woodward commented on LUCENE-8613:
---

CoveringQuery takes a LongValuesSource, rather than a field value; you should 
be able to build a source object that does what you require already?

> CoveringQuery to use multi-valued field length as minimum match
> ---
>
> Key: LUCENE-8613
> URL: https://issues.apache.org/jira/browse/LUCENE-8613
> Project: Lucene - Core
>  Issue Type: Improvement
>Affects Versions: 7.5
>Reporter: Tristan Stevens
>Priority: Minor
>
> Currently CoveringQuery uses a long field's value for the minimumMatch for 
> each document.
> There are use cases where this value may be derived from the length of a 
> multi-valued field (including docValues).
> This request is to support an alternative constructor and implementation for 
> CoveringQuery to take a field name that can be used for the per-document 
> minimum match value.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (LUCENE-8613) CoveringQuery to use multi-valued field length as minimum match

[Tristan Stevens (JIRA)]

Tristan Stevens created LUCENE-8613:
---

 Summary: CoveringQuery to use multi-valued field length as minimum 
match
 Key: LUCENE-8613
 URL: https://issues.apache.org/jira/browse/LUCENE-8613
 Project: Lucene - Core
  Issue Type: Improvement
Affects Versions: 7.5
Reporter: Tristan Stevens


Currently CoveringQuery uses a long field's value for the minimumMatch for each 
document.
There are use cases where this value may be derived from the length of a 
multi-valued field (including docValues).
This request is to support an alternative constructor and implementation for 
CoveringQuery to take a field name that can be used for the per-document 
minimum match value.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Created] (SOLR-13080) TermsQParserPlugin automaton method fails to sort input

[Daniel Lowe (JIRA)]

Daniel Lowe created SOLR-13080:
--

 Summary: TermsQParserPlugin automaton method fails to sort input
 Key: SOLR-13080
 URL: https://issues.apache.org/jira/browse/SOLR-13080
 Project: Solr
  Issue Type: Bug
  Security Level: Public (Default Security Level. Issues are Public)
  Components: query parsers
Affects Versions: 7.5
Reporter: Daniel Lowe


The contract for Automata.makeStringUnion is that the input is sorted. As 
BytesRef implements Comparable. The simplest fix would probably to make

Arrays.sort(bytesRefs);

The first line of automaton's makeFilter method in TermsQParserPlugin.

 



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-repro - Build # 2513 - Still Unstable

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-repro/2513/

[...truncated 39 lines...]
[repro] Jenkins log URL: 
https://builds.apache.org/job/Lucene-Solr-Tests-http2/104/consoleText

[repro] Revision: c0af83e45f7fd1a81de9aca059f924b0503f44c5

[repro] Repro line:  ant test  -Dtestcase=SaslZkACLProviderTest 
-Dtests.method=testSaslZkACLProvider -Dtests.seed=95E99E3A0E083C2A 
-Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar-MA 
-Dtests.timezone=Asia/Pyongyang -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[repro] Repro line:  ant test  -Dtestcase=SaslZkACLProviderTest 
-Dtests.seed=95E99E3A0E083C2A -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=ar-MA -Dtests.timezone=Asia/Pyongyang -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[repro] Repro line:  ant test  -Dtestcase=TestSimTriggerIntegration 
-Dtests.method=testListeners -Dtests.seed=95E99E3A0E083C2A -Dtests.multiplier=2 
-Dtests.slow=true -Dtests.locale=es -Dtests.timezone=America/Chicago 
-Dtests.asserts=true -Dtests.file.encoding=US-ASCII

[repro] Repro line:  ant test  -Dtestcase=TestSolrConfigHandlerCloud 
-Dtests.method=test -Dtests.seed=95E99E3A0E083C2A -Dtests.multiplier=2 
-Dtests.slow=true -Dtests.locale=vi-VN -Dtests.timezone=Africa/Libreville 
-Dtests.asserts=true -Dtests.file.encoding=US-ASCII

[repro] Repro line:  ant test  -Dtestcase=TestCollectionAPI -Dtests.method=test 
-Dtests.seed=95E99E3A0E083C2A -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=no -Dtests.timezone=Etc/GMT0 -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[repro] Repro line:  ant test  -Dtestcase=OverseerTest 
-Dtests.method=testOverseerFailure -Dtests.seed=95E99E3A0E083C2A 
-Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ru 
-Dtests.timezone=Canada/Yukon -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[repro] Repro line:  ant test  -Dtestcase=OverseerTest 
-Dtests.seed=95E99E3A0E083C2A -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=ru -Dtests.timezone=Canada/Yukon -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[repro] git rev-parse --abbrev-ref HEAD
[repro] git rev-parse HEAD
[repro] Initial local git branch/revision: 
3be5b59907acc3049121e980bee71df01185d40e
[repro] git fetch
[repro] git checkout c0af83e45f7fd1a81de9aca059f924b0503f44c5

[...truncated 2 lines...]
[repro] git merge --ff-only

[...truncated 1 lines...]
[repro] ant clean

[...truncated 6 lines...]
[repro] Test suites by module:
[repro]solr/core
[repro]   TestSolrConfigHandlerCloud
[repro]   TestCollectionAPI
[repro]   OverseerTest
[repro]   TestSimTriggerIntegration
[repro]   SaslZkACLProviderTest
[repro] ant compile-test

[...truncated 3593 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=25 
-Dtests.class="*.TestSolrConfigHandlerCloud|*.TestCollectionAPI|*.OverseerTest|*.TestSimTriggerIntegration|*.SaslZkACLProviderTest"
 -Dtests.showOutput=onerror  -Dtests.seed=95E99E3A0E083C2A -Dtests.multiplier=2 
-Dtests.slow=true -Dtests.locale=vi-VN -Dtests.timezone=Africa/Libreville 
-Dtests.asserts=true -Dtests.file.encoding=US-ASCII

[...truncated 5249 lines...]
[repro] Setting last failure code to 256

[repro] Failures:
[repro]   0/5 failed: org.apache.solr.cloud.OverseerTest
[repro]   0/5 failed: org.apache.solr.cloud.api.collections.TestCollectionAPI
[repro]   0/5 failed: org.apache.solr.handler.TestSolrConfigHandlerCloud
[repro]   3/5 failed: 
org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[repro]   5/5 failed: org.apache.solr.cloud.SaslZkACLProviderTest

[repro] Re-testing 100% failures at the tip of jira/http2
[repro] git fetch
[repro] git checkout jira/http2

[...truncated 4 lines...]
[repro] git merge --ff-only

[...truncated 179 lines...]
[repro] ant clean

[...truncated 8 lines...]
[repro] Test suites by module:
[repro]solr/core
[repro]   SaslZkACLProviderTest
[repro] ant compile-test

[...truncated 3593 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 
-Dtests.class="*.SaslZkACLProviderTest" -Dtests.showOutput=onerror  
-Dtests.seed=95E99E3A0E083C2A -Dtests.multiplier=2 -Dtests.slow=true 
-Dtests.locale=ar-MA -Dtests.timezone=Asia/Pyongyang -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[...truncated 833 lines...]
[repro] Setting last failure code to 256

[repro] Failures at the tip of jira/http2:
[repro]   5/5 failed: org.apache.solr.cloud.SaslZkACLProviderTest

[repro] Re-testing 100% failures at the tip of jira/http2 without a seed
[repro] ant clean

[...truncated 8 lines...]
[repro] Test suites by module:
[repro]solr/core
[repro]   SaslZkACLProviderTest
[repro] ant compile-test

[...truncated 3593 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=5 
-Dtests.class="*.SaslZkACLProviderTest" -Dtests.showOutput=onerror  
-Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar-MA 
-Dtests.timezone=Asia/Pyongyang -Dtests.asserts=true 
-Dtests.file.encoding=US-ASCII

[...truncated 874 lines...]

[jira] [Commented] (LUCENE-8612) Add the ability to enforce gaps between intervals

[Alan Woodward (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8612?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16723955#comment-16723955
 ] 

Alan Woodward commented on LUCENE-8612:
---

Updated patch, removing the specialized NotWithinFunction (no longer 
necessary), and adding the positional logic I described above, as I think it 
makes more sense to treat an extended interval as a 'block', and therefore 
requiring that prefix positions exist in the document.

> Add the ability to enforce gaps between intervals
> -
>
> Key: LUCENE-8612
> URL: https://issues.apache.org/jira/browse/LUCENE-8612
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Alan Woodward
>Assignee: Alan Woodward
>Priority: Major
> Attachments: LUCENE-8612.patch, LUCENE-8612.patch
>
>
> At the moment you can search for intervals with a maximum number of positions 
> between them, but you cannot enforce gaps.  It would be useful to be able to 
> search for `a b [2 spaces] c`.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[jira] [Updated] (LUCENE-8612) Add the ability to enforce gaps between intervals

[Alan Woodward (JIRA)]

 [ 
https://issues.apache.org/jira/browse/LUCENE-8612?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Alan Woodward updated LUCENE-8612:
--
Attachment: LUCENE-8612.patch

> Add the ability to enforce gaps between intervals
> -
>
> Key: LUCENE-8612
> URL: https://issues.apache.org/jira/browse/LUCENE-8612
> Project: Lucene - Core
>  Issue Type: Task
>Reporter: Alan Woodward
>Assignee: Alan Woodward
>Priority: Major
> Attachments: LUCENE-8612.patch, LUCENE-8612.patch
>
>
> At the moment you can search for intervals with a maximum number of positions 
> between them, but you cannot enforce gaps.  It would be useful to be able to 
> search for `a b [2 spaces] c`.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-SmokeRelease-7.x - Build # 403 - Failure

[Apache Jenkins Server]

Build: https://builds.apache.org/job/Lucene-Solr-SmokeRelease-7.x/403/

No tests ran.

Build Log:
[...truncated 23485 lines...]
[asciidoctor:convert] asciidoctor: ERROR: about-this-guide.adoc: line 1: 
invalid part, must have at least one section (e.g., chapter, appendix, etc.)
[asciidoctor:convert] asciidoctor: ERROR: solr-glossary.adoc: line 1: invalid 
part, must have at least one section (e.g., chapter, appendix, etc.)
 [java] Processed 2458 links (2009 relative) to 3221 anchors in 247 files
 [echo] Validated Links & Anchors via: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/solr/build/solr-ref-guide/bare-bones-html/

-dist-changes:
 [copy] Copying 4 files to 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/solr/package/changes

package:

-unpack-solr-tgz:

-ensure-solr-tgz-exists:
[mkdir] Created dir: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/solr/build/solr.tgz.unpacked
[untar] Expanding: 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/solr/package/solr-7.7.0.tgz
 into 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/solr/build/solr.tgz.unpacked

generate-maven-artifacts:

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = 
/home/jenkins/jenkins-slave/workspace/Lucene-Solr-SmokeRelease-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:

[jira] [Commented] (LUCENE-8585) Create jump-tables for DocValues at index-time

[Toke Eskildsen (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8585?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16723918#comment-16723918
 ] 

Toke Eskildsen commented on LUCENE-8585:


[~jpountz] how do you see the chances of getting this in 8.0? Freezing seems 
awfully close and a I would expect some baking time for a change such as 
LUCENE-8585. On the other hand, it seems a bit backwards to roll a 8.0 with a 
planned codec-update pending for 8.1. If you are in doubt (as I am), should we 
discuss this in the relevant thread on the mailing list?

> Create jump-tables for DocValues at index-time
> --
>
> Key: LUCENE-8585
> URL: https://issues.apache.org/jira/browse/LUCENE-8585
> Project: Lucene - Core
>  Issue Type: Improvement
>  Components: core/codecs
>Affects Versions: master (8.0)
>Reporter: Toke Eskildsen
>Priority: Minor
>  Labels: performance
> Attachments: LUCENE-8585.patch, LUCENE-8585.patch, 
> make_patch_lucene8585.sh
>
>  Time Spent: 4h 50m
>  Remaining Estimate: 0h
>
> As noted in LUCENE-7589, lookup of DocValues should use jump-tables to avoid 
> long iterative walks. This is implemented in LUCENE-8374 at search-time 
> (first request for DocValues from a field in a segment), with the benefit of 
> working without changes to existing Lucene 7 indexes and the downside of 
> introducing a startup time penalty and a memory overhead.
> As discussed in LUCENE-8374, the codec should be updated to create these 
> jump-tables at index time. This eliminates the segment-open time & memory 
> penalties, with the potential downside of increasing index-time for DocValues.
> The three elements of LUCENE-8374 should be transferable to index-time 
> without much alteration of the core structures:
>  * {{IndexedDISI}} block offset and index skips: A {{long}} (64 bits) for 
> every 65536 documents, containing the offset of the block in 33 bits and the 
> index (number of set bits) up to the block in 31 bits.
>  It can be build sequentially and should be stored as a simple sequence of 
> consecutive longs for caching of lookups.
>  As it is fairly small, relative to document count, it might be better to 
> simply memory cache it?
>  * {{IndexedDISI}} DENSE (> 4095, < 65536 set bits) blocks: A {{short}} (16 
> bits) for every 8 {{longs}} (512 bits) for a total of 256 bytes/DENSE_block. 
> Each {{short}} represents the number of set bits up to right before the 
> corresponding sub-block of 512 docIDs.
>  The \{{shorts}} can be computed sequentially or when the DENSE block is 
> flushed (probably the easiest). They should be stored as a simple sequence of 
> consecutive shorts for caching of lookups, one logically independent sequence 
> for each DENSE block. The logical position would be one sequence at the start 
> of every DENSE block.
>  Whether it is best to read all the 16 {{shorts}} up front when a DENSE block 
> is accessed or whether it is best to only read any individual {{short}} when 
> needed is not clear at this point.
>  * Variable Bits Per Value: A {{long}} (64 bits) for every 16384 numeric 
> values. Each {{long}} holds the offset to the corresponding block of values.
>  The offsets can be computed sequentially and should be stored as a simple 
> sequence of consecutive {{longs}} for caching of lookups.
>  The vBPV-offsets has the largest space overhead og the 3 jump-tables and a 
> lot of the 64 bits in each long are not used for most indexes. They could be 
> represented as a simple {{PackedInts}} sequence or {{MonotonicLongValues}}, 
> with the downsides of a potential lookup-time overhead and the need for doing 
> the compression after all offsets has been determined.
> I have no experience with the codec-parts responsible for creating 
> index-structures. I'm quite willing to take a stab at this, although I 
> probably won't do much about it before January 2019. Should anyone else wish 
> to adopt this JIRA-issue or co-work on it, I'll be happy to share.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 2213 - Still Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/2213/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

5 tests failed.
FAILED:  
org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud.testCreateCollectionAddShardWithReplicaTypeUsingPolicy

Error Message:
NRT replica should be in 127.0.0.1:10018_solr

Stack Trace:
java.lang.AssertionError: NRT replica should be in 127.0.0.1:10018_solr
at 
__randomizedtesting.SeedInfo.seed([FB9AAA3F29B5C5A7:6BC8BFAC2479045B]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at 
org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud.lambda$testCreateCollectionAddShardWithReplicaTypeUsingPolicy$6(TestSimPolicyCloud.java:279)
at 
org.apache.solr.common.cloud.DocCollection.lambda$null$0(DocCollection.java:188)
at java.util.HashMap.forEach(HashMap.java:1289)
at 
org.apache.solr.common.cloud.DocCollection.lambda$forEachReplica$1(DocCollection.java:188)
at java.util.HashMap.forEach(HashMap.java:1289)
at 
org.apache.solr.common.cloud.DocCollection.forEachReplica(DocCollection.java:188)
at 
org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud.testCreateCollectionAddShardWithReplicaTypeUsingPolicy(TestSimPolicyCloud.java:293)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at 
org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:239)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at 

[jira] [Commented] (LUCENE-8604) Add TestRuleLimitSysouts tests, do some cleanup and add hard limit

[Dawid Weiss (JIRA)]

[ 
https://issues.apache.org/jira/browse/LUCENE-8604?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel=16723828#comment-16723828
 ] 

Dawid Weiss commented on LUCENE-8604:
-

I changed the code slightly. The hard limit for all sysouts is 2 GB (nothing 
beyond that emitted from a single suite will get printed). Precommit and tests 
pass. I'd like to commit shortly, this is fairly useful to prevent accidental 
disk spills in the future.

> Add TestRuleLimitSysouts tests, do some cleanup and add hard limit
> --
>
> Key: LUCENE-8604
> URL: https://issues.apache.org/jira/browse/LUCENE-8604
> Project: Lucene - Core
>  Issue Type: Improvement
>Reporter: Dawid Weiss
>Assignee: Dawid Weiss
>Priority: Trivial
>  Time Spent: 10m
>  Remaining Estimate: 0h
>
> PR from github here. 
> https://github.com/apache/lucene-solr/pull/524



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

-
To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org
For additional commands, e-mail: dev-h...@lucene.apache.org

[JENKINS] Lucene-Solr-BadApples-7.x-Linux (64bit/jdk-9.0.4) - Build # 135 - Unstable!

[Policeman Jenkins Server]

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-BadApples-7.x-Linux/135/
Java: 64bit/jdk-9.0.4 -XX:+UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  
org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitShardWithRule

Error Message:
Error from server at http://127.0.0.1:37985: Could not find collection : 
shardSplitWithRule_rewrite

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error 
from server at http://127.0.0.1:37985: Could not find collection : 
shardSplitWithRule_rewrite
at 
__randomizedtesting.SeedInfo.seed([D9EF0F3A06B63591:58E1DE889B446EBA]:0)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at 
org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:484)
at 
org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:414)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1110)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at 
org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at 
org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
at 
org.apache.solr.cloud.api.collections.ShardSplitTest.doSplitShardWithRule(ShardSplitTest.java:661)
at 
org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitShardWithRule(ShardSplitTest.java:628)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at 
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at 
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1063)
at 
org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1035)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at 
org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at 
org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at 
com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at 
com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at 
org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at 
com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at 
org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at