[ https://issues.apache.org/jira/browse/SOLR-6052?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14273160#comment-14273160 ]
Erik Hatcher commented on SOLR-6052: ------------------------------------ How is this a Solr bug? Looks like the script creates "too many documents", no? If this is an issue that needs to be addressed, please re-open with more elaboration on what is being attempted and what is expected here. > Too many documents Exception > ---------------------------- > > Key: SOLR-6052 > URL: https://issues.apache.org/jira/browse/SOLR-6052 > Project: Solr > Issue Type: Bug > Affects Versions: 4.3, 4.4, 4.5, 4.6, 4.7 > Reporter: yamazaki > > ERROR org.apache.solr.core.CoreContainer – Unable to create core: collection1 > org.apache.solr.common.SolrException: Error opening new searcher > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:821) > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:618) > at > org.apache.solr.core.CoreContainer.createFromLocal(CoreContainer.java:949) > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:984) > at org.apache.solr.core.CoreContainer$2.call(CoreContainer.java:597) > at org.apache.solr.core.CoreContainer$2.call(CoreContainer.java:592) > at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) > at java.util.concurrent.FutureTask.run(FutureTask.java:138) > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) > at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) > at java.util.concurrent.FutureTask.run(FutureTask.java:138) > at > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918) > at java.lang.Thread.run(Thread.java:662) > Caused by: org.apache.solr.common.SolrException: Error opening new searcher > at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1438) > at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1550) > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:796) > ... 13 more > Caused by: org.apache.solr.common.SolrException: Error opening Reader > at > org.apache.solr.search.SolrIndexSearcher.getReader(SolrIndexSearcher.java:172) > at > org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:183) > at > org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:179) > at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1414) > ... 15 more > Caused by: java.lang.IllegalArgumentException: Too many documents, composite > IndexReaders cannot exceed 2147483647 > at > org.apache.lucene.index.BaseCompositeReader.<init>(BaseCompositeReader.java:77) > at > org.apache.lucene.index.DirectoryReader.<init>(DirectoryReader.java:368) > at > org.apache.lucene.index.StandardDirectoryReader.<init>(StandardDirectoryReader.java:42) > at > org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:71) > at > org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:783) > at > org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:52) > at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:88) > at > org.apache.solr.core.StandardIndexReaderFactory.newReader(StandardIndexReaderFactory.java:34) > at > org.apache.solr.search.SolrIndexSearcher.getReader(SolrIndexSearcher.java:169) > ... 18 more > ERROR org.apache.solr.core.CoreContainer – > null:org.apache.solr.common.SolrException: Unable to create core: collection1 > at > org.apache.solr.core.CoreContainer.recordAndThrow(CoreContainer.java:1450) > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:993) > at org.apache.solr.core.CoreContainer$2.call(CoreContainer.java:597) > at org.apache.solr.core.CoreContainer$2.call(CoreContainer.java:592) > at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) > at java.util.concurrent.FutureTask.run(FutureTask.java:138) > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:439) > at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) > at java.util.concurrent.FutureTask.run(FutureTask.java:138) > at > java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918) > at java.lang.Thread.run(Thread.java:662) > Caused by: org.apache.solr.common.SolrException: Error opening new searcher > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:821) > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:618) > at > org.apache.solr.core.CoreContainer.createFromLocal(CoreContainer.java:949) > at org.apache.solr.core.CoreContainer.create(CoreContainer.java:984) > ... 10 more > Caused by: org.apache.solr.common.SolrException: Error opening new searcher > at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1438) > at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1550) > at org.apache.solr.core.SolrCore.<init>(SolrCore.java:796) > ... 13 more > Caused by: org.apache.solr.common.SolrException: Error opening Reader > at > org.apache.solr.search.SolrIndexSearcher.getReader(SolrIndexSearcher.java:172) > at > org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:183) > at > org.apache.solr.search.SolrIndexSearcher.<init>(SolrIndexSearcher.java:179) > at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1414) > ... 15 more > Caused by: java.lang.IllegalArgumentException: Too many documents, composite > IndexReaders cannot exceed 2147483647 > at > org.apache.lucene.index.BaseCompositeReader.<init>(BaseCompositeReader.java:77) > at > org.apache.lucene.index.DirectoryReader.<init>(DirectoryReader.java:368) > at > org.apache.lucene.index.StandardDirectoryReader.<init>(StandardDirectoryReader.java:42) > at > org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:71) > at > org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:783) > at > org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:52) > at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:88) > at > org.apache.solr.core.StandardIndexReaderFactory.newReader(StandardIndexReaderFactory.java:34) > at > org.apache.solr.search.SolrIndexSearcher.getReader(SolrIndexSearcher.java:169) > ... 18 more > sample solrconfig.xml > <?xml version="1.0" encoding="UTF-8" ?> > <config> > <luceneMatchVersion>LUCENE_43</luceneMatchVersion> > <lib dir="/opt/solr/dist" regex="solr-cell-\d.*\.jar" /> > <lib dir="/opt/solr/contrib/extraction/lib" regex=".*\.jar" /> > <lib dir="/opt/solr/dist" regex="solr-clustering-\d.*\.jar" /> > <lib dir="/opt/solr/contrib/clustering/lib" regex=".*\.jar" /> > <lib dir="/opt/solr/dist" regex="solr-langid-\d.*\.jar" /> > <lib dir="/opt/solr/contrib/langid/lib" regex=".*\.jar" /> > <lib dir="/opt/solr/dist" regex="solr-velocity-\d.*\.jar" /> > <lib dir="/opt/solr/contrib/velocity/lib" regex=".*\.jar" /> > <dataDir>${solr.data.dir:}</dataDir> > <directoryFactory name="DirectoryFactory" > > class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/> > <codecFactory class="solr.SchemaCodecFactory"/> > <indexConfig> > <ramBufferSizeMB>256</ramBufferSizeMB> > <lockType>${solr.lock.type:native}</lockType> > </indexConfig> > <jmx /> > <updateHandler class="solr.DirectUpdateHandler2"> > <updateLog> > <str name="dir">${solr.ulog.dir:}</str> > </updateLog> > <autoCommit> > <maxDocs>10000</maxDocs> > <maxTime>60000</maxTime> > <openSearcher>false</openSearcher> > </autoCommit> > <autoSoftCommit> > <maxDocs>10</maxDocs> > <maxTime>1000</maxTime> > </autoSoftCommit> > </updateHandler> > <query> > <maxBooleanClauses>1024</maxBooleanClauses> > <filterCache class="solr.FastLRUCache" > size="16384" > initialSize="4096" > autowarmCount="1024"/> > <queryResultCache class="solr.FastLRUCache" > size="16384" > initialSize="4096" > autowarmCount="1024"/> > <documentCache class="solr.FastLRUCache" > size="16384" > initialSize="4096" > autowarmCount="1024"/> > <enableLazyFieldLoading>true</enableLazyFieldLoading> > <queryResultWindowSize>20</queryResultWindowSize> > <queryResultMaxDocsCached>200</queryResultMaxDocsCached> > <useColdSearcher>false</useColdSearcher> > <maxWarmingSearchers>2</maxWarmingSearchers> > </query> > <requestDispatcher handleSelect="false" > > <requestParsers enableRemoteStreaming="true" > multipartUploadLimitInKB="2048000" > formdataUploadLimitInKB="2048"/> > <httpCaching never304="true" /> > </requestDispatcher> > <requestHandler name="/select" class="solr.SearchHandler"> > <lst name="defaults"> > <str name="echoParams">explicit</str> > <int name="rows">10</int> > <str name="df">text</str> > </lst> > </requestHandler> > <requestHandler name="/update" class="solr.UpdateRequestHandler"> > </requestHandler> > <requestHandler name="/update/json" class="solr.JsonUpdateRequestHandler"> > <lst name="defaults"> > <str name="stream.contentType">application/json</str> > </lst> > </requestHandler> > <requestHandler name="/admin/" class="solr.admin.AdminHandlers" /> > <requestHandler name="/admin/ping" class="solr.PingRequestHandler"> > <lst name="invariants"> > <str name="q">solrpingquery</str> > </lst> > <lst name="defaults"> > <str name="echoParams">all</str> > </lst> > </requestHandler> > <queryResponseWriter name="json" class="solr.JSONResponseWriter"> > <str name="content-type">text/plain; charset=UTF-8</str> > </queryResponseWriter> > </config> > sample scheme.xml > <?xml version="1.0" encoding="UTF-8" ?> > <schema name="twitter" version="1.5"> > <!-- types --> > <types> > <fieldType name="string" class="solr.StrField" sortMissingLast="true" /> > <fieldType name="long" class="solr.TrieLongField" precisionStep="0" > positionIncrementGap="0"/> > <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" > positionIncrementGap="0"/> > <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" > positionIncrementGap="0"/> > <fieldType name="text_cjk" class="solr.TextField" > positionIncrementGap="100"> > <analyzer> > <charFilter class="solr.MappingCharFilterFactory"/> > <tokenizer class="solr.StandardTokenizerFactory"/> > <filter class="solr.CJKWidthFilterFactory"/> > <filter class="solr.LowerCaseFilterFactory"/> > <filter class="solr.CJKBigramFilterFactory" outputUnigrams="true"/> > </analyzer> > </fieldType> > </types> > <!-- fields --> > <fields> > <field name="key" type="string" indexed="true" stored="true" > required="true" /> > <field name="status_id" type="tlong" indexed="true" stored="true" > required="true"/> > <field name="text" type="text_cjk" indexed="true" stored="true" > required="true"/> > <field name="from_user_id_str" type="string" indexed="true" stored="true" > required="true"/> > <field name="created_at" type="tdate" indexed="true" stored="true" > required="true"/> > <field name="_version_" type="long" indexed="true" stored="true" > multiValued="false"/> > </fields> > <uniqueKey>key</uniqueKey> > <defaultSearchField>text</defaultSearchField> > <solrQueryParser defaultOperator="AND"/> > </schema> > sample data add source code, python > #!/usr/bin/env python > # -*- coding: utf-8 -*- > import datetime > # use https://github.com/toastdriven/pysolr > from pysolr import( > Solr, > ) > def main(): > s_time = datetime.datetime.utcnow() > print 'start.: ({})'.format(str(s_time)) > solr = Solr('http://localhost:8983/solr/collection1', timeout=60) > docs = [] > max_range = 22 * (10 ** 8) # Java Integer.MAX_VALUE over > for x in xrange(1, max_range): > docs.append( > { > 'key': '{}'.format(x), > 'status_id': x, > 'text': '{} 番目の記事'.format(x).decode('utf-8'), > 'from_user_id_str': '1', > 'created_at': '2014-05-01T20:06:53Z', > } > ) > if x % (10 ** 4) == 0: > solr.add(docs) > solr.commit() > docs = [] > e_time = datetime.datetime.utcnow() > print '{} end.: ({})'.format(x, str(e_time - s_time)) > solr.add(docs) > solr.commit() > docs = [] > e_time = datetime.datetime.utcnow() > print 'end.: ({})'.format(str(e_time - s_time)) > if __name__ == '__main__': > main() -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@lucene.apache.org For additional commands, e-mail: dev-h...@lucene.apache.org