IGNITE-3562: Updated Lucene dependency to version 5.5.2. This closes #1987.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/478d3b5d Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/478d3b5d Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/478d3b5d Branch: refs/heads/ignite-5578 Commit: 478d3b5d3361c3d74d0da4b6a78e9944d8b95630 Parents: a0b5644 Author: Andrey V. Mashenkov <andrey.mashen...@gmail.com> Authored: Tue Jul 11 22:28:45 2017 +0300 Committer: devozerov <voze...@gridgain.com> Committed: Tue Jul 11 22:28:45 2017 +0300 ---------------------------------------------------------------------- .../cache/GridCacheLuceneQueryIndexTest.java | 466 ------------------- modules/indexing/pom.xml | 12 + .../query/h2/opt/GridLuceneDirectory.java | 47 +- .../query/h2/opt/GridLuceneIndex.java | 75 +-- .../query/h2/opt/GridLuceneInputStream.java | 94 ++-- .../query/h2/opt/GridLuceneLockFactory.java | 45 +- .../query/h2/opt/GridLuceneOutputStream.java | 72 +-- .../cache/GridCacheFullTextQuerySelfTest.java | 367 +++++++++++++++ ...teCacheFullTextQueryNodeJoiningSelfTest.java | 4 +- .../IgniteCacheQuerySelfTestSuite.java | 7 +- parent/pom.xml | 4 +- 11 files changed, 563 insertions(+), 630 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLuceneQueryIndexTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLuceneQueryIndexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLuceneQueryIndexTest.java deleted file mode 100644 index 585ef1b..0000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheLuceneQueryIndexTest.java +++ /dev/null @@ -1,466 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.query.annotations.QueryTextField; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.jetbrains.annotations.Nullable; - -import static org.apache.ignite.cache.CacheMode.LOCAL; -import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; - -/** - * - */ -public class GridCacheLuceneQueryIndexTest extends GridCommonAbstractTest { - /** */ - private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); - - /** - * - */ - public GridCacheLuceneQueryIndexTest() { - super(false); - } - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - TcpDiscoverySpi disco = new TcpDiscoverySpi(); - - disco.setIpFinder(ipFinder); - - cfg.setDiscoverySpi(disco); - - cfg.setIncludeEventTypes(); - cfg.setConnectorConfiguration(null); - - CacheConfiguration cacheCfg1 = defaultCacheConfiguration(); - - cacheCfg1.setName("local1"); - cacheCfg1.setCacheMode(LOCAL); - cacheCfg1.setWriteSynchronizationMode(FULL_SYNC); - - CacheConfiguration cacheCfg2 = defaultCacheConfiguration(); - - cacheCfg2.setName("local2"); - cacheCfg2.setCacheMode(LOCAL); - cacheCfg2.setWriteSynchronizationMode(FULL_SYNC); - - cfg.setCacheConfiguration(cacheCfg1, cacheCfg2); - - return cfg; - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - stopAllGrids(); - } - - /** {@inheritDoc} */ - @Override protected long getTestTimeout() { - return 10 * 60 * 1000; - } - - /** - * Tests puts one by one. - * - * @throws Exception In case of error. - */ - public void testLuceneIndex() throws Exception { - final Ignite g = startGrid(0); - - final IgniteCache<Integer, ObjectValue> cache1 = g.cache("local1"); - final IgniteCache<Integer, ObjectValue> cache2 = g.cache("local2"); - - final AtomicInteger threadIdxGen = new AtomicInteger(); - - final int keyCnt = 10000; - - final IgniteInternalFuture<?> fut = multithreadedAsync( - new Callable<Object>() { - @Nullable @Override public Object call() throws Exception { - int threadIdx = threadIdxGen.getAndIncrement() % 2; - - for (int i = 0; i < keyCnt; i++) { - if (threadIdx == 0) - cache1.put(i, new ObjectValue("test full text more" + i)); - else - cache2.put(i, new ObjectValue("test full text more" + i)); - - if (i % 200 == 0) - info("Put entries count: " + i); - } - - return null; - } - }, - 10); - - IgniteInternalFuture<?> fut1 = multithreadedAsync( - new Callable<Object>() { - @Nullable @Override public Object call() throws Exception { - while (!fut.isDone()) { - Thread.sleep(10000); - -// ((GridKernal)g).internalCache("local1").context().queries().index().printH2Stats(); -// ((GridKernal)g).internalCache("local2").context().queries().index().printH2Stats(); - } - - return null; - } - }, - 1); - - fut.get(); - fut1.get(); - - assert cache1.size() == keyCnt; - assert cache2.size() == keyCnt; - } - - /** - * Tests with putAll. - * - * @throws Exception In case of error. - */ - public void testLuceneIndex1() throws Exception { - final Ignite g = startGrid(0); - - final IgniteCache<Integer, ObjectValue> cache1 = g.cache("local1"); - final IgniteCache<Integer, ObjectValue> cache2 = g.cache("local2"); - - final AtomicInteger threadIdxGen = new AtomicInteger(); - - final int keyCnt = 10000; - - final IgniteInternalFuture<?> fut = multithreadedAsync( - new Callable<Object>() { - @Nullable @Override public Object call() throws Exception { - int threadIdx = threadIdxGen.getAndIncrement() % 2; - - Map<Integer, ObjectValue> map = new HashMap<>(); - - for (int i = 0; i < keyCnt; i++) { - if (i % 200 == 0 && !map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - - info("Put entries count: " + i); - - map = new HashMap<>(); - } - - map.put(i, new ObjectValue("String value " + i)); - } - - if (!map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - } - - return null; - } - }, - 10); - - IgniteInternalFuture<?> fut1 = multithreadedAsync( - new Callable<Object>() { - @Nullable @Override public Object call() throws Exception { - while (!fut.isDone()) { - Thread.sleep(10000); - -// ((GridKernal)g).internalCache("local1").context().queries().index().printH2Stats(); -// ((GridKernal)g).internalCache("local2").context().queries().index().printH2Stats(); - } - - return null; - } - }, - 1); - - fut.get(); - fut1.get(); - - assert cache1.size() == keyCnt; - assert cache2.size() == keyCnt; - } - - /** - * Test same value with putAll. - * - * @throws Exception In case of error. - */ - public void testLuceneIndex2() throws Exception { - final Ignite g = startGrid(0); - - final IgniteCache<Integer, ObjectValue> cache1 = g.cache("local1"); - final IgniteCache<Integer, ObjectValue> cache2 = g.cache("local2"); - - final AtomicInteger threadIdxGen = new AtomicInteger(); - - final int keyCnt = 10000; - - final ObjectValue val = new ObjectValue("String value"); - - final IgniteInternalFuture<?> fut = multithreadedAsync( - new Callable<Object>() { - @Nullable @Override public Object call() throws Exception { - int threadIdx = threadIdxGen.getAndIncrement() % 2; - - Map<Integer, ObjectValue> map = new HashMap<>(); - - for (int i = 0; i < keyCnt; i++) { - if (i % 200 == 0 && !map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - - info("Put entries count: " + i); - - map = new HashMap<>(); - } - - map.put(i, val); - } - - if (!map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - } - - return null; - } - }, - 10); - - IgniteInternalFuture<?> fut1 = multithreadedAsync( - new Callable<Object>() { - @Nullable @Override public Object call() throws Exception { - while (!fut.isDone()) { - Thread.sleep(10000); - -// ((GridKernal)g).internalCache("local1").context().queries().index().printH2Stats(); -// ((GridKernal)g).internalCache("local2").context().queries().index().printH2Stats(); - } - - return null; - } - }, - 1); - - fut.get(); - fut1.get(); - - assert cache1.size() == keyCnt; - assert cache2.size() == keyCnt; - } - - /** - * Test limited values set and custom keys with putAll. - * - * @throws Exception In case of error. - */ - public void testLuceneIndex3() throws Exception { - final Ignite g = startGrid(0); - - final IgniteCache<ObjectKey, ObjectValue> cache1 = g.cache("local1"); - final IgniteCache<ObjectKey, ObjectValue> cache2 = g.cache("local2"); - - final AtomicInteger threadIdxGen = new AtomicInteger(); - - final int keyCnt = 10000; - - final ObjectValue[] vals = new ObjectValue[10]; - - for (int i = 0; i < vals.length; i++) - vals[i] = new ObjectValue("Object value " + i); - - final IgniteInternalFuture<?> fut = multithreadedAsync( - new Callable<Object>() { - @Nullable @Override public Object call() throws Exception { - int threadIdx = threadIdxGen.getAndIncrement() % 2; - - Map<ObjectKey, ObjectValue> map = new HashMap<>(); - - for (int i = 0; i < keyCnt; i++) { - if (i % 200 == 0 && !map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - - info("Put entries count: " + i); - - map = new HashMap<>(); - } - - map.put(new ObjectKey(String.valueOf(i)), F.rand(vals)); - } - - if (!map.isEmpty()) { - if (threadIdx == 0) - cache1.putAll(map); - else - cache2.putAll(map); - } - - return null; - } - }, - 1); - - IgniteInternalFuture<?> fut1 = multithreadedAsync( - new Callable<Object>() { - @Nullable @Override public Object call() throws Exception { - while (!fut.isDone()) { - Thread.sleep(10000); - -// ((GridKernal)g).internalCache("local1").context().queries().index().printH2Stats(); -// ((GridKernal)g).internalCache("local2").context().queries().index().printH2Stats(); - } - - return null; - } - }, - 1); - - fut.get(); - fut1.get(); - - assert cache1.size() == keyCnt; - assert cache2.size() == keyCnt; - } - - /** - * Test value object. - */ - private static class ObjectValue implements Serializable { - /** String value. */ - @QueryTextField - private String strVal; - - /** - * @param strVal String value. - */ - ObjectValue(String strVal) { - this.strVal = strVal; - } - - /** - * @return Value. - */ - public String stringValue() { - return strVal; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (o == null || getClass() != o.getClass()) - return false; - - ObjectValue other = (ObjectValue)o; - - return strVal == null ? other.strVal == null : strVal.equals(other.strVal); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return strVal != null ? strVal.hashCode() : 0; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(ObjectValue.class, this); - } - } - - /** - * Test value key. - */ - private static class ObjectKey implements Serializable { - /** String key. */ - @QueryTextField - private String strKey; - - /** - * @param strKey String key. - */ - ObjectKey(String strKey) { - this.strKey = strKey; - } - - /** - * @return Key. - */ - public String stringKey() { - return strKey; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (o == null || getClass() != o.getClass()) - return false; - - ObjectKey other = (ObjectKey)o; - - return strKey == null ? other.strKey == null : strKey.equals(other.strKey); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return strKey != null ? strKey.hashCode() : 0; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(ObjectKey.class, this); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/indexing/pom.xml ---------------------------------------------------------------------- diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml index 5c74f37..62fc402 100644 --- a/modules/indexing/pom.xml +++ b/modules/indexing/pom.xml @@ -54,6 +54,18 @@ </dependency> <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + <version>${lucene.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-queryparser</artifactId> + <version>${lucene.version}</version> + </dependency> + + <dependency> <groupId>com.h2database</groupId> <artifactId>h2</artifactId> <version>${h2.version}</version> http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java index 480922c..ff20987 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneDirectory.java @@ -20,20 +20,23 @@ package org.apache.ignite.internal.processors.query.h2.opt; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory; +import org.apache.lucene.store.BaseDirectory; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; /** * A memory-resident {@link Directory} implementation. */ -public class GridLuceneDirectory extends Directory { +public class GridLuceneDirectory extends BaseDirectory { /** */ protected final Map<String, GridLuceneFile> fileMap = new ConcurrentHashMap<>(); @@ -49,14 +52,9 @@ public class GridLuceneDirectory extends Directory { * @param mem Memory. */ public GridLuceneDirectory(GridUnsafeMemory mem) { - this.mem = mem; + super(new GridLuceneLockFactory()); - try { - setLockFactory(new GridLuceneLockFactory()); - } - catch (IOException e) { - throw new IllegalStateException(e); - } + this.mem = mem; } /** {@inheritDoc} */ @@ -75,28 +73,16 @@ public class GridLuceneDirectory extends Directory { } /** {@inheritDoc} */ - @Override public final boolean fileExists(String name) { - ensureOpen(); - - return fileMap.containsKey(name); - } - - /** {@inheritDoc} */ - @Override public final long fileModified(String name) { + @Override public void renameFile(String source, String dest) throws IOException { ensureOpen(); - throw new IllegalStateException(name); - } + GridLuceneFile file = fileMap.get(source); - /** - * Set the modified time of an existing file to now. - * - * @throws IOException if the file does not exist - */ - @Override public void touchFile(String name) throws IOException { - ensureOpen(); + if (file == null) + throw new FileNotFoundException(source); - throw new IllegalStateException(name); + fileMap.put(dest, file); + fileMap.remove(source); } /** {@inheritDoc} */ @@ -137,7 +123,7 @@ public class GridLuceneDirectory extends Directory { } /** {@inheritDoc} */ - @Override public IndexOutput createOutput(String name) throws IOException { + @Override public IndexOutput createOutput(final String name, final IOContext context) throws IOException { ensureOpen(); GridLuceneFile file = newRAMFile(); @@ -155,6 +141,11 @@ public class GridLuceneDirectory extends Directory { return new GridLuceneOutputStream(file); } + /** {@inheritDoc} */ + @Override public void sync(final Collection<String> names) throws IOException { + // Noop. No fsync needed as all data is in-memory. + } + /** * Returns a new {@link GridLuceneFile} for storing data. This method can be * overridden to return different {@link GridLuceneFile} impls, that e.g. override. @@ -166,7 +157,7 @@ public class GridLuceneDirectory extends Directory { } /** {@inheritDoc} */ - @Override public IndexInput openInput(String name) throws IOException { + @Override public IndexInput openInput(final String name, final IOContext context) throws IOException { ensureOpen(); GridLuceneFile file = fileMap.get(name); http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java index 93ebc71..eed5ee4 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneIndex.java @@ -36,20 +36,26 @@ import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.spi.indexing.IndexingQueryFilter; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.DateTools; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; -import org.apache.lucene.queryParser.MultiFieldQueryParser; -import org.apache.lucene.search.Filter; +import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TermRangeFilter; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.util.Version; +import org.apache.lucene.util.BytesRef; import org.h2.util.JdbcUtils; import org.jetbrains.annotations.Nullable; @@ -108,8 +114,8 @@ public class GridLuceneIndex implements AutoCloseable { dir = new GridLuceneDirectory(mem == null ? new GridUnsafeMemory(0) : mem); try { - writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_30, new StandardAnalyzer( - Version.LUCENE_30))); + writer = new IndexWriter(dir, + new IndexWriterConfig(new StandardAnalyzer())); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -163,7 +169,7 @@ public class GridLuceneIndex implements AutoCloseable { boolean stringsFound = false; if (type.valueTextIndex() || type.valueClass() == String.class) { - doc.add(new Field(VAL_STR_FIELD_NAME, val.toString(), Field.Store.YES, Field.Index.ANALYZED)); + doc.add(new TextField(VAL_STR_FIELD_NAME, val.toString(), Field.Store.YES)); stringsFound = true; } @@ -172,32 +178,34 @@ public class GridLuceneIndex implements AutoCloseable { Object fieldVal = type.value(idxdFields[i], key, val); if (fieldVal != null) { - doc.add(new Field(idxdFields[i], fieldVal.toString(), Field.Store.YES, Field.Index.ANALYZED)); + doc.add(new TextField(idxdFields[i], fieldVal.toString(), Field.Store.YES)); stringsFound = true; } } - String keyStr = org.apache.commons.codec.binary.Base64.encodeBase64String(k.valueBytes(coctx)); + BytesRef keyByteRef = new BytesRef(k.valueBytes(coctx)); try { - // Delete first to avoid duplicates. - writer.deleteDocuments(new Term(KEY_FIELD_NAME, keyStr)); + final Term term = new Term(KEY_FIELD_NAME, keyByteRef); + + if (!stringsFound) { + writer.deleteDocuments(term); - if (!stringsFound) return; // We did not find any strings to be indexed, will not store data at all. + } - doc.add(new Field(KEY_FIELD_NAME, keyStr, Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new StringField(KEY_FIELD_NAME, keyByteRef, Field.Store.YES)); if (type.valueClass() != String.class) - doc.add(new Field(VAL_FIELD_NAME, v.valueBytes(coctx))); + doc.add(new StoredField(VAL_FIELD_NAME, v.valueBytes(coctx))); - doc.add(new Field(VER_FIELD_NAME, ver.toString().getBytes())); + doc.add(new StoredField(VER_FIELD_NAME, ver.toString().getBytes())); - doc.add(new Field(EXPIRATION_TIME_FIELD_NAME, DateTools.timeToString(expires, - DateTools.Resolution.MILLISECOND), Field.Store.YES, Field.Index.NOT_ANALYZED)); + doc.add(new LongField(EXPIRATION_TIME_FIELD_NAME, expires, Field.Store.YES)); - writer.addDocument(doc); + // Next implies remove than add atomically operation. + writer.updateDocument(term, doc); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -216,7 +224,7 @@ public class GridLuceneIndex implements AutoCloseable { public void remove(CacheObject key) throws IgniteCheckedException { try { writer.deleteDocuments(new Term(KEY_FIELD_NAME, - org.apache.commons.codec.binary.Base64.encodeBase64String(key.valueBytes(objectContext())))); + new BytesRef(key.valueBytes(objectContext())))); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -247,7 +255,8 @@ public class GridLuceneIndex implements AutoCloseable { updateCntr.addAndGet(-updates); } - reader = IndexReader.open(writer, true); + //We can cache reader\searcher and change this to 'openIfChanged' + reader = DirectoryReader.open(writer, true); } catch (IOException e) { throw new IgniteCheckedException(e); @@ -255,17 +264,22 @@ public class GridLuceneIndex implements AutoCloseable { IndexSearcher searcher = new IndexSearcher(reader); - MultiFieldQueryParser parser = new MultiFieldQueryParser(Version.LUCENE_30, idxdFields, + MultiFieldQueryParser parser = new MultiFieldQueryParser(idxdFields, writer.getAnalyzer()); - // Filter expired items. - Filter f = new TermRangeFilter(EXPIRATION_TIME_FIELD_NAME, DateTools.timeToString(U.currentTimeMillis(), - DateTools.Resolution.MILLISECOND), null, false, false); - TopDocs docs; try { - docs = searcher.search(parser.parse(qry), f, Integer.MAX_VALUE); + // Filter expired items. + Query filter = NumericRangeQuery.newLongRange(EXPIRATION_TIME_FIELD_NAME, U.currentTimeMillis(), + null, false, false); + + BooleanQuery query = new BooleanQuery.Builder() + .add(parser.parse(qry), BooleanClause.Occur.MUST) + .add(filter, BooleanClause.Occur.FILTER) + .build(); + + docs = searcher.search(query, Integer.MAX_VALUE); } catch (Exception e) { throw new IgniteCheckedException(e); @@ -342,7 +356,7 @@ public class GridLuceneIndex implements AutoCloseable { * @return {@code True} if key passes filter. */ private boolean filter(K key, V val) { - return filters == null || filters.apply(key, val) ; + return filters == null || filters.apply(key, val); } /** @@ -383,11 +397,11 @@ public class GridLuceneIndex implements AutoCloseable { if (ctx != null && ctx.deploy().enabled()) ldr = ctx.cache().internalCache(cacheName).context().deploy().globalLoader(); - K k = unmarshall(org.apache.commons.codec.binary.Base64.decodeBase64(doc.get(KEY_FIELD_NAME)), ldr); + K k = unmarshall(doc.getBinaryValue(KEY_FIELD_NAME).bytes, ldr); V v = type.valueClass() == String.class ? (V)doc.get(VAL_STR_FIELD_NAME) : - this.<V>unmarshall(doc.getBinaryValue(VAL_FIELD_NAME), ldr); + this.<V>unmarshall(doc.getBinaryValue(VAL_FIELD_NAME).bytes, ldr); assert v != null; @@ -416,7 +430,6 @@ public class GridLuceneIndex implements AutoCloseable { /** {@inheritDoc} */ @Override protected void onClose() throws IgniteCheckedException { - U.closeQuiet(searcher); U.closeQuiet(reader); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java index eda97f3..4820af1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneInputStream.java @@ -21,7 +21,6 @@ import java.io.EOFException; import java.io.IOException; import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; import static org.apache.ignite.internal.processors.query.h2.opt.GridLuceneOutputStream.BUFFER_SIZE; @@ -61,11 +60,23 @@ public class GridLuceneInputStream extends IndexInput { * @throws IOException If failed. */ public GridLuceneInputStream(String name, GridLuceneFile f) throws IOException { + this(name, f, f.getLength()); + } + + /** + * Constructor. + * + * @param name Name. + * @param f File. + * @param length inputStream length. + * @throws IOException If failed. + */ + public GridLuceneInputStream(String name, GridLuceneFile f, final long length) throws IOException { super("RAMInputStream(name=" + name + ")"); file = f; - length = file.getLength(); + this.length = length; if (length / BUFFER_SIZE >= Integer.MAX_VALUE) throw new IOException("RAMInputStream too large length=" + length + ": " + name); @@ -149,39 +160,14 @@ public class GridLuceneInputStream extends IndexInput { } /** {@inheritDoc} */ - @Override public void copyBytes(IndexOutput out, long numBytes) throws IOException { - assert numBytes >= 0 : "numBytes=" + numBytes; - - GridLuceneOutputStream gridOut = out instanceof GridLuceneOutputStream ? (GridLuceneOutputStream)out : null; - - long left = numBytes; - - while (left > 0) { - if (bufPosition == bufLength) { - ++currBufIdx; - - switchCurrentBuffer(true); - } - - final int bytesInBuf = bufLength - bufPosition; - final int toCp = (int)(bytesInBuf < left ? bytesInBuf : left); - - if (gridOut != null) - gridOut.writeBytes(currBuf + bufPosition, toCp); - else { - byte[] buff = new byte[toCp]; - - mem.readBytes(currBuf + bufPosition, buff); - - out.writeBytes(buff, toCp); - } - - bufPosition += toCp; + @Override + public IndexInput slice(final String sliceDescription, final long offset, final long length) throws IOException { + if (offset < 0 || length < 0 || offset + length > this.length) + throw new IllegalArgumentException("slice() " + sliceDescription + " out of bounds: " + this); - left -= toCp; - } + final String newResourceDescription = (sliceDescription == null) ? toString() : (toString() + " [slice=" + sliceDescription + "]"); - assert left == 0 : "Insufficient bytes to copy: numBytes=" + numBytes + " copied=" + (numBytes - left); + return new SlicedInputStream(newResourceDescription, offset, length); } /** @@ -226,4 +212,46 @@ public class GridLuceneInputStream extends IndexInput { bufPosition = (int)(pos % BUFFER_SIZE); } + + /** */ + private class SlicedInputStream extends GridLuceneInputStream { + /** */ + private final long offset; + + /** */ + public SlicedInputStream(String newResourceDescription, long offset, long length) throws IOException { + super(newResourceDescription, GridLuceneInputStream.this.file, offset + length); + + this.offset = offset; + + seek(0L); + } + + /** {@inheritDoc} */ + @Override + public void seek(long pos) throws IOException { + if (pos < 0L) { + throw new IllegalArgumentException("Seeking to negative position: " + this); + } + super.seek(pos + offset); + } + + /** {@inheritDoc} */ + @Override + public long getFilePointer() { + return super.getFilePointer() - offset; + } + + /** {@inheritDoc} */ + @Override + public long length() { + return super.length() - offset; + } + + /** {@inheritDoc} */ + @Override + public IndexInput slice(String sliceDescription, long ofs, long len) throws IOException { + return super.slice(sliceDescription, offset + ofs, len); + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneLockFactory.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneLockFactory.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneLockFactory.java index c2ee768..e14a408 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneLockFactory.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneLockFactory.java @@ -19,8 +19,11 @@ package org.apache.ignite.internal.processors.query.h2.opt; import java.io.IOException; import org.apache.ignite.internal.util.GridConcurrentHashSet; +import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.LockObtainFailedException; /** * Lucene {@link LockFactory} implementation. @@ -31,13 +34,11 @@ public class GridLuceneLockFactory extends LockFactory { private final GridConcurrentHashSet<String> locks = new GridConcurrentHashSet<>(); /** {@inheritDoc} */ - @Override public Lock makeLock(String lockName) { - return new LockImpl(lockName); - } - - /** {@inheritDoc} */ - @Override public void clearLock(String lockName) throws IOException { - locks.remove(lockName); + @Override public Lock obtainLock(Directory dir, String lockName) throws IOException { + if (locks.add(lockName)) + return new LockImpl(lockName); + else + throw new LockObtainFailedException("lock instance already obtained: (dir=" + dir + ", lockName=" + lockName + ")"); } /** @@ -47,6 +48,9 @@ public class GridLuceneLockFactory extends LockFactory { /** */ private final String lockName; + /** */ + private volatile boolean closed; + /** * @param lockName Lock name. */ @@ -55,18 +59,33 @@ public class GridLuceneLockFactory extends LockFactory { } /** {@inheritDoc} */ - @Override public boolean obtain() throws IOException { - return locks.add(lockName); + @Override public void ensureValid() throws IOException { + if (closed) + throw new AlreadyClosedException("Lock instance already released: " + this); + + // check we are still in the locks map (some debugger or something crazy didn't remove us) + if (!locks.contains(lockName)) + throw new AlreadyClosedException("Lock instance was invalidated from map: " + this); } /** {@inheritDoc} */ - @Override public void release() throws IOException { - locks.remove(lockName); + @Override public void close() throws IOException { + if (closed) + return; + + try { + if (!locks.remove(lockName)) + throw new AlreadyClosedException("Lock was already released: " + this); + } + finally { + closed = true; + } } /** {@inheritDoc} */ - @Override public boolean isLocked() throws IOException { - return locks.contains(lockName); + @Override + public String toString() { + return super.toString() + ": " + lockName; } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java index 8d3d79c..caea226 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridLuceneOutputStream.java @@ -18,7 +18,10 @@ package org.apache.ignite.internal.processors.query.h2.opt; import java.io.IOException; +import java.util.zip.CRC32; +import java.util.zip.Checksum; import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory; +import org.apache.lucene.store.BufferedChecksum; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IndexOutput; @@ -50,12 +53,17 @@ public class GridLuceneOutputStream extends IndexOutput { /** */ private final GridUnsafeMemory mem; + /** */ + private final Checksum crc; + /** * Constructor. * * @param f File. */ public GridLuceneOutputStream(GridLuceneFile f) { + super("RAMOutputStream(name=\"noname\")"); + file = f; mem = f.getDirectory().memory(); @@ -64,6 +72,8 @@ public class GridLuceneOutputStream extends IndexOutput { // first needed buffer lazily currBufIdx = -1; currBuf = 0; + + crc = new BufferedChecksum(new CRC32()); } /** @@ -77,6 +87,7 @@ public class GridLuceneOutputStream extends IndexOutput { bufLength = 0; file.setLength(0); + crc.reset(); } /** {@inheritDoc} */ @@ -85,23 +96,8 @@ public class GridLuceneOutputStream extends IndexOutput { } /** {@inheritDoc} */ - @Override public void seek(long pos) throws IOException { - // set the file length in case we seek back - // and flush() has not been called yet - setFileLength(); - - if (pos < bufStart || pos >= bufStart + bufLength) { - currBufIdx = (int)(pos / BUFFER_SIZE); - - switchCurrentBuffer(); - } - - bufPosition = (int)(pos % BUFFER_SIZE); - } - - /** {@inheritDoc} */ - @Override public long length() { - return file.getLength(); + @Override public long getChecksum() throws IOException { + return crc.getValue(); } /** {@inheritDoc} */ @@ -112,6 +108,8 @@ public class GridLuceneOutputStream extends IndexOutput { switchCurrentBuffer(); } + crc.update(b); + mem.writeByte(currBuf + bufPosition++, b); } @@ -119,6 +117,8 @@ public class GridLuceneOutputStream extends IndexOutput { @Override public void writeBytes(byte[] b, int offset, int len) throws IOException { assert b != null; + crc.update(b, offset, len); + while (len > 0) { if (bufPosition == bufLength) { currBufIdx++; @@ -159,8 +159,8 @@ public class GridLuceneOutputStream extends IndexOutput { file.setLength(pointer); } - /** {@inheritDoc} */ - @Override public void flush() throws IOException { + /** Forces any buffered output to be written. */ + private void flush() throws IOException { setFileLength(); } @@ -169,15 +169,6 @@ public class GridLuceneOutputStream extends IndexOutput { return currBufIdx < 0 ? 0 : bufStart + bufPosition; } - /** - * Returns byte usage of all buffers. - * - * @return Bytes used. - */ - public long sizeInBytes() { - return (long)file.numBuffers() * (long)BUFFER_SIZE; - } - /** {@inheritDoc} */ @Override public void copyBytes(DataInput input, long numBytes) throws IOException { assert numBytes >= 0 : "numBytes=" + numBytes; @@ -210,29 +201,4 @@ public class GridLuceneOutputStream extends IndexOutput { bufPosition += toCp; } } - - /** - * For direct usage by {@link GridLuceneInputStream}. - * - * @param ptr Pointer. - * @param len Length. - * @throws IOException If failed. - */ - void writeBytes(long ptr, int len) throws IOException { - while (len > 0) { - if (bufPosition == bufLength) { - currBufIdx++; - switchCurrentBuffer(); - } - - int remainInBuf = BUFFER_SIZE - bufPosition; - int bytesToCp = len < remainInBuf ? len : remainInBuf; - - mem.copyMemory(ptr, currBuf + bufPosition, bytesToCp); - - ptr += bytesToCp; - len -= bytesToCp; - bufPosition += bytesToCp; - } - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java new file mode 100644 index 0000000..747038d --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQuerySelfTest.java @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import javax.cache.Cache; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cache.query.Query; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.TextQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.cache.query.annotations.QueryTextField; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; + +/** + * FullTest queries left test. + */ +public class GridCacheFullTextQuerySelfTest extends GridCommonAbstractTest { + /** Cache size. */ + private static final int MAX_ITEM_COUNT = 100; + + /** Cache name */ + private static final String PERSON_CACHE = "Person"; + + /** */ + private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + + disco.setIpFinder(ipFinder); + + cfg.setDiscoverySpi(disco); + + cfg.setIncludeEventTypes(); + + cfg.setConnectorConfiguration(null); + + CacheConfiguration<Integer, Person> cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName(PERSON_CACHE) + .setCacheMode(PARTITIONED) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL) + .setWriteSynchronizationMode(FULL_SYNC) + .setBackups(0) + .setIndexedTypes(Integer.class, Person.class); + + cfg.setCacheConfiguration(cacheCfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrids(2); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testLocalTextQueryWithKeepBinary() throws Exception { + checkTextQuery(true, true); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testLocalTextQuery() throws Exception { + checkTextQuery(true, false); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testTextQueryWithKeepBinary() throws Exception { + checkTextQuery(false, true); + } + + /** + * JUnit. + * + * @throws Exception In case of error. + */ + public void testTextQuery() throws Exception { + checkTextQuery(false, true); + } + + /** + * @param loc local query flag. + * @param keepBinary keep binary flag. + */ + private void checkTextQuery(boolean loc, boolean keepBinary) throws Exception { + final IgniteEx ignite = grid(0); + + // 1. Populate cache with data, calculating expected count in parallel. + Set<Integer> exp = populateCache(ignite, loc, MAX_ITEM_COUNT, new IgnitePredicate<Integer>() { + @Override + public boolean apply(Integer x) { + return String.valueOf(x).startsWith("1"); + } + }); + + // 2. Validate results. + TextQuery qry = new TextQuery<>(Person.class, "1*").setLocal(loc); + + validateQueryResults(ignite, qry, exp, keepBinary); + + clearCache(ignite); + } + + /** + * Clear cache with check. + */ + private static void clearCache(IgniteEx ignite) { + IgniteCache<Integer, Person> cache = ignite.cache(PERSON_CACHE); + + cache.clear(); + + List all = cache.query(new TextQuery<>(Person.class, "1*")).getAll(); + + assertTrue(all.isEmpty()); + } + + /** + * Fill cache. + * + * @throws IgniteCheckedException if failed. + */ + private static Set<Integer> populateCache(IgniteEx ignite, boolean loc, int cnt, + IgnitePredicate<Integer> expectedEntryFilter) throws IgniteCheckedException { + IgniteInternalCache<Integer, Person> cache = ignite.cachex(PERSON_CACHE); + + assertNotNull(cache); + + Random rand = new Random(); + + HashSet<Integer> exp = new HashSet<>(); + + Affinity<Integer> aff = cache.affinity(); + + ClusterNode localNode = cache.context().localNode(); + + for (int i = 0; i < cnt; i++) { + int val = rand.nextInt(cnt); + + cache.put(val, new Person(String.valueOf(val), val)); + + if (expectedEntryFilter.apply(val) && (!loc || aff.isPrimary(localNode, val))) + exp.add(val); + } + + return exp; + } + + /** + * Check query results. + * + * @throws IgniteCheckedException if failed. + */ + private static void validateQueryResults(IgniteEx ignite, Query qry, Set<Integer> exp, + boolean keepBinary) throws IgniteCheckedException { + IgniteCache<Integer, Person> cache = ignite.cache(PERSON_CACHE); + + if (keepBinary) { + IgniteCache<Integer, BinaryObject> cache0 = cache.withKeepBinary(); + + try (QueryCursor<Cache.Entry<Integer, BinaryObject>> cursor = cache0.query(qry)) { + Set<Integer> exp0 = new HashSet<>(exp); + + List<Cache.Entry<Integer, ?>> all = new ArrayList<>(); + + for (Cache.Entry<Integer, BinaryObject> entry : cursor.getAll()) { + all.add(entry); + + assertEquals(entry.getKey().toString(), entry.getValue().field("name")); + + assertEquals(entry.getKey(), entry.getValue().field("age")); + + exp0.remove(entry.getKey()); + } + + checkForMissedKeys(ignite, exp0, all); + } + + try (QueryCursor<Cache.Entry<Integer, BinaryObject>> cursor = cache0.query(qry)) { + Set<Integer> exp0 = new HashSet<>(exp); + + List<Cache.Entry<Integer, ?>> all = new ArrayList<>(); + + for (Cache.Entry<Integer, BinaryObject> entry : cursor.getAll()) { + all.add(entry); + + assertEquals(entry.getKey().toString(), entry.getValue().field("name")); + + assertEquals(entry.getKey(), entry.getValue().field("age")); + + exp0.remove(entry.getKey()); + } + + checkForMissedKeys(ignite, exp0, all); + } + } + else { + try (QueryCursor<Cache.Entry<Integer, Person>> cursor = cache.query(qry)) { + Set<Integer> exp0 = new HashSet<>(exp); + + List<Cache.Entry<Integer, ?>> all = new ArrayList<>(); + + for (Cache.Entry<Integer, Person> entry : cursor.getAll()) { + all.add(entry); + + assertEquals(entry.getKey().toString(), entry.getValue().name); + + assertEquals(entry.getKey(), Integer.valueOf(entry.getValue().age)); + + exp0.remove(entry.getKey()); + } + + checkForMissedKeys(ignite, exp0, all); + } + + try (QueryCursor<Cache.Entry<Integer, Person>> cursor = cache.query(qry)) { + Set<Integer> exp0 = new HashSet<>(exp); + + List<Cache.Entry<Integer, ?>> all = new ArrayList<>(); + + for (Cache.Entry<Integer, Person> entry : cursor.getAll()) { + all.add(entry); + + assertEquals(entry.getKey().toString(), entry.getValue().name); + + assertEquals(entry.getKey().intValue(), entry.getValue().age); + + exp0.remove(entry.getKey()); + } + + checkForMissedKeys(ignite, exp0, all); + } + } + } + + /** + * Check if there is missed keys. + * + * @throws IgniteCheckedException if failed. + */ + private static void checkForMissedKeys(IgniteEx ignite, Collection<Integer> exp, + List<Cache.Entry<Integer, ?>> all) throws IgniteCheckedException { + if (exp.size() == 0) + return; + + IgniteInternalCache<Integer, Person> cache = ignite.cachex(PERSON_CACHE); + + assertNotNull(cache); + + StringBuilder sb = new StringBuilder(); + + Affinity<Integer> aff = cache.affinity(); + + for (Integer key : exp) { + Integer part = aff.partition(key); + + sb.append( + String.format("Query did not return expected key '%d' (exists: %s), partition '%d', partition nodes: ", + key, cache.get(key) != null, part)); + + Collection<ClusterNode> partNodes = aff.mapPartitionToPrimaryAndBackups(part); + + for (ClusterNode node : partNodes) + sb.append(node).append(" "); + + sb.append(";\n"); + } + + sb.append("Returned keys: "); + + for (Cache.Entry e : all) + sb.append(e.getKey()).append(" "); + + sb.append(";\n"); + + fail(sb.toString()); + } + + /** + * Test model class. + */ + public static class Person implements Serializable { + /** */ + @QueryTextField + String name; + + /** */ + @QuerySqlField(index = true) + int age; + + /** */ + @QuerySqlField final Date birthday; + + /** + * Constructor + */ + public Person(String name, int age) { + this.name = name; + this.age = age % 2000; + + Calendar cal = Calendar.getInstance(); + cal.add(Calendar.YEAR, -age); + + birthday = cal.getTime(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java index 2a75bd3..ba0324f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheFullTextQueryNodeJoiningSelfTest.java @@ -22,6 +22,7 @@ import java.util.LinkedHashMap; import javax.cache.Cache; import org.apache.ignite.Ignite; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheRebalanceMode; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.QueryIndex; import org.apache.ignite.cache.QueryIndexType; @@ -61,6 +62,7 @@ public class IgniteCacheFullTextQueryNodeJoiningSelfTest extends GridCommonAbstr cache.setAtomicityMode(atomicityMode()); cache.setWriteSynchronizationMode(FULL_SYNC); cache.setBackups(1); + cache.setRebalanceMode(CacheRebalanceMode.SYNC); QueryEntity qryEntity = new QueryEntity(); @@ -121,7 +123,7 @@ public class IgniteCacheFullTextQueryNodeJoiningSelfTest extends GridCommonAbstr QueryCursor<Cache.Entry<AffinityKey<Integer>, IndexedEntity>> res = started.cache(DEFAULT_CACHE_NAME) .query(new TextQuery<AffinityKey<Integer>, IndexedEntity>(IndexedEntity.class, "indexed")); - assertEquals(1000, res.getAll().size()); + assertEquals("Failed iteration: " + i, 1000, res.getAll().size()); } } finally { http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java index 012ed29..258eed8 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java @@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.CacheReplicatedQueryMetricsDi import org.apache.ignite.internal.processors.cache.CacheReplicatedQueryMetricsLocalSelfTest; import org.apache.ignite.internal.processors.cache.CacheSqlQueryValueCopySelfTest; import org.apache.ignite.internal.processors.cache.GridCacheCrossCacheQuerySelfTest; +import org.apache.ignite.internal.processors.cache.GridCacheFullTextQuerySelfTest; import org.apache.ignite.internal.processors.cache.GridCacheQueryIndexDisabledSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheQueryIndexingDisabledSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheQueryInternalKeysSelfTest; @@ -266,13 +267,13 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite { suite.addTestSuite(IgniteCacheAtomicNearEnabledFieldsQuerySelfTest.class); suite.addTestSuite(IgniteCachePartitionedFieldsQueryP2PEnabledSelfTest.class); suite.addTestSuite(IgniteCacheFieldsQueryNoDataSelfTest.class); - suite.addTestSuite(GridCacheQueryIndexingDisabledSelfTest.class); - suite.addTestSuite(GridOrderedMessageCancelSelfTest.class); - suite.addTestSuite(CacheQueryEvictDataLostTest.class); + // Full text queries. + suite.addTestSuite(GridCacheFullTextQuerySelfTest.class); + // Ignite cache and H2 comparison. suite.addTestSuite(BaseH2CompareQueryTest.class); suite.addTestSuite(H2CompareBigQueryTest.class); http://git-wip-us.apache.org/repos/asf/ignite/blob/478d3b5d/parent/pom.xml ---------------------------------------------------------------------- diff --git a/parent/pom.xml b/parent/pom.xml index a481657..7443753 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -88,8 +88,8 @@ <jtidy.version>r938</jtidy.version> <kafka.version>0.10.0.1</kafka.version> <karaf.version>4.0.2</karaf.version> - <lucene.bundle.version>3.5.0_1</lucene.bundle.version> - <lucene.version>3.5.0</lucene.version> + <lucene.bundle.version>5.5.2_1</lucene.bundle.version> + <lucene.version>5.5.2</lucene.version> <oro.bundle.version>2.0.8_6</oro.bundle.version> <osgi.core.version>5.0.0</osgi.core.version> <osgi.enterprise.version>5.0.0</osgi.enterprise.version>