amit-jain commented on code in PR #2817: URL: https://github.com/apache/jackrabbit-oak/pull/2817#discussion_r4346679654
########## oak-search-lucene-ng/src/main/java/org/apache/jackrabbit/oak/plugins/index/luceneNg/internal/LuceneNgQueryBuilder.java: ########## @@ -0,0 +1,615 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.luceneNg.internal; + +import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.api.PropertyValue; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.luceneNg.LuceneNgIndexConstants; +import org.apache.jackrabbit.oak.plugins.index.luceneNg.LuceneNgIndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner.PlanResult; +import org.apache.jackrabbit.oak.plugins.index.search.util.QueryUtils; +import org.apache.jackrabbit.oak.plugins.memory.PropertyValues; +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction; +import org.apache.jackrabbit.oak.spi.query.QueryConstants; +import org.apache.jackrabbit.oak.spi.query.QueryIndex.OrderEntry; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextAnd; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextContains; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextOr; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor; +import org.apache.jackrabbit.util.ISO8601; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.jcr.PropertyType; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Translates an Oak query plan into the Lucene query and sort executed against a LuceneNg index. + */ +public final class LuceneNgQueryBuilder { + + private static final Logger LOG = LoggerFactory.getLogger(LuceneNgQueryBuilder.class); + + private LuceneNgQueryBuilder() { + } + + /** + * The Lucene query for the filter of a plan. Property restrictions become clauses only when + * the planner matched them to an indexed property definition; the query engine evaluates + * the others. + */ + @NotNull + public static Query query(@NotNull Filter filter, @NotNull PlanResult planResult) { + List<Query> propertyQueries = new ArrayList<>(); + for (PropertyRestriction pr : filter.getPropertyRestrictions()) { + Query propertyQuery = restrictionQuery(pr, planResult); + if (propertyQuery != null) { + propertyQueries.add(propertyQuery); + } + } + + FullTextExpression ft = filter.getFullTextConstraint(); + Query contentQuery; + if (ft != null) { + Query fullTextQuery; + try (Analyzer analyzer = new StandardAnalyzer()) { + fullTextQuery = fullTextQuery(ft, analyzer, planResult); + } + if (propertyQueries.isEmpty()) { + contentQuery = fullTextQuery; + } else { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(fullTextQuery, Occur.MUST); + propertyQueries.forEach(q -> bq.add(q, Occur.MUST)); Review Comment: **Fulltext on a relative property: the other conditions are applied to the child document, so rows are missed** When the fulltext term is relative (e.g. `CONTAINS(a.[jcr:content/title], 'x')`), the shared planner relativizes the plan: it searches `title` on the `jcr:content` documents and maps hits back to the parent. In that case it also turns off the non-fulltext conditions (`FulltextIndexPlanner.setParentPath`: *"we only restrict non-full-text conditions if there is no relative property in the full-text constraint"*), exposed as `planResult.evaluateNonFullTextConstraints()`. This builder doesn't check it and ANDs every matched property restriction (and the node-type clause, L134) into the fulltext query, i.e. onto the **child** documents, which don't have those properties. Repro (index rules `oak:Unstructured → status (propertyIndex), description (analyzed)` and `nt:base → title (analyzed)`; `/content/a` status=live and `/content/b` status=draft, each with a `jcr:content` child title="hello x"): ```sql SELECT [jcr:path] FROM [oak:Unstructured] AS a WHERE CONTAINS(a.[jcr:content/title], 'x') AND a.[status] = 'live' ``` | | `EXPLAIN` luceneQuery | rows | |---|---|---| | oak-lucene | `full:title:x` | `/content/a` (engine filters `status`) | | luceneNg | `+full:title:x +status:live` | none | Legacy only adds these clauses when the planner allows it ([`LucenePropertyIndex` L966-968](https://github.com/bhabegger/jackrabbit-oak/blob/2bcbbf22eff607b9105c038e02994f9c39ca9b69/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java#L966-L968) → `addNonFullTextConstraints`). Rows the index never returns can't be recovered by the query engine. Suggested fix: when `ft != null`, add the property and node-type clauses only if `planResult.evaluateNonFullTextConstraints()`. (The path clause is fine; it's already translated for transformed plans.) ########## oak-search-lucene-ng/src/main/java/org/apache/jackrabbit/oak/plugins/index/luceneNg/internal/LuceneNgQueryBuilder.java: ########## @@ -0,0 +1,615 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.luceneNg.internal; + +import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.api.PropertyValue; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.luceneNg.LuceneNgIndexConstants; +import org.apache.jackrabbit.oak.plugins.index.luceneNg.LuceneNgIndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner.PlanResult; +import org.apache.jackrabbit.oak.plugins.index.search.util.QueryUtils; +import org.apache.jackrabbit.oak.plugins.memory.PropertyValues; +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction; +import org.apache.jackrabbit.oak.spi.query.QueryConstants; +import org.apache.jackrabbit.oak.spi.query.QueryIndex.OrderEntry; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextAnd; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextContains; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextOr; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor; +import org.apache.jackrabbit.util.ISO8601; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.jcr.PropertyType; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Translates an Oak query plan into the Lucene query and sort executed against a LuceneNg index. + */ +public final class LuceneNgQueryBuilder { + + private static final Logger LOG = LoggerFactory.getLogger(LuceneNgQueryBuilder.class); + + private LuceneNgQueryBuilder() { + } + + /** + * The Lucene query for the filter of a plan. Property restrictions become clauses only when + * the planner matched them to an indexed property definition; the query engine evaluates + * the others. + */ + @NotNull + public static Query query(@NotNull Filter filter, @NotNull PlanResult planResult) { + List<Query> propertyQueries = new ArrayList<>(); + for (PropertyRestriction pr : filter.getPropertyRestrictions()) { + Query propertyQuery = restrictionQuery(pr, planResult); + if (propertyQuery != null) { + propertyQueries.add(propertyQuery); + } + } + + FullTextExpression ft = filter.getFullTextConstraint(); + Query contentQuery; + if (ft != null) { + Query fullTextQuery; + try (Analyzer analyzer = new StandardAnalyzer()) { + fullTextQuery = fullTextQuery(ft, analyzer, planResult); + } + if (propertyQueries.isEmpty()) { + contentQuery = fullTextQuery; + } else { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(fullTextQuery, Occur.MUST); + propertyQueries.forEach(q -> bq.add(q, Occur.MUST)); + contentQuery = bq.build(); + } + } else if (propertyQueries.isEmpty()) { + contentQuery = new MatchAllDocsQuery(); + } else if (propertyQueries.size() == 1) { + contentQuery = propertyQueries.get(0); + } else { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + propertyQueries.forEach(q -> bq.add(q, Occur.MUST)); + contentQuery = bq.build(); + } + + Query pathQuery = pathQuery(filter, planResult); + Query typeQuery = nodeTypeQuery(filter, planResult); + if (pathQuery == null && typeQuery == null) { + return contentQuery; + } + BooleanQuery.Builder combined = new BooleanQuery.Builder(); + combined.add(contentQuery, Occur.MUST); + if (pathQuery != null) { + combined.add(pathQuery, Occur.FILTER); + } + if (typeQuery != null) { + combined.add(typeQuery, Occur.FILTER); + } + return combined.build(); + } + + @Nullable + private static Query restrictionQuery(PropertyRestriction pr, PlanResult planResult) { + String name = pr.propertyName; + if (QueryConstants.REP_FACET.equals(name) || name.startsWith(QueryConstants.FUNCTION_RESTRICTION_PREFIX) + || name.startsWith("rep:") || name.startsWith("oak:")) { + return null; + } + if (QueryConstants.RESTRICTION_LOCAL_NAME.equals(name)) { + return planResult.evaluateNodeNameRestriction() ? localNameQuery(pr) : null; + } + PropertyDefinition definition = planResult.getPropDefn(pr); + if (definition == null) { + return null; + } + String field = planResult.getPropertyName(pr); + return propertyQuery(field, pr, definition, + ((LuceneNgIndexDefinition) planResult.indexDefinition).fieldType(field, definition)); + } + + @Nullable + private static Query nodeTypeQuery(Filter filter, PlanResult result) { + if (filter.matchesAllTypes()) { + return null; + } + BooleanQuery.Builder types = new BooleanQuery.Builder(); + PropertyDefinition primary = result.indexingRule.getConfig(JcrConstants.JCR_PRIMARYTYPE); + if (primary != null && primary.index && primary.propertyIndex) { + for (String type : filter.getPrimaryTypes()) { + types.add(new TermQuery(new Term(JcrConstants.JCR_PRIMARYTYPE, type)), Occur.SHOULD); + } + } + PropertyDefinition mixin = result.indexingRule.getConfig(JcrConstants.JCR_MIXINTYPES); + if (mixin != null && mixin.index && mixin.propertyIndex) { + for (String type : filter.getMixinTypes()) { + types.add(new TermQuery(new Term(JcrConstants.JCR_MIXINTYPES, type)), Occur.SHOULD); + } + } + BooleanQuery query = types.build(); + return query.clauses().isEmpty() ? null : query; + } + + /** + * When the plan evaluates relative properties on descendants (a transformed path), documents + * are those descendants: {@code parentPathSegment} is appended to the restricted path, and + * direct children are the documents of the subtree at the depth of a child plus the depth + * of the relative path. + */ + @Nullable + private static Query pathQuery(Filter filter, PlanResult planResult) { + Filter.PathRestriction restriction = filter.getPathRestriction(); + if (restriction == null) { + return null; + } + String path = filter.getPath(); + boolean transformed = planResult.isPathTransformed(); + String segment = transformed ? planResult.getParentPathSegment() : ""; + if (segment.contains("*")) { + return null; + } + switch (restriction) { + case ALL_CHILDREN: + if ("/".equals(path)) { + return null; + } + return new TermQuery(new Term(FieldNames.ANCESTORS, path)); + case DIRECT_CHILDREN: + if (transformed) { + Query depth = IntPoint.newExactQuery(FieldNames.PATH_DEPTH, + PathUtils.getDepth(path) + planResult.getParentDepth() + 1); + if ("/".equals(path)) { + return depth; + } + return new BooleanQuery.Builder() + .add(new TermQuery(new Term(FieldNames.ANCESTORS, path)), Occur.MUST) + .add(depth, Occur.MUST) + .build(); + } + return new TermQuery(new Term(LuceneNgIndexConstants.FIELD_PARENT_PATH, path)); + case EXACT: + return new TermQuery(new Term(FieldNames.PATH, withSegment(path, segment))); + case PARENT: + if ("/".equals(path)) { + // root has no parent: a path no document has + return new TermQuery(new Term(FieldNames.PATH, "\u0000")); + } + int lastSlash = path.lastIndexOf('/'); + String parentPath = lastSlash == 0 ? "/" : path.substring(0, lastSlash); + return new TermQuery(new Term(FieldNames.PATH, withSegment(parentPath, segment))); + case NO_RESTRICTION: + default: + return null; + } + } + + private static String withSegment(String path, String segment) { + return segment.isEmpty() ? path : "/".equals(path) ? segment : path + segment; + } + + /** + * The Lucene query for a restriction on the indexed field {@code propertyName}, according to + * the declared type of the property. + */ + @NotNull + static Query propertyQuery(@NotNull String propertyName, @NotNull PropertyRestriction pr, + @NotNull PropertyDefinition definition) { + return propertyQuery(propertyName, pr, definition, definition.getType()); + } + + /** + * @param propertyType the type that decides the Lucene field of the property + */ + @NotNull + static Query propertyQuery(@NotNull String propertyName, @NotNull PropertyRestriction pr, + @NotNull PropertyDefinition definition, int propertyType) { + + if (pr.isNotNullRestriction()) { + if (definition.notNullCheckEnabled) { + return new TermQuery(new Term(FieldNames.NOT_NULL_PROPS, definition.name)); + } + switch (propertyType) { + case PropertyType.LONG: + case PropertyType.DATE: + return LongPoint.newRangeQuery(propertyName, Long.MIN_VALUE, Long.MAX_VALUE); + case PropertyType.DOUBLE: + case PropertyType.DECIMAL: + return DoublePoint.newRangeQuery(propertyName, Double.NEGATIVE_INFINITY, Double.NaN); + case PropertyType.UNDEFINED: + BooleanQuery.Builder existence = new BooleanQuery.Builder(); + existence.add(new TermRangeQuery(propertyName, null, null, true, true), Occur.SHOULD); + existence.add(LongPoint.newRangeQuery(propertyName, Long.MIN_VALUE, Long.MAX_VALUE), Occur.SHOULD); + existence.add(DoublePoint.newRangeQuery(propertyName, Double.NEGATIVE_INFINITY, Double.NaN), + Occur.SHOULD); + return existence.build(); + default: + return new TermRangeQuery(propertyName, null, null, true, true); + } + } + + if (pr.isNullRestriction()) { + if (definition.nullCheckEnabled) { + return new TermQuery(new Term(FieldNames.NULL_PROPS, definition.name)); + } + return new MatchAllDocsQuery(); + } + + switch (propertyType) { + case PropertyType.LONG: + return numericQuery(propertyName, pr, LONG_POINT); + case PropertyType.DOUBLE: + case PropertyType.DECIMAL: + return numericQuery(propertyName, pr, DOUBLE_POINT); + case PropertyType.DATE: + return dateQuery(propertyName, pr); + case PropertyType.BOOLEAN: + return booleanQuery(propertyName, pr); + default: + return stringQuery(propertyName, pr); + } + } + + private interface NumericPoint<T extends Number> { + T convert(PropertyValue pv); + /** The smallest value greater than {@code val}, or {@code null} if there is none. */ + @Nullable T nextAbove(T val); + /** The greatest value smaller than {@code val}, or {@code null} if there is none. */ + @Nullable T nextBelow(T val); + T min(); + T max(); + Query exact(String field, T val); + Query range(String field, T lo, T hi); + Query set(String field, List<PropertyValue> list); + } + + private static final NumericPoint<Long> LONG_POINT = new NumericPoint<>() { + public Long convert(PropertyValue pv) { return pv.getValue(Type.LONG); } + public Long nextAbove(Long v) { return v == Long.MAX_VALUE ? null : v + 1; } + public Long nextBelow(Long v) { return v == Long.MIN_VALUE ? null : v - 1; } + public Long min() { return Long.MIN_VALUE; } + public Long max() { return Long.MAX_VALUE; } + public Query exact(String f, Long v) { return LongPoint.newExactQuery(f, v); } + public Query range(String f, Long lo, Long hi) { return LongPoint.newRangeQuery(f, lo, hi); } + public Query set(String f, List<PropertyValue> list) { + return LongPoint.newSetQuery(f, list.stream().mapToLong(pv -> pv.getValue(Type.LONG)).toArray()); + } + }; + + private static final NumericPoint<Double> DOUBLE_POINT = new NumericPoint<>() { + public Double convert(PropertyValue pv) { return pv.getValue(Type.DOUBLE); } + public Double nextAbove(Double v) { return Math.nextUp(v); } + public Double nextBelow(Double v) { return Math.nextDown(v); } + public Double min() { return -Double.MAX_VALUE; } + public Double max() { return Double.MAX_VALUE; } + public Query exact(String f, Double v) { return DoublePoint.newExactQuery(f, v); } + public Query range(String f, Double lo, Double hi) { return DoublePoint.newRangeQuery(f, lo, hi); } + public Query set(String f, List<PropertyValue> list) { + return DoublePoint.newSetQuery(f, list.stream().mapToDouble(pv -> pv.getValue(Type.DOUBLE)).toArray()); + } + }; + + private static <T extends Number> Query numericQuery(String propertyName, PropertyRestriction pr, + NumericPoint<T> np) { + T first = pr.first != null ? np.convert(pr.first) : null; + T last = pr.last != null ? np.convert(pr.last) : null; + + if (first != null && pr.first.equals(pr.last) && pr.firstIncluding && pr.lastIncluding) { + return np.exact(propertyName, first); + } else if (first != null || last != null) { + T lo = first == null ? np.min() : pr.firstIncluding ? first : np.nextAbove(first); + T hi = last == null ? np.max() : pr.lastIncluding ? last : np.nextBelow(last); + if (lo == null || hi == null) { + return new MatchNoDocsQuery(); + } + return np.range(propertyName, lo, hi); + } else if (pr.list != null) { + return np.set(propertyName, pr.list); + } else if (pr.isNot && pr.not != null) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(new MatchAllDocsQuery(), Occur.MUST); + bq.add(np.exact(propertyName, np.convert(pr.not)), Occur.MUST_NOT); + return bq.build(); + } + throw new IllegalArgumentException("Unsupported property restriction: " + pr); + } + + private static Query dateQuery(String propertyName, PropertyRestriction pr) { + PropertyRestriction longPr = new PropertyRestriction(); + longPr.propertyName = propertyName; + longPr.first = pr.first != null ? PropertyValues.newLong(dateToMillis(pr.first)) : null; + longPr.last = pr.last != null ? PropertyValues.newLong(dateToMillis(pr.last)) : null; + longPr.not = pr.not != null ? PropertyValues.newLong(dateToMillis(pr.not)) : null; + longPr.firstIncluding = pr.firstIncluding; + longPr.lastIncluding = pr.lastIncluding; + longPr.isNot = pr.isNot; + longPr.list = pr.list != null + ? pr.list.stream().map(LuceneNgQueryBuilder::dateToMillis).map(PropertyValues::newLong) + .collect(Collectors.toList()) + : null; + return numericQuery(propertyName, longPr, LONG_POINT); + } + + private static long dateToMillis(PropertyValue pv) { + try { + Calendar date = ISO8601.parse(pv.getValue(Type.DATE)); + if (date == null) { + throw new IllegalArgumentException("Invalid DATE query value"); + } + return date.getTimeInMillis(); + } catch (IllegalArgumentException | IllegalStateException e) { + // Conversion exceptions can contain the input; do not retain them as causes. + LOG.error("Failed to parse DATE query value ({})", e.getClass().getSimpleName()); + throw new IllegalArgumentException("Invalid DATE query value"); + } + } + + private static Query booleanQuery(String propertyName, PropertyRestriction pr) { + if (pr.first != null && pr.first.equals(pr.last)) { + return new TermQuery(new Term(propertyName, pr.first.getValue(Type.BOOLEAN).toString())); + } else if (pr.isNot && pr.not != null) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(new MatchAllDocsQuery(), Occur.MUST); + bq.add(new TermQuery(new Term(propertyName, pr.not.getValue(Type.BOOLEAN).toString())), Occur.MUST_NOT); + return bq.build(); + } + throw new IllegalArgumentException("Unsupported boolean restriction: " + pr); + } + + private static Query stringQuery(String propertyName, PropertyRestriction pr) { + String first = pr.first != null ? pr.first.getValue(Type.STRING) : null; + String last = pr.last != null ? pr.last.getValue(Type.STRING) : null; + + if (pr.isLike) { + return new WildcardQuery(new Term(propertyName, QueryUtils.sqlLikeToLuceneWildcardQuery(first))); + } else if (first != null && pr.first.equals(pr.last) && pr.firstIncluding && pr.lastIncluding) { + return new TermQuery(new Term(propertyName, first)); + } else if (first != null || last != null) { + return new TermRangeQuery(propertyName, + first != null ? new BytesRef(first) : null, last != null ? new BytesRef(last) : null, + first == null || pr.firstIncluding, last == null || pr.lastIncluding); + } else if (pr.list != null) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + for (PropertyValue pv : pr.list) { + bq.add(new TermQuery(new Term(propertyName, pv.getValue(Type.STRING))), Occur.SHOULD); + } + return bq.build(); + } else if (pr.isNot && pr.not != null) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(new MatchAllDocsQuery(), Occur.MUST); + bq.add(new TermQuery(new Term(propertyName, pr.not.getValue(Type.STRING))), Occur.MUST_NOT); + return bq.build(); + } + throw new IllegalArgumentException("Unsupported string restriction: " + pr); + } + + @Nullable + static Query localNameQuery(PropertyRestriction pr) { + if (pr.first != null && pr.first.equals(pr.last) && pr.firstIncluding && pr.lastIncluding) { + return new TermQuery(new Term(FieldNames.NODE_NAME, pr.first.getValue(Type.STRING))); + } + if (pr.isLike && pr.first != null) { + return new WildcardQuery(new Term(FieldNames.NODE_NAME, + QueryUtils.sqlLikeToLuceneWildcardQuery(pr.first.getValue(Type.STRING)))); + } + return null; + } + + private static Query fullTextQuery(FullTextExpression ft, Analyzer analyzer, PlanResult planResult) { + AtomicReference<Query> result = new AtomicReference<>(); + ft.accept(new FullTextVisitor() { + + @Override + public boolean visit(FullTextContains contains) { + return contains.getBase().accept(this); + } + + @Override + public boolean visit(FullTextOr or) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + for (FullTextExpression e : or.list) { + bq.add(fullTextQuery(e, analyzer, planResult), Occur.SHOULD); + } + result.set(bq.build()); + return true; + } + + @Override + public boolean visit(FullTextAnd and) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + for (FullTextExpression e : and.list) { + bq.add(fullTextQuery(e, analyzer, planResult), Occur.MUST); + } + result.set(bq.build()); + return true; + } + + @Override + public boolean visit(FullTextTerm term) { + Query q = termQuery(term.getText(), fullTextField(term.getPropertyName(), planResult), analyzer); + if (term.isNot()) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(new MatchAllDocsQuery(), Occur.MUST); + bq.add(q, Occur.MUST_NOT); + q = bq.build(); + } + String boost = term.getBoost(); + if (boost != null) { + try { + q = new BoostQuery(q, Float.parseFloat(boost)); + } catch (NumberFormatException e) { + LOG.warn("Ignoring unparseable boost value '{}' on fulltext term", boost); + } + } + result.set(q); + return true; + } + }); + return result.get(); + } + + private static String fullTextField(@Nullable String propertyName, PlanResult planResult) { + String name = propertyName != null && planResult.isPathTransformed() + ? PathUtils.getName(propertyName) + : propertyName; + return name == null || "*".equals(name) ? FieldNames.FULLTEXT : FieldNames.createAnalyzedFieldName(name); Review Comment: **`CONTAINS(a.[jcr:content/*], ...)` on a relative-node aggregate searches a field that is never written, so no rows** With an aggregate include `{ path: "jcr:content", relativeNode: true }`, the planner accepts `jcr:content/*` without relativizing the plan. Here the term then becomes the analyzed-property field `full:jcr:content/*`, but the document maker folds aggregated text only into `:fulltext` (`LuceneNgDocumentMaker.indexAggregateValue`), so the field doesn't exist. Repro (index rule `oak:Unstructured`, aggregate `oak:Unstructured → jcr:content, relativeNode=true`; `/content/a/jcr:content` title="hello", `/content/b/jcr:content` title="other"): ```sql SELECT [jcr:path] FROM [oak:Unstructured] AS a WHERE CONTAINS(a.[jcr:content/*], 'hello') ``` | | `EXPLAIN` luceneQuery | rows | |---|---|---| | oak-lucene | `fullnode:jcr:content:hello` | `/content/a` | | luceneNg | `full:jcr:content/*:hello` | none | Legacy writes relative-node aggregates to `FieldNames.createFulltextFieldName(rootIncludePath)` (`fullnode:jcr:content`, `LuceneDocumentMaker.indexAggregateValue`) and maps `x/*` to the same field at query time ([`getLuceneFieldName` L1537-1561](https://github.com/bhabegger/jackrabbit-oak/blob/2bcbbf22eff607b9105c038e02994f9c39ca9b69/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java#L1537-L1561)). Note that mapping `x/*` to `:fulltext` instead would return wrong rows: `:fulltext` also holds the node's own and other aggregated text, and Oak does not re-check `CONTAINS` when a fulltext index answered the query (`FullTextSearchImpl.evaluate`). So the fix should follow legacy: write `fullnode:<path>` for `result.isRelativeNode()` aggregates, and map non-transformed `x/*` terms to `FieldNames.createFulltextFieldName(x)`. ########## oak-search-lucene-ng/src/main/java/org/apache/jackrabbit/oak/plugins/index/luceneNg/internal/LuceneNgQueryBuilder.java: ########## @@ -0,0 +1,615 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.luceneNg.internal; + +import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.api.PropertyValue; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.luceneNg.LuceneNgIndexConstants; +import org.apache.jackrabbit.oak.plugins.index.luceneNg.LuceneNgIndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner.PlanResult; +import org.apache.jackrabbit.oak.plugins.index.search.util.QueryUtils; +import org.apache.jackrabbit.oak.plugins.memory.PropertyValues; +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction; +import org.apache.jackrabbit.oak.spi.query.QueryConstants; +import org.apache.jackrabbit.oak.spi.query.QueryIndex.OrderEntry; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextAnd; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextContains; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextOr; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm; +import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor; +import org.apache.jackrabbit.util.ISO8601; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.BytesRef; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.jcr.PropertyType; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Translates an Oak query plan into the Lucene query and sort executed against a LuceneNg index. + */ +public final class LuceneNgQueryBuilder { + + private static final Logger LOG = LoggerFactory.getLogger(LuceneNgQueryBuilder.class); + + private LuceneNgQueryBuilder() { + } + + /** + * The Lucene query for the filter of a plan. Property restrictions become clauses only when + * the planner matched them to an indexed property definition; the query engine evaluates + * the others. + */ + @NotNull + public static Query query(@NotNull Filter filter, @NotNull PlanResult planResult) { + List<Query> propertyQueries = new ArrayList<>(); + for (PropertyRestriction pr : filter.getPropertyRestrictions()) { + Query propertyQuery = restrictionQuery(pr, planResult); + if (propertyQuery != null) { + propertyQueries.add(propertyQuery); + } + } + + FullTextExpression ft = filter.getFullTextConstraint(); + Query contentQuery; + if (ft != null) { + Query fullTextQuery; + try (Analyzer analyzer = new StandardAnalyzer()) { + fullTextQuery = fullTextQuery(ft, analyzer, planResult); + } + if (propertyQueries.isEmpty()) { + contentQuery = fullTextQuery; + } else { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(fullTextQuery, Occur.MUST); + propertyQueries.forEach(q -> bq.add(q, Occur.MUST)); + contentQuery = bq.build(); + } + } else if (propertyQueries.isEmpty()) { + contentQuery = new MatchAllDocsQuery(); + } else if (propertyQueries.size() == 1) { + contentQuery = propertyQueries.get(0); + } else { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + propertyQueries.forEach(q -> bq.add(q, Occur.MUST)); + contentQuery = bq.build(); + } + + Query pathQuery = pathQuery(filter, planResult); + Query typeQuery = nodeTypeQuery(filter, planResult); + if (pathQuery == null && typeQuery == null) { + return contentQuery; + } + BooleanQuery.Builder combined = new BooleanQuery.Builder(); + combined.add(contentQuery, Occur.MUST); + if (pathQuery != null) { + combined.add(pathQuery, Occur.FILTER); + } + if (typeQuery != null) { + combined.add(typeQuery, Occur.FILTER); + } + return combined.build(); + } + + @Nullable + private static Query restrictionQuery(PropertyRestriction pr, PlanResult planResult) { + String name = pr.propertyName; + if (QueryConstants.REP_FACET.equals(name) || name.startsWith(QueryConstants.FUNCTION_RESTRICTION_PREFIX) + || name.startsWith("rep:") || name.startsWith("oak:")) { + return null; + } + if (QueryConstants.RESTRICTION_LOCAL_NAME.equals(name)) { + return planResult.evaluateNodeNameRestriction() ? localNameQuery(pr) : null; + } + PropertyDefinition definition = planResult.getPropDefn(pr); + if (definition == null) { + return null; + } + String field = planResult.getPropertyName(pr); + return propertyQuery(field, pr, definition, + ((LuceneNgIndexDefinition) planResult.indexDefinition).fieldType(field, definition)); + } + + @Nullable + private static Query nodeTypeQuery(Filter filter, PlanResult result) { + if (filter.matchesAllTypes()) { + return null; + } + BooleanQuery.Builder types = new BooleanQuery.Builder(); + PropertyDefinition primary = result.indexingRule.getConfig(JcrConstants.JCR_PRIMARYTYPE); + if (primary != null && primary.index && primary.propertyIndex) { + for (String type : filter.getPrimaryTypes()) { + types.add(new TermQuery(new Term(JcrConstants.JCR_PRIMARYTYPE, type)), Occur.SHOULD); + } + } + PropertyDefinition mixin = result.indexingRule.getConfig(JcrConstants.JCR_MIXINTYPES); + if (mixin != null && mixin.index && mixin.propertyIndex) { + for (String type : filter.getMixinTypes()) { + types.add(new TermQuery(new Term(JcrConstants.JCR_MIXINTYPES, type)), Occur.SHOULD); + } + } + BooleanQuery query = types.build(); + return query.clauses().isEmpty() ? null : query; + } + + /** + * When the plan evaluates relative properties on descendants (a transformed path), documents + * are those descendants: {@code parentPathSegment} is appended to the restricted path, and + * direct children are the documents of the subtree at the depth of a child plus the depth + * of the relative path. + */ + @Nullable + private static Query pathQuery(Filter filter, PlanResult planResult) { + Filter.PathRestriction restriction = filter.getPathRestriction(); + if (restriction == null) { + return null; + } + String path = filter.getPath(); + boolean transformed = planResult.isPathTransformed(); + String segment = transformed ? planResult.getParentPathSegment() : ""; + if (segment.contains("*")) { + return null; + } + switch (restriction) { + case ALL_CHILDREN: + if ("/".equals(path)) { + return null; + } + return new TermQuery(new Term(FieldNames.ANCESTORS, path)); + case DIRECT_CHILDREN: + if (transformed) { + Query depth = IntPoint.newExactQuery(FieldNames.PATH_DEPTH, + PathUtils.getDepth(path) + planResult.getParentDepth() + 1); + if ("/".equals(path)) { + return depth; + } + return new BooleanQuery.Builder() + .add(new TermQuery(new Term(FieldNames.ANCESTORS, path)), Occur.MUST) + .add(depth, Occur.MUST) + .build(); + } + return new TermQuery(new Term(LuceneNgIndexConstants.FIELD_PARENT_PATH, path)); + case EXACT: + return new TermQuery(new Term(FieldNames.PATH, withSegment(path, segment))); + case PARENT: + if ("/".equals(path)) { + // root has no parent: a path no document has + return new TermQuery(new Term(FieldNames.PATH, "\u0000")); + } + int lastSlash = path.lastIndexOf('/'); + String parentPath = lastSlash == 0 ? "/" : path.substring(0, lastSlash); + return new TermQuery(new Term(FieldNames.PATH, withSegment(parentPath, segment))); + case NO_RESTRICTION: + default: + return null; + } + } + + private static String withSegment(String path, String segment) { + return segment.isEmpty() ? path : "/".equals(path) ? segment : path + segment; + } + + /** + * The Lucene query for a restriction on the indexed field {@code propertyName}, according to + * the declared type of the property. + */ + @NotNull + static Query propertyQuery(@NotNull String propertyName, @NotNull PropertyRestriction pr, + @NotNull PropertyDefinition definition) { + return propertyQuery(propertyName, pr, definition, definition.getType()); + } + + /** + * @param propertyType the type that decides the Lucene field of the property + */ + @NotNull + static Query propertyQuery(@NotNull String propertyName, @NotNull PropertyRestriction pr, + @NotNull PropertyDefinition definition, int propertyType) { + + if (pr.isNotNullRestriction()) { + if (definition.notNullCheckEnabled) { + return new TermQuery(new Term(FieldNames.NOT_NULL_PROPS, definition.name)); + } + switch (propertyType) { + case PropertyType.LONG: + case PropertyType.DATE: + return LongPoint.newRangeQuery(propertyName, Long.MIN_VALUE, Long.MAX_VALUE); + case PropertyType.DOUBLE: + case PropertyType.DECIMAL: + return DoublePoint.newRangeQuery(propertyName, Double.NEGATIVE_INFINITY, Double.NaN); + case PropertyType.UNDEFINED: + BooleanQuery.Builder existence = new BooleanQuery.Builder(); + existence.add(new TermRangeQuery(propertyName, null, null, true, true), Occur.SHOULD); + existence.add(LongPoint.newRangeQuery(propertyName, Long.MIN_VALUE, Long.MAX_VALUE), Occur.SHOULD); + existence.add(DoublePoint.newRangeQuery(propertyName, Double.NEGATIVE_INFINITY, Double.NaN), + Occur.SHOULD); + return existence.build(); + default: + return new TermRangeQuery(propertyName, null, null, true, true); + } + } + + if (pr.isNullRestriction()) { + if (definition.nullCheckEnabled) { + return new TermQuery(new Term(FieldNames.NULL_PROPS, definition.name)); + } + return new MatchAllDocsQuery(); + } + + switch (propertyType) { + case PropertyType.LONG: + return numericQuery(propertyName, pr, LONG_POINT); + case PropertyType.DOUBLE: + case PropertyType.DECIMAL: + return numericQuery(propertyName, pr, DOUBLE_POINT); + case PropertyType.DATE: + return dateQuery(propertyName, pr); + case PropertyType.BOOLEAN: + return booleanQuery(propertyName, pr); + default: + return stringQuery(propertyName, pr); + } + } + + private interface NumericPoint<T extends Number> { + T convert(PropertyValue pv); + /** The smallest value greater than {@code val}, or {@code null} if there is none. */ + @Nullable T nextAbove(T val); + /** The greatest value smaller than {@code val}, or {@code null} if there is none. */ + @Nullable T nextBelow(T val); + T min(); + T max(); + Query exact(String field, T val); + Query range(String field, T lo, T hi); + Query set(String field, List<PropertyValue> list); + } + + private static final NumericPoint<Long> LONG_POINT = new NumericPoint<>() { + public Long convert(PropertyValue pv) { return pv.getValue(Type.LONG); } + public Long nextAbove(Long v) { return v == Long.MAX_VALUE ? null : v + 1; } + public Long nextBelow(Long v) { return v == Long.MIN_VALUE ? null : v - 1; } + public Long min() { return Long.MIN_VALUE; } + public Long max() { return Long.MAX_VALUE; } + public Query exact(String f, Long v) { return LongPoint.newExactQuery(f, v); } + public Query range(String f, Long lo, Long hi) { return LongPoint.newRangeQuery(f, lo, hi); } + public Query set(String f, List<PropertyValue> list) { + return LongPoint.newSetQuery(f, list.stream().mapToLong(pv -> pv.getValue(Type.LONG)).toArray()); + } + }; + + private static final NumericPoint<Double> DOUBLE_POINT = new NumericPoint<>() { + public Double convert(PropertyValue pv) { return pv.getValue(Type.DOUBLE); } + public Double nextAbove(Double v) { return Math.nextUp(v); } + public Double nextBelow(Double v) { return Math.nextDown(v); } + public Double min() { return -Double.MAX_VALUE; } + public Double max() { return Double.MAX_VALUE; } + public Query exact(String f, Double v) { return DoublePoint.newExactQuery(f, v); } + public Query range(String f, Double lo, Double hi) { return DoublePoint.newRangeQuery(f, lo, hi); } + public Query set(String f, List<PropertyValue> list) { + return DoublePoint.newSetQuery(f, list.stream().mapToDouble(pv -> pv.getValue(Type.DOUBLE)).toArray()); + } + }; + + private static <T extends Number> Query numericQuery(String propertyName, PropertyRestriction pr, + NumericPoint<T> np) { + T first = pr.first != null ? np.convert(pr.first) : null; + T last = pr.last != null ? np.convert(pr.last) : null; + + if (first != null && pr.first.equals(pr.last) && pr.firstIncluding && pr.lastIncluding) { + return np.exact(propertyName, first); + } else if (first != null || last != null) { + T lo = first == null ? np.min() : pr.firstIncluding ? first : np.nextAbove(first); + T hi = last == null ? np.max() : pr.lastIncluding ? last : np.nextBelow(last); + if (lo == null || hi == null) { + return new MatchNoDocsQuery(); + } + return np.range(propertyName, lo, hi); + } else if (pr.list != null) { + return np.set(propertyName, pr.list); + } else if (pr.isNot && pr.not != null) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(new MatchAllDocsQuery(), Occur.MUST); + bq.add(np.exact(propertyName, np.convert(pr.not)), Occur.MUST_NOT); + return bq.build(); + } + throw new IllegalArgumentException("Unsupported property restriction: " + pr); + } + + private static Query dateQuery(String propertyName, PropertyRestriction pr) { + PropertyRestriction longPr = new PropertyRestriction(); + longPr.propertyName = propertyName; + longPr.first = pr.first != null ? PropertyValues.newLong(dateToMillis(pr.first)) : null; + longPr.last = pr.last != null ? PropertyValues.newLong(dateToMillis(pr.last)) : null; + longPr.not = pr.not != null ? PropertyValues.newLong(dateToMillis(pr.not)) : null; + longPr.firstIncluding = pr.firstIncluding; + longPr.lastIncluding = pr.lastIncluding; + longPr.isNot = pr.isNot; + longPr.list = pr.list != null + ? pr.list.stream().map(LuceneNgQueryBuilder::dateToMillis).map(PropertyValues::newLong) + .collect(Collectors.toList()) + : null; + return numericQuery(propertyName, longPr, LONG_POINT); + } + + private static long dateToMillis(PropertyValue pv) { + try { + Calendar date = ISO8601.parse(pv.getValue(Type.DATE)); + if (date == null) { + throw new IllegalArgumentException("Invalid DATE query value"); + } + return date.getTimeInMillis(); + } catch (IllegalArgumentException | IllegalStateException e) { + // Conversion exceptions can contain the input; do not retain them as causes. + LOG.error("Failed to parse DATE query value ({})", e.getClass().getSimpleName()); + throw new IllegalArgumentException("Invalid DATE query value"); + } + } + + private static Query booleanQuery(String propertyName, PropertyRestriction pr) { + if (pr.first != null && pr.first.equals(pr.last)) { + return new TermQuery(new Term(propertyName, pr.first.getValue(Type.BOOLEAN).toString())); + } else if (pr.isNot && pr.not != null) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(new MatchAllDocsQuery(), Occur.MUST); + bq.add(new TermQuery(new Term(propertyName, pr.not.getValue(Type.BOOLEAN).toString())), Occur.MUST_NOT); + return bq.build(); + } + throw new IllegalArgumentException("Unsupported boolean restriction: " + pr); + } + + private static Query stringQuery(String propertyName, PropertyRestriction pr) { + String first = pr.first != null ? pr.first.getValue(Type.STRING) : null; + String last = pr.last != null ? pr.last.getValue(Type.STRING) : null; + + if (pr.isLike) { + return new WildcardQuery(new Term(propertyName, QueryUtils.sqlLikeToLuceneWildcardQuery(first))); + } else if (first != null && pr.first.equals(pr.last) && pr.firstIncluding && pr.lastIncluding) { + return new TermQuery(new Term(propertyName, first)); + } else if (first != null || last != null) { + return new TermRangeQuery(propertyName, + first != null ? new BytesRef(first) : null, last != null ? new BytesRef(last) : null, + first == null || pr.firstIncluding, last == null || pr.lastIncluding); + } else if (pr.list != null) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + for (PropertyValue pv : pr.list) { + bq.add(new TermQuery(new Term(propertyName, pv.getValue(Type.STRING))), Occur.SHOULD); + } + return bq.build(); + } else if (pr.isNot && pr.not != null) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(new MatchAllDocsQuery(), Occur.MUST); + bq.add(new TermQuery(new Term(propertyName, pr.not.getValue(Type.STRING))), Occur.MUST_NOT); + return bq.build(); + } + throw new IllegalArgumentException("Unsupported string restriction: " + pr); + } + + @Nullable + static Query localNameQuery(PropertyRestriction pr) { + if (pr.first != null && pr.first.equals(pr.last) && pr.firstIncluding && pr.lastIncluding) { + return new TermQuery(new Term(FieldNames.NODE_NAME, pr.first.getValue(Type.STRING))); + } + if (pr.isLike && pr.first != null) { + return new WildcardQuery(new Term(FieldNames.NODE_NAME, + QueryUtils.sqlLikeToLuceneWildcardQuery(pr.first.getValue(Type.STRING)))); + } + return null; + } + + private static Query fullTextQuery(FullTextExpression ft, Analyzer analyzer, PlanResult planResult) { + AtomicReference<Query> result = new AtomicReference<>(); + ft.accept(new FullTextVisitor() { + + @Override + public boolean visit(FullTextContains contains) { + return contains.getBase().accept(this); + } + + @Override + public boolean visit(FullTextOr or) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + for (FullTextExpression e : or.list) { + bq.add(fullTextQuery(e, analyzer, planResult), Occur.SHOULD); + } + result.set(bq.build()); + return true; + } + + @Override + public boolean visit(FullTextAnd and) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + for (FullTextExpression e : and.list) { + bq.add(fullTextQuery(e, analyzer, planResult), Occur.MUST); + } + result.set(bq.build()); + return true; + } + + @Override + public boolean visit(FullTextTerm term) { + Query q = termQuery(term.getText(), fullTextField(term.getPropertyName(), planResult), analyzer); + if (term.isNot()) { + BooleanQuery.Builder bq = new BooleanQuery.Builder(); + bq.add(new MatchAllDocsQuery(), Occur.MUST); + bq.add(q, Occur.MUST_NOT); + q = bq.build(); + } + String boost = term.getBoost(); + if (boost != null) { + try { + q = new BoostQuery(q, Float.parseFloat(boost)); + } catch (NumberFormatException e) { + LOG.warn("Ignoring unparseable boost value '{}' on fulltext term", boost); + } + } + result.set(q); + return true; + } + }); + return result.get(); + } + + private static String fullTextField(@Nullable String propertyName, PlanResult planResult) { + String name = propertyName != null && planResult.isPathTransformed() + ? PathUtils.getName(propertyName) + : propertyName; + return name == null || "*".equals(name) ? FieldNames.FULLTEXT : FieldNames.createAnalyzedFieldName(name); + } + + private static Query termQuery(String text, String field, Analyzer analyzer) { + + // Wildcard terms bypass tokenization to preserve the wildcard characters + if (text.contains("*") || text.contains("?")) { + String lower = text.toLowerCase(Locale.ENGLISH); + if (lower.indexOf('*') == lower.length() - 1 && !lower.contains("?")) { + return new PrefixQuery(new Term(field, lower.substring(0, lower.length() - 1))); + } + return new WildcardQuery(new Term(field, lower)); + } + + List<String> tokens = tokenize(text, analyzer); + if (tokens.isEmpty()) { + return new BooleanQuery.Builder().build(); + } + if (tokens.size() == 1) { + return new TermQuery(new Term(field, tokens.get(0))); + } + PhraseQuery.Builder pq = new PhraseQuery.Builder(); + for (String token : tokens) { + pq.add(new Term(field, token)); + } + return pq.build(); + } + + static List<String> tokenize(String text, Analyzer analyzer) { + List<String> tokens = new ArrayList<>(); + try (TokenStream stream = analyzer.tokenStream(FieldNames.FULLTEXT, new StringReader(text))) { + CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class); + stream.reset(); + while (stream.incrementToken()) { + tokens.add(termAtt.toString()); + } + stream.end(); + } catch (IOException e) { + LOG.error("Failed to tokenize fulltext query ({})", e.getClass().getSimpleName()); + throw new IllegalStateException("Failed to tokenize fulltext query"); + } + return tokens; + } + + /** + * The Lucene sort for the order entries of a plan. + * + * @param docValuesTypes doc-values type of each indexed field, {@link DocValuesType#NONE} + * for fields without doc values + */ + @NotNull + public static Sort sort(@NotNull List<OrderEntry> sortOrder, @NotNull LuceneNgIndexDefinition definition, + @NotNull Function<String, DocValuesType> docValuesTypes) { + List<SortField> fields = new ArrayList<>(); + for (OrderEntry order : sortOrder) { + fields.add(sortField(order, definition, docValuesTypes)); + } + return new Sort(fields.toArray(new SortField[0])); + } + + /** + * Ordered values are written as SORTED_SET (strings) or SORTED_NUMERIC (numbers) doc values + * for single- and multi-valued properties alike; these sort each document by its smallest value. + */ + static SortField sortField(OrderEntry order, LuceneNgIndexDefinition definition, + Function<String, DocValuesType> docValuesTypes) { + String propertyName = order.getPropertyName(); + if (QueryConstants.JCR_SCORE.equals(propertyName)) { + return SortField.FIELD_SCORE; + } + SortField.Type fieldType = sortFieldType(declaredType(definition, propertyName, order.getPropertyType().tag())); + boolean reverse = order.getOrder() == OrderEntry.Order.DESCENDING; + DocValuesType docValuesType = docValuesTypes.apply(propertyName); + if (docValuesType == DocValuesType.SORTED_SET) { + return new SortedSetSortField(propertyName, reverse); + } + if (docValuesType == DocValuesType.SORTED_NUMERIC) { + return new SortedNumericSortField(propertyName, fieldType, reverse); + } + return new SortField(propertyName, fieldType, reverse); Review Comment: **`ORDER BY` a numeric property: nodes without the property sort as `0`** The numeric sort fields have no missing value, so Lucene uses `0`. The query engine sorts a missing value first ascending and last descending, and it doesn't re-sort when the index handles the sort, so the order is wrong. Repro (index rule `nt:base → size (propertyIndex, type=Long, ordered), category (propertyIndex)`; `a` size=-5, `b` size=7, `c` no size, all category="x"): ```sql SELECT [jcr:path] FROM [nt:base] WHERE [category] = 'x' ORDER BY [size] ``` | | ascending | descending | |---|---|---| | oak-lucene | `c, a, b` | `b, a, c` | | luceneNg | `a, c, b` | | Legacy sets the missing value explicitly ([`LucenePropertyIndex` L833-839](https://github.com/bhabegger/jackrabbit-oak/blob/2bcbbf22eff607b9105c038e02994f9c39ca9b69/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java#L833-L839), toggle `FT_OAK-12344`): `Long.MIN_VALUE` for LONG, `Double.NEGATIVE_INFINITY` for DOUBLE. Suggested fix: the same on both the `SortedNumericSortField` and the plain `SortField` branch (Date and Decimal included); with `reverse` the missing value then sorts last. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
