GMishx opened a new issue, #6075:
URL: https://github.com/apache/couchdb/issues/6075
### Version
3.5.1
### Describe the problem you're encountering
I have a nouveau index created which is working as expected. But as soon as
I provide a sorting by relevance, the results are flipped and this behavior is
confusing me.
If I do not provide any sort, the results are returned sorted by the actual
relevance score. When I ask Nouveau to sort the results by score actually, the
results are in non-clear sort order.
### Expected Behaviour
Following queries should return same response.
1. Without sort provided:
```
$ curl -X POST -H 'Content-Type: application/json'
"http://localhost:5984/sw360db/_design/lucene/_nouveau/projects"; -d
"{\"include_docs\" : false, \"limit\" : 1000, \"q\" : \"(name_exact:\\\"test
project\\\"^100 OR (name_exact:test AND name_ngram:project))\" }" | jq
{
"update_latency": 0,
"total_hits_relation": "EQUAL_TO",
"total_hits": 3,
"ranges": {},
"hits": [
{
"order": [
{
"value": 131.96309,
"@type": "float"
},
{
"value": "a49220d3e1ef4001b98e04b22d3f846b",
"@type": "string"
}
],
"id": "a49220d3e1ef4001b98e04b22d3f846b",
"fields": {}
},
{
"order": [
{
"value": 131.9439,
"@type": "float"
},
{
"value": "c88127c3b2d94b59aec712b7b73b026a",
"@type": "string"
}
],
"id": "c88127c3b2d94b59aec712b7b73b026a",
"fields": {}
},
{
"order": [
{
"value": 119.92493,
"@type": "float"
},
{
"value": "e81b1b0b54f640788f538c689ec7ddfa",
"@type": "string"
}
],
"id": "e81b1b0b54f640788f538c689ec7ddfa",
"fields": {}
}
],
"counts": {},
"bookmark":
"W1t7InZhbHVlIjoxMTkuOTI0OTMsIkB0eXBlIjoiZmxvYXQifSx7InZhbHVlIjoiZTgxYjFiMGI1NGY2NDA3ODhmNTM4YzY4OWVjN2RkZmEiLCJAdHlwZSI6InN0cmluZyJ9XSxbeyJ2YWx1ZSI6MTMxLjk0MzksIkB0eXBlIjoiZmxvYXQifSx7InZhbHVlIjoiYzg4MTI3YzNiMmQ5NGI1OWFlYzcxMmI3YjczYjAyNmEiLCJAdHlwZSI6InN0cmluZyJ9XV0"
}
```
2. Using `relevance` as the sort (because I want to mix with `name_sort`
later), the first result is lowest score, second with highest score and third
is the second highest score:
```
$ curl -X POST -H 'Content-Type: application/json'
"http://localhost:5984/sw360db/_design/lucene/_nouveau/projects"; -d
"{\"include_docs\" : false, \"limit\" : 1000, \"q\" : \"(name_exact:\\\"test
project\\\"^100 OR (name_exact:test AND name_ngram:project))\", \"sort\":
[\"relevance\"] }" | jq
{
"update_latency": 0,
"total_hits_relation": "EQUAL_TO",
"total_hits": 3,
"ranges": {},
"hits": [
{
"order": [
{
"value": 119.92493,
"@type": "float"
},
{
"value": "e81b1b0b54f640788f538c689ec7ddfa",
"@type": "string"
}
],
"id": "e81b1b0b54f640788f538c689ec7ddfa",
"fields": {}
},
{
"order": [
{
"value": 131.96309,
"@type": "float"
},
{
"value": "a49220d3e1ef4001b98e04b22d3f846b",
"@type": "string"
}
],
"id": "a49220d3e1ef4001b98e04b22d3f846b",
"fields": {}
},
{
"order": [
{
"value": 131.9439,
"@type": "float"
},
{
"value": "c88127c3b2d94b59aec712b7b73b026a",
"@type": "string"
}
],
"id": "c88127c3b2d94b59aec712b7b73b026a",
"fields": {}
}
],
"counts": {},
"bookmark":
"W1t7InZhbHVlIjoxMTkuOTI0OTMsIkB0eXBlIjoiZmxvYXQifSx7InZhbHVlIjoiZTgxYjFiMGI1NGY2NDA3ODhmNTM4YzY4OWVjN2RkZmEiLCJAdHlwZSI6InN0cmluZyJ9XSxbeyJ2YWx1ZSI6MTMxLjk0MzksIkB0eXBlIjoiZmxvYXQifSx7InZhbHVlIjoiYzg4MTI3YzNiMmQ5NGI1OWFlYzcxMmI3YjczYjAyNmEiLCJAdHlwZSI6InN0cmluZyJ9XV0"
}
```
### Steps to Reproduce
**Nouveau index function used:**
```json
{
"_id": "_design/lucene",
"_rev": "3819-ab590d5a40424f987e7bb4ca16867ff6",
"nouveau": {
"projects": {
"default_analyzer": "standard",
"field_analyzers": {
"attachmentCreatedBy": "email",
"version_ngram": "whitespace",
"projectResponsible": "email",
"clearingState": "keyword",
"version": "keyword",
"state": "keyword",
"name_ngram": "whitespace",
"tag_ngram": "whitespace"
},
"index": "function(doc) { function emitEdgeNGrams(fieldName, text,
minGram, maxGram) {\n if (!text) return;\n var words =
text.toLowerCase().split(/\\\\s+/);\n for (var i = 0; i < words.length; i++)
{\n var word = words[i];\n var limit = Math.min(word.length,
maxGram);\n for (var len = minGram; len <= limit; len++) {\n
index('text', fieldName, word.substring(0, len));\n }\n }\n }\n
function arrayToStringIndex(arr, indexName) { let result = ''; for (let i
in arr) { if (arr[i] && typeof(arr[i]) == 'string' && arr[i].length > 0) {
result += ' ' + arr[i]; } } if (result.trim().length > 0) {
index('text', indexName, result.trim(), {'store': true});
index('string', indexName + '_sort', result.trim()); } } if(!doc.type ||
doc.type != 'project') return; var businessUnit = '__EMPTY__';
if(doc.businessUnit !== undefined && doc.businessUnit != null &&
doc.businessUnit.length > 0) { busines
sUnit = doc.businessUnit; } index('string', 'businessUnit',
businessUnit); if(doc.projectType !== undefined && doc.projectType != null
&& doc.projectType.length >0) { index('text', 'projectType',
doc.projectType); index('string', 'projectType_sort',
doc.projectType.toLowerCase()); } if(doc.projectResponsible !== undefined
&& doc.projectResponsible != null && doc.projectResponsible.length >0) {
index('text', 'projectResponsible', doc.projectResponsible);
index('string', 'projectResponsible_sort', doc.projectResponsible); }
if(doc.name !== undefined && doc.name != null && doc.name.length >0) {
index('text', 'name_exact', doc.name); emitEdgeNGrams('name_ngram',
doc.name, 2, 25); index('string', 'name_sort', doc.name.toLowerCase());
} if(doc.description !== undefined && doc.description != null &&
doc.description.length >0) { index('text', 'description',
doc.description); index('string', 'description_sort', doc.descr
iption); } if(doc.version !== undefined && doc.version != null &&
doc.version.length >0) { index('text', 'version_exact', doc.version);
emitEdgeNGrams('version_ngram', doc.version, 2, 25); index('string',
'version_sort', doc.version.toLowerCase()); } if(doc.state !== undefined
&& doc.state != null && doc.state.length >0) { index('text', 'state',
doc.state); index('string', 'state_sort', doc.state); }
if(doc.clearingState) { index('text', 'clearingState', doc.clearingState);
} var tag = '__EMPTY__'; if(doc.tag !== undefined && doc.tag != null
&& doc.tag.length > 0) { tag = doc.tag; } index('text', 'tag_exact',
tag); emitEdgeNGrams('tag_ngram', tag, 2, 25); index('string',
'tag_sort', tag.toLowerCase()); arrayToStringIndex(doc.additionalData,
'additionalData'); if(doc.releaseRelationNetwork !== undefined &&
doc.releaseRelationNetwork != null && doc.releaseRelationNetwork.length > 0) {
index('text', 'r
eleaseRelationNetwork', doc.releaseRelationNetwork, {'store': true}); }
if(doc.createdOn && doc.createdOn.length) { var dt = new
Date(doc.createdOn); var formattedDt =
`${dt.getFullYear()}${(dt.getMonth()+1).toString().padStart(2,'0')}${dt.getDate().toString().padStart(2,'0')}`;
index('double', 'createdOn', Number(formattedDt)); }
if(doc.attachments && doc.attachments.length > 0) { for(var i in
doc.attachments) { if(doc.attachments[i].createdBy) {
index('text', 'attachmentCreatedBy', doc.attachments[i].createdBy); }
} }}",
"lucene_version": 10
}
}
```
To simplify, I am using following indexes:
1. `index('text', 'name_exact', doc.name);`, For exact matches
2. `emitEdgeNGrams('name_ngram', doc.name, 2, 25);`, Precomputed n-grams
[2,25] to prevent using `proj*` while searching.
3. `index('string', 'name_sort', doc.name.toLowerCase());`, String, used
purely for sorting
---
**Query:**
```json
{
"include_docs": false,
"limit": 1000,
"q": "(name_exact:\"test project\"^100 OR (name_exact:test AND
name_ngram:project))",
"sort": ["name_sort"]
}
```
### Your Environment
curl -s "http://localhost:5984";
{"couchdb":"Welcome","version":"3.5.2","git_sha":"5b4d92103","uuid":"deb34a557c40d5f658d42ca2c78872e1","features":["nouveau","access-ready","partitioned","pluggable-storage-engines","reshard","scheduler"],"vendor":{"name":"The
Apache Software Foundation"}}
### Additional Context
My question basically is: Can I rely on the sorting provided by the nouveau
result?
The use-case here is to show the results with score sorted (descending) so
they find the best result on top. Once results are loaded in application, they
the ability to sort by field (`name_sort` here for example) and then use the
score as the tiebreaker.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
