This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new d09f65d0163 Review and tidy solr/modules/langid code (#4835)
d09f65d0163 is described below
commit d09f65d0163ec0038bf8ae091e4214418837710d
Author: Eric Pugh <[email protected]>
AuthorDate: Tue Sep 15 14:42:21 2026 -0400
Review and tidy solr/modules/langid code (#4835)
---
...angDetectLanguageIdentifierUpdateProcessor.java | 3 +--
.../LanguageIdentifierUpdateProcessor.java | 28 +++++++++-------------
.../OpenNLPLangDetectUpdateProcessor.java | 2 +-
.../update/processor/SolrInputDocumentReader.java | 14 +++++------
...penNLPLangDetectUpdateProcessorFactoryTest.java | 2 +-
.../processor/SolrInputDocumentReaderTest.java | 2 +-
6 files changed, 22 insertions(+), 29 deletions(-)
diff --git
a/solr/modules/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java
b/solr/modules/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java
index 7d4522edc76..183f66f00ec 100644
---
a/solr/modules/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java
+++
b/solr/modules/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessor.java
@@ -65,8 +65,7 @@ public class LangDetectLanguageIdentifierUpdateProcessor
extends LanguageIdentif
List<Language> langlist = orchestrator.detectAll(text);
ArrayList<DetectedLanguage> solrLangList = new ArrayList<>();
for (Language l : langlist) {
- solrLangList.add(
- new DetectedLanguage(l.getIsoCode639_1().toString(), (double)
l.getProbability()));
+ solrLangList.add(new DetectedLanguage(l.getIsoCode639_1(), (double)
l.getProbability()));
}
if (solrLangList.isEmpty()) {
log.debug("Could not determine language, returning empty list");
diff --git
a/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
b/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
index 6ee384bbaee..2b7f60a845d 100644
---
a/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
+++
b/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
@@ -80,7 +80,6 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
protected int maxTotalChars;
// Regex patterns
- protected final Pattern tikaSimilarityPattern =
Pattern.compile(".*\\((.*?)\\)");
protected final Pattern langPattern = Pattern.compile("\\{lang\\}");
public LanguageIdentifierUpdateProcessor(
@@ -95,7 +94,7 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
if (params != null) {
// Document-centric langId params
setEnabled(params.getBool(LANGUAGE_ID, true));
- if (params.get(FIELDS_PARAM, "").length() > 0) {
+ if (!params.get(FIELDS_PARAM, "").isEmpty()) {
inputFields = params.get(FIELDS_PARAM, "").split(",");
}
langField = params.get(LANG_FIELD, DOCID_LANGFIELD_DEFAULT);
@@ -105,7 +104,7 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
params.get(
DOCID_PARAM, uniqueKeyField == null ? DOCID_FIELD_DEFAULT :
uniqueKeyField.getName());
fallbackValue = params.get(FALLBACK);
- if (params.get(FALLBACK_FIELDS, "").length() > 0) {
+ if (!params.get(FALLBACK_FIELDS, "").isEmpty()) {
fallbackFields = params.get(FALLBACK_FIELDS).split(",");
}
overwrite = params.getBool(OVERWRITE, false);
@@ -119,7 +118,7 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
// Mapping params (field centric)
enableMapping = params.getBool(MAP_ENABLE, false);
- if (params.get(MAP_FL, "").length() > 0) {
+ if (!params.get(MAP_FL, "").isEmpty()) {
mapFields = params.get(MAP_FL, "").split(",");
} else {
mapFields = inputFields;
@@ -129,8 +128,8 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
mapIndividual = params.getBool(MAP_INDIVIDUAL, false);
// Process individual fields
- String[] mapIndividualFields = {};
- if (params.get(MAP_INDIVIDUAL_FL, "").length() > 0) {
+ String[] mapIndividualFields;
+ if (!params.get(MAP_INDIVIDUAL_FL, "").isEmpty()) {
mapIndividualFields = params.get(MAP_INDIVIDUAL_FL, "").split(",");
} else {
mapIndividualFields = mapFields;
@@ -225,7 +224,7 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
* @param doc the SolrInputDocument to modify
*/
protected void process(SolrInputDocument doc) {
- String docLang = null;
+ String docLang;
HashSet<String> docLangs = new HashSet<>();
String fallbackLang = getFallbackLang(doc, fallbackFields, fallbackValue);
@@ -247,7 +246,7 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
log.debug("Overwritten old value {}", doc.getFieldValue(langField));
}
}
- if (langField != null && langField.length() != 0) {
+ if (langField != null && !langField.isEmpty()) {
doc.setField(langField, docLang);
}
} else {
@@ -297,7 +296,7 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
}
// Set the languages field to an array of all detected languages
- if (langsField != null && langsField.length() != 0) {
+ if (langsField != null && !langsField.isEmpty()) {
doc.setField(langsField, docLangs.toArray());
}
}
@@ -367,7 +366,7 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
*/
protected String resolveLanguage(List<DetectedLanguage> languages, String
fallbackLang) {
String langStr;
- if (languages.size() == 0) {
+ if (languages.isEmpty()) {
log.debug("No language detected, using fallback {}", fallbackLang);
langStr = fallbackLang;
} else {
@@ -395,7 +394,7 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
}
}
- if (langStr == null || langStr.length() == 0) {
+ if (langStr == null || langStr.isEmpty()) {
log.warn("Language resolved to null or empty string. Fallback not
configured?");
langStr = "";
}
@@ -429,7 +428,7 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
* @return The new schema field name, based on pattern and replace, or null
if illegal
*/
protected String getMappedField(String currentField, String language) {
- String lc = mapLcMap.containsKey(language) ? mapLcMap.get(language) :
language;
+ String lc = mapLcMap.getOrDefault(language, language);
String newFieldName =
langPattern
.matcher(mapPattern.matcher(currentField).replaceFirst(mapReplaceStr))
@@ -474,9 +473,4 @@ public abstract class LanguageIdentifierUpdateProcessor
extends UpdateRequestPro
protected SolrInputDocumentReader solrDocReader(SolrInputDocument doc,
String[] fields) {
return new SolrInputDocumentReader(doc, fields, maxTotalChars,
maxFieldValueChars, " ");
}
-
- /** Concatenates content from input fields defined in langid.fl. For test
purposes only */
- protected String concatFields(SolrInputDocument doc) {
- return SolrInputDocumentReader.asString(solrDocReader(doc, inputFields));
- }
}
diff --git
a/solr/modules/langid/src/java/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessor.java
b/solr/modules/langid/src/java/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessor.java
index 76b79d9f0a3..67094a13632 100644
---
a/solr/modules/langid/src/java/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessor.java
+++
b/solr/modules/langid/src/java/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessor.java
@@ -59,7 +59,7 @@ public class OpenNLPLangDetectUpdateProcessor extends
LanguageIdentifierUpdatePr
protected List<DetectedLanguage> detectLanguage(Reader solrDocReader) {
List<DetectedLanguage> languages = new ArrayList<>();
String content = SolrInputDocumentReader.asString(solrDocReader);
- if (content.length() != 0) {
+ if (!content.isEmpty()) {
LanguageDetectorME ldme = new LanguageDetectorME(model);
Language[] langs = ldme.predictLanguages(content);
for (Language language : langs) {
diff --git
a/solr/modules/langid/src/java/org/apache/solr/update/processor/SolrInputDocumentReader.java
b/solr/modules/langid/src/java/org/apache/solr/update/processor/SolrInputDocumentReader.java
index bfef13053d1..28bf12b0efb 100644
---
a/solr/modules/langid/src/java/org/apache/solr/update/processor/SolrInputDocumentReader.java
+++
b/solr/modules/langid/src/java/org/apache/solr/update/processor/SolrInputDocumentReader.java
@@ -114,7 +114,7 @@ public class SolrInputDocumentReader extends Reader {
nextDocChunk(sb, targetLen);
}
- if (sb.length() == 0) {
+ if (sb.isEmpty()) {
eod = true;
return eodReturnValue;
} else {
@@ -133,7 +133,7 @@ public class SolrInputDocumentReader extends Reader {
do {
SolrInputField f = doc.getField(fields[currentFieldIdx]);
if (f == null) {
- log.debug("Field with name {} did not exist on docuemnt.",
fields[currentFieldIdx]);
+ log.debug("Field with name {} did not exist on document.",
fields[currentFieldIdx]);
incField(sb);
continue;
}
@@ -144,9 +144,9 @@ public class SolrInputDocumentReader extends Reader {
String fvStr = String.valueOf(fvIt.next());
if (currentFieldValueIdx < startFieldValueIdx) continue;
startFieldValueIdx = 0;
- if (sb.length() > 0) {
+ if (!sb.isEmpty()) {
if (maxChunkLength - sb.length() < fieldValueSep.length()) {
- sb.append(fieldValueSep.substring(0, maxChunkLength -
sb.length()));
+ sb.append(fieldValueSep, 0, maxChunkLength - sb.length());
} else {
sb.append(fieldValueSep);
}
@@ -161,7 +161,7 @@ public class SolrInputDocumentReader extends Reader {
if (endOffset - currentFieldValueOffset > maxCharsPerFieldValue) {
endOffset = maxCharsPerFieldValue - currentFieldValueOffset;
}
- sb.append(fvStr.substring(currentFieldValueOffset, endOffset));
+ sb.append(fvStr, currentFieldValueOffset, endOffset);
currentFieldValueOffset = endOffset == fvStr.length() ? 0 : endOffset;
}
if (sb.length() >= maxChunkLength) {
@@ -170,7 +170,7 @@ public class SolrInputDocumentReader extends Reader {
incField(sb);
}
} while (currentFieldIdx <= fields.length - 1 && sb.length() <
maxChunkLength);
- return sb.length() == 0 ? eodReturnValue : sb.length();
+ return sb.isEmpty() ? eodReturnValue : sb.length();
}
private int returnEod() {
@@ -179,7 +179,7 @@ public class SolrInputDocumentReader extends Reader {
}
private int returnValue(StringBuilder sb) {
- if (sb.length() == 0) {
+ if (sb.isEmpty()) {
return returnEod();
} else {
return sb.length();
diff --git
a/solr/modules/langid/src/test/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessorFactoryTest.java
b/solr/modules/langid/src/test/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessorFactoryTest.java
index b1b960a8326..6e3c086f391 100644
---
a/solr/modules/langid/src/test/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessorFactoryTest.java
+++
b/solr/modules/langid/src/test/org/apache/solr/update/processor/OpenNLPLangDetectUpdateProcessorFactoryTest.java
@@ -24,7 +24,7 @@ import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.request.SolrQueryRequest;
import org.junit.Test;
-@ThreadLeakLingering(linger = 0)
+@ThreadLeakLingering()
public class OpenNLPLangDetectUpdateProcessorFactoryTest
extends LanguageIdentifierUpdateProcessorFactoryTestCase {
private static final String TEST_MODEL =
"opennlp-langdetect.eng-swe-spa-rus-deu.bin";
diff --git
a/solr/modules/langid/src/test/org/apache/solr/update/processor/SolrInputDocumentReaderTest.java
b/solr/modules/langid/src/test/org/apache/solr/update/processor/SolrInputDocumentReaderTest.java
index 1a55cd2d0c8..f560f58d3fa 100644
---
a/solr/modules/langid/src/test/org/apache/solr/update/processor/SolrInputDocumentReaderTest.java
+++
b/solr/modules/langid/src/test/org/apache/solr/update/processor/SolrInputDocumentReaderTest.java
@@ -78,7 +78,7 @@ public class SolrInputDocumentReaderTest extends SolrTestCase
{
}
@Test
- public void testGetStringFields() throws Exception {
+ public void testGetStringFields() {
String[] expected = new String[] {"f1", "f2", "f4"};
assertArrayEquals(expected, SolrInputDocumentReader.getStringFields(doc));
}
