This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new bd2b2d5a025 SOLR-9957: /export supports document transformers in fl
(#5113)
bd2b2d5a025 is described below
commit bd2b2d5a02584893c5b5fe16c1a7eee89c2bb6c9
Author: David Smiley <[email protected]>
AuthorDate: Tue Oct 20 21:49:08 2026 +0100
SOLR-9957: /export supports document transformers in fl (#5113)
Albeit some might not work.
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
---
.../unreleased/SOLR-9957-exportTransformers.yml | 10 ++
.../apache/solr/handler/export/ExportWriter.java | 153 +++++++++++++++++----
.../solr/handler/export/ExportWriterStream.java | 13 +-
.../solr/handler/export/TestExportWriter.java | 69 ++++++++++
.../query-guide/pages/exporting-result-sets.adoc | 4 +
5 files changed, 208 insertions(+), 41 deletions(-)
diff --git a/changelog/unreleased/SOLR-9957-exportTransformers.yml
b/changelog/unreleased/SOLR-9957-exportTransformers.yml
new file mode 100644
index 00000000000..aa37e8f4183
--- /dev/null
+++ b/changelog/unreleased/SOLR-9957-exportTransformers.yml
@@ -0,0 +1,10 @@
+title: >
+ The /export handler now supports document transformers (e.g. `[docid]`) and
field aliases in `fl`,
+ and their output is visible to its local streaming expression (`expr`). An
aliased field such as
+ `fl=x:myfield` is now returned as `x`; previously it was silently returned
as `myfield`.
+type: added
+authors:
+ - name: David Smiley
+links:
+ - name: SOLR-9957
+ url: https://issues.apache.org/jira/browse/SOLR-9957
diff --git
a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
index 7b82ad28b14..7872a61be1f 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -26,6 +26,7 @@ import java.io.PrintWriter;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -36,6 +37,7 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BitSetIterator;
@@ -52,10 +54,12 @@ import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.MapWriter.EntryWriter;
import org.apache.solr.common.PushWriter;
+import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StreamParams;
import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrCore;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrQueryRequest;
@@ -64,7 +68,9 @@ import org.apache.solr.response.JSONResponseWriter;
import org.apache.solr.response.JacksonJsonWriter;
import org.apache.solr.response.JavaBinResponseWriter;
import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.schema.BoolField;
import org.apache.solr.schema.DateValueFieldType;
import org.apache.solr.schema.DoubleValueFieldType;
@@ -76,7 +82,10 @@ import org.apache.solr.schema.LongValueFieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.SortableTextField;
import org.apache.solr.schema.StrField;
+import org.apache.solr.search.DocIterationInfo;
+import org.apache.solr.search.DocList;
import org.apache.solr.search.DocValuesIteratorCache;
+import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.search.SortSpec;
@@ -124,7 +133,9 @@ public class ExportWriter implements SolrCore.RawWriter,
Closeable {
private int priorityQueueSize;
StreamExpression streamExpression;
StreamContext streamContext;
+ SolrReturnFields returnFields;
List<FieldWriter> fieldWriters;
+ DocTransformer transformer;
int totalHits = 0;
FixedBitSet[] sets = null;
PushWriter writer;
@@ -268,35 +279,38 @@ public class ExportWriter implements SolrCore.RawWriter,
Closeable {
}
}
SolrParams params = req.getParams();
- String fl = params.get("fl");
- String[] fields = null;
-
- if (fl == null) {
+ if (params.get(CommonParams.FL) == null) {
writeException(
(new IOException(new SyntaxError("export field list (fl) must be
specified."))),
writer,
true);
return;
- } else {
- fields = fl.split(",");
-
- for (int i = 0; i < fields.length; i++) {
+ }
- fields[i] = fields[i].trim();
+ try {
+ returnFields = new SolrReturnFields(req);
+ } catch (Exception e) {
+ writeException(e, writer, true);
+ return;
+ }
- if (fields[i].equals("score")) {
- writeException(
- (new IOException(new SyntaxError("Scoring is not currently
supported with xsort."))),
- writer,
- true);
- return;
- }
- }
+ if (returnFields.wantsScore()) {
+ writeException(
+ (new IOException(new SyntaxError("Scoring is not currently supported
with xsort."))),
+ writer,
+ true);
+ return;
}
try {
- fieldWriters = getFieldWriters(fields, req);
+ fieldWriters = getFieldWriters(returnFields, req);
+ transformer = returnFields.getTransformer();
+ if (transformer != null) {
+ transformer.setContext(
+ new ExportResultContext(
+ returnFields, searcher, info.getResponseBuilder().getQuery(),
req));
+ }
} catch (Exception e) {
writeException(e, writer, true);
return;
@@ -483,33 +497,73 @@ public class ExportWriter implements SolrCore.RawWriter,
Closeable {
throws IOException {
int ord = sortDoc.ord;
LeafReaderContext context = leaves.get(ord);
+ if (transformer == null) {
+ for (FieldWriter fieldWriter : writers) {
+ fieldWriter.write(sortDoc, context, ew);
+ }
+ return;
+ }
+
+ SolrDocument doc = new SolrDocument();
+ EntryWriter docWriter =
+ new EntryWriter() {
+ @Override
+ public EntryWriter put(CharSequence k, Object v) {
+ doc.setField(k.toString(), materialize(v));
+ return this;
+ }
+ };
for (FieldWriter fieldWriter : writers) {
- fieldWriter.write(sortDoc, context, ew);
+ fieldWriter.write(sortDoc, context, docWriter);
+ }
+ transformer.transform(doc, sortDoc.docBase + sortDoc.docId,
DocIterationInfo.NONE);
+ for (Map.Entry<String, Object> entry : doc) {
+ if (returnFields.wantsField(entry.getKey())) {
+ ew.put(entry.getKey(), entry.getValue());
+ }
}
}
- public List<FieldWriter> getFieldWriters(String[] fields, SolrQueryRequest
req)
+ /** Converts lazily written values to plain Java objects that can outlive
the write call. */
+ static Object materialize(Object v) {
+ if (v instanceof IteratorWriter iteratorWriter) {
+ List<Object> lst = new ArrayList<>();
+ iteratorWriter.toList(lst);
+ return lst;
+ } else if (v instanceof MapWriter mapWriter) {
+ return new SimpleOrderedMap<>(mapWriter);
+ }
+ return v;
+ }
+
+ List<FieldWriter> getFieldWriters(SolrReturnFields solrReturnFields,
SolrQueryRequest req)
throws IOException {
DocValuesIteratorCache dvIterCache = new
DocValuesIteratorCache(req.getSearcher(), false);
- SolrReturnFields solrReturnFields = new SolrReturnFields(fields, req);
boolean includeStoredFields =
req.getParams().getBool(INCLUDE_STORED_FIELDS_PARAM, false);
+ // Also fetch fields only a transformer reads, e.g. the f= of [geo];
wantsField excludes them
+ Set<String> fieldsToFetch =
+ solrReturnFields.getLuceneFieldNames() != null
+ ? solrReturnFields.getLuceneFieldNames()
+ : Set.of();
+ // Non-exportable fields named here error; ones matched only by a glob are
skipped.
+ // Includes fieldsToFetch, since requested names hold an alias, not its
source field.
+ Set<String> explicitlyNamedFields = new HashSet<>(fieldsToFetch);
+ if (solrReturnFields.getRequestedFieldNames() != null) {
+ explicitlyNamedFields.addAll(solrReturnFields.getRequestedFieldNames());
+ }
+
List<FieldWriter> writers = new ArrayList<>();
Set<SchemaField> docValueFields = new LinkedHashSet<>();
Map<String, SchemaField> storedFields = new LinkedHashMap<>();
for (String field : req.getSearcher().getFieldNames()) {
- if (!solrReturnFields.wantsField(field)) {
+ if (!solrReturnFields.wantsField(field) &&
!fieldsToFetch.contains(field)) {
continue;
}
SchemaField schemaField = req.getSchema().getField(field);
FieldType fieldType = schemaField.getType();
- Set<String> requestFieldNames =
- solrReturnFields.getRequestedFieldNames() == null
- ? Set.of()
- : solrReturnFields.getRequestedFieldNames();
-
if (canUseDocValues(schemaField, fieldType)) {
// Prefer DocValues when available
docValueFields.add(schemaField);
@@ -517,7 +571,7 @@ public class ExportWriter implements SolrCore.RawWriter,
Closeable {
// Field is stored-only (no usable DocValues)
if (includeStoredFields) {
storedFields.put(field, schemaField);
- } else if (requestFieldNames.contains(field)) {
+ } else if (explicitlyNamedFields.contains(field)) {
// Explicitly requested field without DocValues and
includeStoredFields=false
throw new IOException(
schemaField
@@ -525,7 +579,7 @@ public class ExportWriter implements SolrCore.RawWriter,
Closeable {
+ "Try setting includeStoredFields=true to retrieve this
field from stored values.");
}
// Else: glob matched stored-only field without includeStoredFields -
silently skip
- } else if (requestFieldNames.contains(field)) {
+ } else if (explicitlyNamedFields.contains(field)) {
// Explicitly requested field that has neither DocValues nor stored
if (fieldType instanceof SortableTextField &&
!schemaField.useDocValuesAsStored()) {
throw new IOException(
@@ -610,6 +664,47 @@ public class ExportWriter implements SolrCore.RawWriter,
Closeable {
return writers;
}
+ /** Context for transformers in {@code fl}; there is no {@link DocList} when
exporting. */
+ private static final class ExportResultContext extends ResultContext {
+ private final ReturnFields returnFields;
+ private final SolrIndexSearcher searcher;
+ private final Query query;
+ private final SolrQueryRequest req;
+
+ ExportResultContext(
+ ReturnFields returnFields, SolrIndexSearcher searcher, Query query,
SolrQueryRequest req) {
+ this.returnFields = returnFields;
+ this.searcher = searcher;
+ this.query = query;
+ this.req = req;
+ }
+
+ @Override
+ public DocList getDocList() {
+ return null;
+ }
+
+ @Override
+ public ReturnFields getReturnFields() {
+ return returnFields;
+ }
+
+ @Override
+ public SolrIndexSearcher getSearcher() {
+ return searcher;
+ }
+
+ @Override
+ public Query getQuery() {
+ return query;
+ }
+
+ @Override
+ public SolrQueryRequest getRequest() {
+ return req;
+ }
+ }
+
private static boolean canUseDocValues(SchemaField schemaField, FieldType
fieldType) {
return schemaField.hasDocValues()
// Special handling for SortableTextField: unlike other field types,
it requires
diff --git
a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java
b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java
index 86bbefb8e37..3c21cb8202c 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java
@@ -18,7 +18,6 @@ package org.apache.solr.handler.export;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.TimeoutException;
@@ -35,9 +34,6 @@ import
org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-import org.apache.solr.common.IteratorWriter;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.util.SimpleOrderedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,14 +61,7 @@ public class ExportWriterStream extends TupleStream
implements Expressible {
@Override
public EntryWriter put(CharSequence k, Object v) throws IOException {
- if (v instanceof IteratorWriter) {
- List<Object> lst = new ArrayList<>();
- ((IteratorWriter) v).toList(lst);
- v = lst;
- } else if (v instanceof MapWriter) {
- v = new SimpleOrderedMap<>((MapWriter) v);
- }
- tuple.put(k.toString(), v);
+ tuple.put(k.toString(), ExportWriter.materialize(v));
return this;
}
}
diff --git
a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java
b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java
index 61384d50553..5be050c8063 100644
--- a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java
+++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java
@@ -1308,6 +1308,75 @@ public class TestExportWriter extends SolrTestCaseJ4 {
jq.contains("\"sortabledv\""));
}
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDocTransformers() throws Exception {
+ assertU(delQ("*:*"));
+ assertU(adoc("id", "1", "intdv", "10", "intdv_m", "1", "intdv_m", "2"));
+ assertU(adoc("id", "2", "intdv", "20"));
+ assertU(commit());
+ assertU(adoc("id", "3", "intdv", "30"));
+ assertU(commit());
+
+ Map<String, Object> selectDocIds = new HashMap<>();
+ String selectRsp = h.query(req("q", "*:*", "fl", "id,[docid]", "wt",
"json"));
+ for (Map<String, Object> doc :
+ (List<Map<String, Object>>)
+ Utils.getObjectByPath(
+ mapper.readValue(selectRsp, Map.class), false,
"/response/docs")) {
+ selectDocIds.put((String) doc.get("id"), doc.get("[docid]"));
+ }
+ assertEquals(3, selectDocIds.size());
+
+ // [docid] matches /select's, and fields are materialized (incl.
multiValued)
+ List<Map<String, Object>> docs =
+ (List<Map<String, Object>>)
+ queryJsonReturnDocs(
+ reqWithPath("/export", "q", "*:*", "fl", "id,intdv_m,[docid]",
"sort", "id asc"));
+ assertEquals(3, docs.size());
+ for (Map<String, Object> doc : docs) {
+ assertEquals(doc.toString(), selectDocIds.get(doc.get("id")),
doc.get("[docid]"));
+ }
+ assertEquals(List.of(1, 2), docs.get(0).get("intdv_m"));
+
+ // alias renames; transformer local params may contain commas
+ docs =
+ (List<Map<String, Object>>)
+ queryJsonReturnDocs(
+ reqWithPath(
+ "/export", "q", "*:*", "fl", "id,x:intdv,v:[value
v='a,b']", "sort", "id asc"));
+ assertEquals(3, docs.size());
+ assertEquals(Map.of("id", "1", "x", 10, "v", "a,b"), docs.get(0));
+
+ // transformed fields are visible to streaming expressions
+ docs =
+ (List<Map<String, Object>>)
+ queryJsonReturnDocs(
+ reqWithPath(
+ "/export",
+ "q",
+ "*:*",
+ "fl",
+ "id,[docid]",
+ "sort",
+ "id asc",
+ "expr",
+ "top(n=2,input(),sort=\"id desc\")"));
+ assertEquals(2, docs.size());
+ assertEquals("3", docs.get(0).get("id"));
+ assertEquals(selectDocIds.get("3"), docs.get(0).get("[docid]"));
+
+ // score is still rejected
+ docs =
+ (List<Map<String, Object>>)
+ queryJsonReturnDocs(
+ reqWithPath("/export", "q", "*:*", "fl", "id,score", "sort",
"id asc"));
+ assertEquals(1, docs.size());
+ assertTrue(
+ docs.get(0).toString(),
+
docs.get(0).get(StreamParams.EXCEPTION).toString().contains("Scoring"));
+ }
+
@SuppressWarnings("rawtypes")
private void validateSort(int numDocs) throws Exception {
// 10 fields
diff --git
a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc
b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc
index 503d12eb272..823bc51e161 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc
@@ -101,6 +101,10 @@ Note that sort fields still require docValues, regardless
of this setting.
Wildcard patterns can be used for the field list (e.g. `fl=*_i`) and will be
expanded to the list of fields that match the pattern and are able to be
exported, see <<Field Requirements>>.
+Field aliases (e.g. `fl=id,price:price_d`) and
xref:document-transformers.adoc[document transformers] (e.g. `fl=id,[docid]`)
are supported, and their output is also visible to a local streaming expression.
+Transformers are applied one document at a time, and those that depend on
scores or on a list of matching documents won't work.
+Some transformers, like `[subquery]` and `[child]`, produce output that might
not work well, or at all, with `/export` and streaming expressions.
+
Returning scores is not supported at this time.
=== Specifying the Local Streaming Expression
