This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 7619c842b1d SOLR-17655: Remove ExternalFileField (#3244)
7619c842b1d is described below
commit 7619c842b1d7e7f3f4241b18390dc9f9ed6526aa
Author: Eric Pugh <[email protected]>
AuthorDate: Wed Apr 2 09:25:57 2025 +0200
SOLR-17655: Remove ExternalFileField (#3244)
---
solr/CHANGES.txt | 2 +
.../org/apache/solr/schema/ExternalFileField.java | 145 --------
.../solr/schema/ExternalFileFieldReloader.java | 90 -----
.../solr/search/function/FileFloatSource.java | 380 ---------------------
.../java/org/apache/solr/util/VersionedFile.java | 118 -------
.../solr/collection1/conf/bad-schema-eff.xml | 44 ---
.../solr/collection1/conf/schema-eff.xml | 40 ---
.../solr/collection1/conf/schema-trie.xml | 4 -
.../collection1/conf/schema-vector-catchall.xml | 2 -
.../test-files/solr/collection1/conf/schema.xml | 2 -
.../test-files/solr/collection1/conf/schema11.xml | 18 -
.../collection1/conf/solrconfig-functionquery.xml | 3 -
.../solr/schema/ExternalFileFieldSortTest.java | 66 ----
.../solr/search/function/TestFunctionQuery.java | 160 +--------
.../AbstractAtomicUpdatesMultivalueTestBase.java | 6 -
.../pages/external-files-processes.adoc | 69 +---
.../pages/field-types-included-with-solr.adoc | 2 -
.../modules/query-guide/pages/result-grouping.adoc | 2 +-
.../pages/major-changes-in-solr-10.adoc | 4 +-
.../test-files/solr/collection1/conf/schema.xml | 2 -
20 files changed, 8 insertions(+), 1151 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 322094ab053..148e967c45d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -111,6 +111,8 @@ Deprecation Removals
* SOLR-17518: Removed deprecated methods `getXml()` and `writeXml()` from
`UpdateRequest`. (Pierre Salagnac)
+* SOLR-17655: Removed deprecated ExternalFileField field type. (Eric Pugh)
+
Dependency Upgrades
---------------------
(No changes)
diff --git a/solr/core/src/java/org/apache/solr/schema/ExternalFileField.java
b/solr/core/src/java/org/apache/solr/schema/ExternalFileField.java
deleted file mode 100644
index 2baa7e22781..00000000000
--- a/solr/core/src/java/org/apache/solr/schema/ExternalFileField.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.schema;
-
-import java.io.IOException;
-import java.util.Map;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.SortField;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.request.SolrRequestInfo;
-import org.apache.solr.response.TextResponseWriter;
-import org.apache.solr.search.QParser;
-import org.apache.solr.search.function.FileFloatSource;
-import org.apache.solr.uninverting.UninvertingReader.Type;
-
-/**
- * Get values from an external file instead of the index.
- *
- * <p><code>keyField</code> will normally be the unique key field, but it
doesn't have to be.
- *
- * <ul>
- * <li>It's OK to have a keyField value that can't be found in the index
- * <li>It's OK to have some documents without a keyField in the file (defVal
is used as the
- * default)
- * <li>It's OK for a keyField value to point to multiple documents (no
uniqueness requirement)
- * </ul>
- *
- * The format of the external file is simply newline separated
keyFieldValue=floatValue. <br>
- * Example: <br>
- * <code>doc33=1.414</code> <br>
- * <code>doc34=3.14159</code> <br>
- * <code>doc40=42</code>
- *
- * <p>Solr looks for the external file in the index directory under the name of
- * external_<fieldname> or external_<fieldname>.*
- *
- * <p>If any files of the latter pattern appear, the last (after being sorted
by name) will be used
- * and previous versions will be deleted. This is to help support systems
where one may not be able
- * to overwrite a file (like Windows, if the file is in use).
- *
- * <p>If the external file has already been loaded, and it is changed, those
changes will not be
- * visible until a commit has been done.
- *
- * <p>The external file may be sorted or unsorted by the key field, but it
will be substantially
- * slower (untested) if it isn't sorted.
- *
- * <p>Fields of this type may currently only be used as a ValueSource in a
FunctionQuery.
- *
- * <p>You can return the value as a field in the document by wrapping it like
so: <code>
- * fl=id,field(inventory_count)</code>.
- *
- * @see ExternalFileFieldReloader
- */
-public class ExternalFileField extends FieldType implements SchemaAware {
- private String keyFieldName;
- private IndexSchema schema;
- private float defVal;
-
- @Override
- protected void init(IndexSchema schema, Map<String, String> args) {
- restrictProps(SORT_MISSING_FIRST | SORT_MISSING_LAST);
- keyFieldName = args.remove("keyField");
- String defValS = args.remove("defVal");
- defVal = defValS == null ? 0 : Float.parseFloat(defValS);
- this.schema = schema;
- }
-
- @Override
- public void write(TextResponseWriter writer, String name, IndexableField f)
throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SortField getSortField(SchemaField field, boolean reverse) {
- FileFloatSource source = getFileFloatSource(field);
- return source.getSortField(reverse);
- }
-
- @Override
- public Type getUninversionType(SchemaField sf) {
- return null;
- }
-
- @Override
- public ValueSource getValueSource(SchemaField field, QParser parser) {
- return getFileFloatSource(field);
- }
-
- /**
- * Get a FileFloatSource for the given field, using the datadir from the
IndexSchema
- *
- * @param field the field to get a source for
- * @return a FileFloatSource
- */
- public FileFloatSource getFileFloatSource(SchemaField field) {
- return getFileFloatSource(
- field,
SolrRequestInfo.getRequestInfo().getReq().getCore().getDataDir());
- }
-
- /**
- * Get a FileFloatSource for the given field. Call this in preference to
- * getFileFloatSource(SchemaField) if this may be called before the Core is
fully initialised (eg
- * in SolrEventListener calls).
- *
- * @param field the field to get a source for
- * @param datadir the data directory in which to look for the external file
- * @return a FileFloatSource
- */
- public FileFloatSource getFileFloatSource(SchemaField field, String datadir)
{
- // Because the float source uses a static cache, all source objects will
- // refer to the same data.
- return new FileFloatSource(field, getKeyField(), defVal, datadir);
- }
-
- // If no key field is defined, we use the unique key field
- private SchemaField getKeyField() {
- return keyFieldName == null ? schema.getUniqueKeyField() :
schema.getField(keyFieldName);
- }
-
- @Override
- public void inform(IndexSchema schema) {
- this.schema = schema;
-
- if (keyFieldName != null &&
schema.getFieldType(keyFieldName).isPointField()) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR,
- "keyField '" + keyFieldName + "' has a Point field type, which is
not supported.");
- }
- }
-}
diff --git
a/solr/core/src/java/org/apache/solr/schema/ExternalFileFieldReloader.java
b/solr/core/src/java/org/apache/solr/schema/ExternalFileFieldReloader.java
deleted file mode 100644
index 10f4945ed40..00000000000
--- a/solr/core/src/java/org/apache/solr/schema/ExternalFileFieldReloader.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.schema;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.lucene.index.IndexReader;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.AbstractSolrEventListener;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.search.function.FileFloatSource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * An event listener to reload ExternalFileFields for new searchers.
- *
- * <p>Opening a new IndexSearcher will invalidate the internal caches used by
{@link
- * ExternalFileField}. By default, these caches are reloaded lazily by the
first search that uses
- * them. For large external files, this can slow down searches unacceptably.
- *
- * <p>To reload the caches when the searcher is first opened, set up event
listeners in your
- * solrconfig.xml:
- *
- * <pre>
- * <listener event="newSearcher"
class="org.apache.solr.schema.ExternalFileFieldReloader"/>
- * <listener event="firstSearcher"
class="org.apache.solr.schema.ExternalFileFieldReloader"/>
- * </pre>
- *
- * The caches will be reloaded for all ExternalFileFields in your schema after
each commit.
- */
-public class ExternalFileFieldReloader extends AbstractSolrEventListener {
-
- private String datadir;
- private List<FileFloatSource> fieldSources = new ArrayList<>();
-
- private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- public ExternalFileFieldReloader(SolrCore core) {
- super(core);
- datadir = core.getDataDir();
- }
-
- @Override
- public void init(NamedList<?> args) {
- cacheFieldSources(getCore().getLatestSchema());
- }
-
- @Override
- public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher
currentSearcher) {
- // We need to reload the caches for the new searcher
- if (null == currentSearcher || newSearcher.getSchema() !=
currentSearcher.getSchema()) {
- cacheFieldSources(newSearcher.getSchema());
- }
- IndexReader reader = newSearcher.getIndexReader();
- for (FileFloatSource fieldSource : fieldSources) {
- fieldSource.refreshCache(reader);
- }
- }
-
- /** Caches FileFloatSource's from all ExternalFileField instances in the
schema */
- public void cacheFieldSources(IndexSchema schema) {
- fieldSources.clear();
- for (SchemaField field : schema.getFields().values()) {
- FieldType type = field.getType();
- if (type instanceof ExternalFileField eff) {
- fieldSources.add(eff.getFileFloatSource(field, datadir));
- if (log.isInfoEnabled()) {
- log.info("Adding ExternalFileFieldReloader listener for field {}",
field.getName());
- }
- }
- }
- }
-}
diff --git
a/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
b/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
deleted file mode 100644
index 2d38957912d..00000000000
--- a/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
+++ /dev/null
@@ -1,380 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.search.function;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.WeakHashMap;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReaderContext;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiTerms;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.ReaderUtil;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.FloatDocValues;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.solr.handler.RequestHandlerBase;
-import org.apache.solr.handler.RequestHandlerUtils;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.schema.FieldType;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.security.AuthorizationContext;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.apache.solr.util.VersionedFile;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Obtains float field values from an external file.
- *
- * @see org.apache.solr.schema.ExternalFileField
- * @see org.apache.solr.schema.ExternalFileFieldReloader
- */
-public class FileFloatSource extends ValueSource {
-
- private SchemaField field;
- private final SchemaField keyField;
- private final float defVal;
- private final String dataDir;
-
- private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- /**
- * Creates a new FileFloatSource
- *
- * @param field the source's SchemaField
- * @param keyField the field to use as a key
- * @param defVal the default value to use if a field has no entry in the
external file
- * @param dataDir the directory in which to look for the external file
- */
- public FileFloatSource(SchemaField field, SchemaField keyField, float
defVal, String dataDir) {
- this.field = field;
- this.keyField = keyField;
- this.defVal = defVal;
- this.dataDir = dataDir;
- }
-
- @Override
- public String description() {
- return "float(" + field + ')';
- }
-
- @Override
- public FunctionValues getValues(Map<Object, Object> context,
LeafReaderContext readerContext)
- throws IOException {
- final int off = readerContext.docBase;
- IndexReaderContext topLevelContext =
ReaderUtil.getTopLevelContext(readerContext);
-
- final float[] arr = getCachedFloats(topLevelContext.reader());
- return new FloatDocValues(this) {
- @Override
- public float floatVal(int doc) {
- return arr[doc + off];
- }
-
- @Override
- public Object objectVal(int doc) {
- return floatVal(doc); // TODO: keep track of missing values
- }
- };
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof FileFloatSource other)) return false;
- return this.field.getName().equals(other.field.getName())
- && this.keyField.getName().equals(other.keyField.getName())
- && this.defVal == other.defVal
- && this.dataDir.equals(other.dataDir);
- }
-
- @Override
- public int hashCode() {
- return FileFloatSource.class.hashCode() + field.getName().hashCode();
- }
- ;
-
- @Override
- public String toString() {
- return "FileFloatSource(field="
- + field.getName()
- + ",keyField="
- + keyField.getName()
- + ",defVal="
- + defVal
- + ",dataDir="
- + dataDir
- + ")";
- }
-
- /** Remove all cached entries. Values are lazily loaded next time
getValues() is called. */
- public static void resetCache() {
- floatCache.resetCache();
- }
-
- /**
- * Refresh the cache for an IndexReader. The new values are loaded in the
background and then
- * swapped in, so queries against the cache should not block while the
reload is happening.
- *
- * @param reader the IndexReader whose cache needs refreshing
- */
- public void refreshCache(IndexReader reader) {
- if (log.isInfoEnabled()) {
- log.info("Refreshing FileFloatSource cache for field {}",
this.field.getName());
- }
- floatCache.refresh(reader, new Entry(this));
- if (log.isInfoEnabled()) {
- log.info("FileFloatSource cache for field {} reloaded",
this.field.getName());
- }
- }
-
- private final float[] getCachedFloats(IndexReader reader) {
- return (float[]) floatCache.get(reader, new Entry(this));
- }
-
- static Cache floatCache =
- new Cache() {
- @Override
- protected Object createValue(IndexReader reader, Object key) {
- return getFloats(((Entry) key).ffs, reader);
- }
- };
-
- /** Internal cache. (from lucene FieldCache) */
- abstract static class Cache {
- private final Map<IndexReader, Map<Object, Object>> readerCache = new
WeakHashMap<>();
-
- protected abstract Object createValue(IndexReader reader, Object key);
-
- public void refresh(IndexReader reader, Object key) {
- Object refreshedValues = createValue(reader, key);
- synchronized (readerCache) {
- Map<Object, Object> innerCache = readerCache.computeIfAbsent(reader, k
-> new HashMap<>());
- innerCache.put(key, refreshedValues);
- }
- }
-
- public Object get(IndexReader reader, Object key) {
- Map<Object, Object> innerCache;
- Object value;
- synchronized (readerCache) {
- innerCache = readerCache.get(reader);
- if (innerCache == null) {
- innerCache = new HashMap<>();
- readerCache.put(reader, innerCache);
- value = null;
- } else {
- value = innerCache.get(key);
- }
- if (value == null) {
- value = new CreationPlaceholder();
- innerCache.put(key, value);
- }
- }
- if (value instanceof CreationPlaceholder) {
- synchronized (value) {
- CreationPlaceholder progress = (CreationPlaceholder) value;
- if (progress.value == null) {
- progress.value = createValue(reader, key);
- synchronized (readerCache) {
- innerCache.put(key, progress.value);
- onlyForTesting = progress.value;
- }
- }
- return progress.value;
- }
- }
-
- return value;
- }
-
- public void resetCache() {
- synchronized (readerCache) {
- // Map.clear() is optional and can throw UnsupportedOperationException,
- // but readerCache is WeakHashMap and it supports clear().
- readerCache.clear();
- }
- }
- }
-
- static Object onlyForTesting; // set to the last value
-
- static final class CreationPlaceholder {
- Object value;
- }
-
- /** Expert: Every composite-key in the internal cache is of this type. */
- private static class Entry {
- final FileFloatSource ffs;
-
- public Entry(FileFloatSource ffs) {
- this.ffs = ffs;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof Entry other)) return false;
- return ffs.equals(other.ffs);
- }
-
- @Override
- public int hashCode() {
- return ffs.hashCode();
- }
- }
-
- private static float[] getFloats(FileFloatSource ffs, IndexReader reader) {
- float[] vals = new float[reader.maxDoc()];
- if (ffs.defVal != 0) {
- Arrays.fill(vals, ffs.defVal);
- }
- InputStream is;
- String fname = "external_" + ffs.field.getName();
- try {
- is = VersionedFile.getLatestFile(ffs.dataDir, fname);
- } catch (IOException e) {
- // log, use defaults
- log.error("Error opening external value source file: ", e);
- return vals;
- }
-
- BufferedReader r = new BufferedReader(new InputStreamReader(is,
StandardCharsets.UTF_8));
-
- String idName = ffs.keyField.getName();
- FieldType idType = ffs.keyField.getType();
-
- // warning: lucene's termEnum.skipTo() is not optimized... it simply does
a next()
- // because of this, simply ask the reader for a new termEnum rather than
- // trying to use skipTo()
-
- List<String> notFound = new ArrayList<>();
- int notFoundCount = 0;
- int otherErrors = 0;
-
- char delimiter = '=';
-
- BytesRefBuilder internalKey = new BytesRefBuilder();
-
- try {
- TermsEnum termsEnum = MultiTerms.getTerms(reader, idName).iterator();
- PostingsEnum postingsEnum = null;
-
- // removing deleted docs shouldn't matter
- // final Bits liveDocs = MultiLeafReader.getLiveDocs(reader);
-
- for (String line; (line = r.readLine()) != null; ) {
- int delimIndex = line.lastIndexOf(delimiter);
- if (delimIndex < 0) continue;
-
- int endIndex = line.length();
- String key = line.substring(0, delimIndex);
- String val = line.substring(delimIndex + 1, endIndex);
-
- float fval;
- try {
- idType.readableToIndexed(key, internalKey);
- fval = Float.parseFloat(val);
- } catch (Exception e) {
- if (++otherErrors <= 10) {
- log.error(
- "Error loading external value source: {} {}{}",
- fname,
- e,
- (otherErrors < 10 ? "" : "\tSkipping future errors for this
file."));
- }
- continue; // go to next line in file.. leave values as default.
- }
-
- if (!termsEnum.seekExact(internalKey.get())) {
- if (notFoundCount < 10) { // collect first 10 not found for logging
- notFound.add(key);
- }
- notFoundCount++;
- continue;
- }
-
- postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
- int doc;
- while ((doc = postingsEnum.nextDoc()) !=
DocIdSetIterator.NO_MORE_DOCS) {
- vals[doc] = fval;
- }
- }
-
- } catch (IOException e) {
- // log, use defaults
- log.error("Error loading external value source: ", e);
- } finally {
- // swallow exceptions on close so we don't override any
- // exceptions that happened in the loop
- try {
- r.close();
- } catch (Exception e) {
- }
- }
- if (log.isInfoEnabled()) {
- String tmp = (notFoundCount == 0 ? "" : " :" + notFoundCount + " missing
keys " + notFound);
- log.info("Loaded external value source {}{}", fname, tmp);
- }
- return vals;
- }
-
- public static class ReloadCacheRequestHandler extends RequestHandlerBase {
- private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @Override
- public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
- FileFloatSource.resetCache();
- log.debug("readerCache has been reset.");
-
- UpdateRequestProcessor processor =
- req.getCore().getUpdateProcessingChain(null).createProcessor(req,
rsp);
- try {
- RequestHandlerUtils.handleCommit(req, processor, req.getParams(),
true);
- } finally {
- try {
- processor.finish();
- } finally {
- processor.close();
- }
- }
- }
-
- @Override
- public String getDescription() {
- return "Reload readerCache request handler";
- }
-
- @Override
- public Name getPermissionName(AuthorizationContext request) {
- return Name.UPDATE_PERM;
- }
- }
-}
diff --git a/solr/core/src/java/org/apache/solr/util/VersionedFile.java
b/solr/core/src/java/org/apache/solr/util/VersionedFile.java
deleted file mode 100644
index dbc9dc8430a..00000000000
--- a/solr/core/src/java/org/apache/solr/util/VersionedFile.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Stream;
-import org.apache.solr.common.SolrException;
-
-/**
- * @since solr 1.3
- */
-public class VersionedFile {
- /* Open the latest version of a file... fileName if that exists, or
- * the last fileName.* after being sorted lexicographically.
- * Older versions of the file are deleted (and queued for deletion if
- * that fails).
- * TODO SOLR-8282 dirName should be a Path instead of string
- */
- public static InputStream getLatestFile(String dirName, String fileName)
throws IOException {
- Collection<Path> oldFiles = null;
- final String prefix = fileName + '.';
- Path f = Path.of(dirName, fileName);
- InputStream is = null;
-
- // there can be a race between checking for a file and opening it...
- // the user may have just put a new version in and deleted an old version.
- // try multiple times in a row.
- for (int retry = 0; retry < 10 && is == null; retry++) {
- try {
- if (!Files.exists(f)) {
- Path dir = Path.of(dirName);
- List<Path> fileList;
-
- try (Stream<Path> files = Files.list(dir)) {
- fileList =
- files
- .filter((file) ->
file.getFileName().toString().startsWith(prefix))
- .sorted()
- .toList();
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Unable to list files in
" + dir, e);
- }
-
- f = dir.resolve(fileList.getLast());
- oldFiles = new ArrayList<>();
- for (int i = 0; i < fileList.size() - 1; i++) {
- oldFiles.add(dir.resolve(fileList.get(i)));
- }
- }
-
- is = Files.newInputStream(f);
- } catch (Exception e) {
- // swallow exception for now
- }
- }
-
- // allow exception to be thrown from the final try.
- if (is == null) {
- is = Files.newInputStream(f);
- }
-
- // delete old files only after we have successfully opened the newest
- if (oldFiles != null) {
- delete(oldFiles);
- }
-
- return is;
- }
-
- private static final Set<Path> deleteList = new HashSet<>();
-
- private static synchronized void delete(Collection<Path> files) {
- synchronized (deleteList) {
- deleteList.addAll(files);
- List<Path> deleted = new ArrayList<>();
- for (Path df : deleteList) {
- try {
- try {
- Files.deleteIfExists(df);
- } catch (IOException cause) {
- // TODO: should this class care if a file couldn't be deleted?
- // this just emulates previous behavior, where only
SecurityException would be handled.
- }
- // deleteList.remove(df);
- deleted.add(df);
- } catch (SecurityException e) {
- if (!Files.exists(df)) {
- deleted.add(df);
- }
- }
- }
- deleteList.removeAll(deleted);
- }
- }
-}
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-eff.xml
b/solr/core/src/test-files/solr/collection1/conf/bad-schema-eff.xml
deleted file mode 100644
index 21f390ae894..00000000000
--- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-eff.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<schema name="example" version="1.7">
-
-
- <field name="id" type="string" indexed="true" stored="true" required="true"
multiValued="false"/>
- <field name="eff" type="eff"/>
-
-
- <!-- Field to use to determine and enforce document uniqueness.
- Unless this field is marked with required="false", it will be a
required field
- -->
- <uniqueKey>id</uniqueKey>
-
-
- <!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
- <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
-
- <fieldType name="pint" class="solr.IntPointField"/>
- <field name="keyfield" type="pint" indexed="true" stored="true"
multiValued="false"/>
-
- <!-- Our external file field type -->
- <!-- Begin bad stuff: keyfield is points-based -->
- <fieldType name="eff" class="solr.ExternalFileField" keyField="keyfield"/>
- <!-- End bad stuff -->
-
-</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-eff.xml
b/solr/core/src/test-files/solr/collection1/conf/schema-eff.xml
deleted file mode 100644
index 208c011c8fc..00000000000
--- a/solr/core/src/test-files/solr/collection1/conf/schema-eff.xml
+++ /dev/null
@@ -1,40 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<schema name="example" version="1.7">
-
-
- <field name="id" type="string" indexed="true" stored="true" required="true"
multiValued="false"/>
- <field name="eff" type="eff"/>
-
-
- <!-- Field to use to determine and enforce document uniqueness.
- Unless this field is marked with required="false", it will be a
required field
- -->
- <uniqueKey>id</uniqueKey>
-
-
- <!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
- <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
-
- <!-- Our external file field type -->
- <fieldType name="eff" class="solr.ExternalFileField"/>
-
-
-</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml
b/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml
index f42afb34710..8d2634fbe32 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-trie.xml
@@ -229,8 +229,6 @@
-->
<fieldType name="ignored" stored="false" indexed="false"
class="solr.StrField"/>
- <fieldType name="file" keyField="id" defVal="1" stored="false"
indexed="false" class="solr.ExternalFileField" />
-
<!-- Valid attributes for fields:
name: mandatory - the name for the field
type: mandatory - the name of a previously defined type from the
<fieldType>s
@@ -282,8 +280,6 @@
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="*_ws" type="text_ws" indexed="true" stored="true"/>
- <dynamicField name="*_extf" type="file"/>
-
<dynamicField name="*_random" type="random"/>
<!-- uncomment the following to ignore any fields that don't already match
an existing
diff --git
a/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml
b/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml
index 27afe79d551..7b69316dbb7 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-vector-catchall.xml
@@ -530,7 +530,6 @@
<fieldType name="binary" class="solr.BinaryField" />
<fieldType name="collation" class="solr.CollationField" language="en" />
- <fieldType name="externalFile" class="solr.ExternalFileField" />
<fieldType name="icuCollation" class="solr.ICUCollationField" locale="en"
/>
<fieldType name="latLonPointSpatial" class="solr.LatLonPointSpatialField"
/>
<fieldType name="randomSort" class="solr.RandomSortField" />
@@ -666,7 +665,6 @@
<field name="datePointRemove" type="pdate" indexed="true" stored="true"
multiValued="true"/>
<field name="dateRangeRemove" type="dateRange" indexed="true"
stored="true" multiValued="true"/>
<field name="doublePointRemove" type="pdouble" indexed="true"
stored="true" multiValued="true"/>
- <field name="externalFileRemove" type="externalFile" indexed="true"
stored="true" multiValued="true"/>
<field name="floatPointRemove" type="pfloat" indexed="true" stored="true"
multiValued="true"/>
<field name="icuCollationRemove" type="icuCollation" indexed="true"
stored="true" multiValued="true"/>
<field name="intPointRemove" type="pint" indexed="true" stored="true"
multiValued="true"/>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml
b/solr/core/src/test-files/solr/collection1/conf/schema.xml
index e8795c81104..c0446bbf33a 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml
@@ -532,7 +532,6 @@
<fieldType name="binary" class="solr.BinaryField" />
<fieldType name="collation" class="solr.CollationField" language="en" />
- <fieldType name="externalFile" class="solr.ExternalFileField" />
<fieldType name="icuCollation" class="solr.ICUCollationField" locale="en" />
<fieldType name="latLonPointSpatial" class="solr.LatLonPointSpatialField" />
<fieldType name="randomSort" class="solr.RandomSortField" />
@@ -668,7 +667,6 @@
<field name="datePointRemove" type="pdate" indexed="true" stored="true"
multiValued="true"/>
<field name="dateRangeRemove" type="dateRange" indexed="true" stored="true"
multiValued="true"/>
<field name="doublePointRemove" type="pdouble" indexed="true" stored="true"
multiValued="true"/>
- <field name="externalFileRemove" type="externalFile" indexed="true"
stored="true" multiValued="true"/>
<field name="floatPointRemove" type="pfloat" indexed="true" stored="true"
multiValued="true"/>
<field name="icuCollationRemove" type="icuCollation" indexed="true"
stored="true" multiValued="true"/>
<field name="intPointRemove" type="pint" indexed="true" stored="true"
multiValued="true"/>
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema11.xml
b/solr/core/src/test-files/solr/collection1/conf/schema11.xml
index 7ec301503ad..a974e7e5579 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema11.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema11.xml
@@ -220,11 +220,6 @@
-->
<fieldType name="ignored" stored="false" indexed="false"
class="solr.StrField" />
- <fieldType name="file" keyField="id" defVal="1" stored="false"
indexed="false" class="solr.ExternalFileField" />
-
- <fieldType name="sfile" keyField="sfile_s" defVal="1" stored="false"
indexed="false" class="solr.ExternalFileField" />
-
-
<fieldType name="tint" class="${solr.tests.IntegerFieldType}"
docValues="${solr.tests.numeric.dv}" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tfloat" class="${solr.tests.FloatFieldType}"
docValues="${solr.tests.numeric.dv}" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tlong" class="${solr.tests.LongFieldType}"
docValues="${solr.tests.numeric.dv}" omitNorms="true" positionIncrementGap="0"/>
@@ -239,14 +234,6 @@
<fieldType name="tdoubles" class="${solr.tests.DoubleFieldType}"
docValues="${solr.tests.numeric.dv}" omitNorms="true" positionIncrementGap="0"
precisionStep="0" multiValued="true" />
<fieldType name="tdates" class="${solr.tests.DateFieldType}"
docValues="${solr.tests.numeric.dv}" omitNorms="true" positionIncrementGap="0"
precisionStep="0" multiValued="true" />
-
- <!-- eff_tint class can't be randomized, because ExternalFileField disallows
point-based keyField. See SOLR-10846. -->
- <fieldType name="eff_tint" class="solr.TrieDoubleField"
docValues="${solr.tests.numeric.dv}" precisionStep="8"/>
- <field name="eff_tint" type="eff_tint" indexed="true" stored="true"/>
- <fieldType name="eff_tfloat" keyField="eff_tint" defVal="0"
- stored="false" indexed="true"
- class="solr.ExternalFileField" />
-
<fieldType name="text_no_analyzer" stored="false" indexed="true"
class="solr.TextField" />
<fieldType name="text_length" class="solr.TextField">
@@ -284,8 +271,6 @@
<field name="signatureField" type="string" indexed="true" stored="false"/>
- <field name="eff_trie" type="eff_tfloat" />
-
<field name="text_no_analyzer" type="text_no_analyzer" indexed="true" />
<field name="_version_" type="long" indexed="true" stored="true"
multiValued="false" />
@@ -445,9 +430,6 @@
</analyzer>
</fieldType>
- <dynamicField name="*_extf" type="file"/>
- <dynamicField name="*_extfs" type="sfile"/>
-
<dynamicField name="*_random" type="random" />
<!-- uncomment the following to ignore any fields that don't already match
an existing
diff --git
a/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml
b/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml
index 2086a6c4ba9..5262a8ca650 100644
---
a/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml
+++
b/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml
@@ -32,9 +32,6 @@
<updateHandler class="solr.DirectUpdateHandler2"/>
- <requestHandler name="/reloadCache"
-
class="org.apache.solr.search.function.FileFloatSource$ReloadCacheRequestHandler"
/>
-
<!-- test ValueSourceParser plugins -->
<valueSourceParser name="nvl"
class="org.apache.solr.search.function.NvlValueSourceParser">
<float name="nvlFloatValue">0.0</float>
diff --git
a/solr/core/src/test/org/apache/solr/schema/ExternalFileFieldSortTest.java
b/solr/core/src/test/org/apache/solr/schema/ExternalFileFieldSortTest.java
deleted file mode 100644
index 23a1152c3eb..00000000000
--- a/solr/core/src/test/org/apache/solr/schema/ExternalFileFieldSortTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.schema;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.SolrException;
-import org.junit.Test;
-
-public class ExternalFileFieldSortTest extends SolrTestCaseJ4 {
-
- static void updateExternalFile() throws IOException {
- final Path testHome =
SolrTestCaseJ4.getFile("solr/collection1").getParent();
- String filename = "external_eff";
- Files.copy(testHome.resolve(filename), Path.of(h.getCore().getDataDir(),
filename));
- }
-
- private void addDocuments() {
- for (int i = 1; i <= 10; i++) {
- String id = Integer.toString(i);
- assertU("add a test doc", adoc("id", id));
- }
- assertU("commit", commit());
- }
-
- @Test
- public void testSort() throws Exception {
- initCore("solrconfig-basic.xml", "schema-eff.xml");
- updateExternalFile();
-
- addDocuments();
- assertQ(
- "query",
- req("q", "*:*", "sort", "eff asc"),
- "//result/doc[position()=1]/str[.='3']",
- "//result/doc[position()=2]/str[.='1']",
- "//result/doc[position()=10]/str[.='8']");
-
- assertQ("test exists", req("q", "*:*", "sort", "exists(eff) desc"));
- }
-
- @Test
- public void testPointKeyFieldType() {
- // This one should fail though, no "node" parameter specified
- SolrException e =
- expectThrows(
- SolrException.class, () -> initCore("solrconfig-basic.xml",
"bad-schema-eff.xml"));
- assertTrue(e.getMessage().contains("has a Point field type, which is not
supported."));
- }
-}
diff --git
a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
index 292c6f55f3c..726ad95b1c1 100644
--- a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
+++ b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
@@ -16,14 +16,9 @@
*/
package org.apache.solr.search.function;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Random;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.solr.SolrTestCaseJ4;
@@ -42,15 +37,6 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
initCore("solrconfig-functionquery.xml", "schema11.xml");
}
- static long start = System.nanoTime();
-
- void makeExternalFile(String field, String contents) throws IOException {
- String dir = h.getCore().getDataDir();
- String filename = dir + "/external_" + field + "." + (start++);
-
- Files.writeString(Path.of(filename), contents, StandardCharsets.UTF_8);
- }
-
void createIndex(String field, int... values) {
// lrf.args.put("version","2.0");
for (int val : values) {
@@ -201,111 +187,6 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
doTest("foo_tf"); // a trie float field
}
- @Test
- public void testExternalField() throws Exception {
- clearIndex();
-
- String field = "foo_extf";
-
- int[] ids = {
- 100, -4, 0, 10, 25, 5, 77, 23, 55, -78, -45, -24, 63, 78, 94, 22, 34,
54321, 261, -627
- };
-
- createIndex(null, ids);
-
- // Unsorted field, the largest first
- makeExternalFile(field, "54321=543210\n0=-999\n25=250");
- // test identity (straight field value)
- singleTest(field, "\0", 54321, 543210, 0, 0, 25, 250, 100, 1);
- Object orig = FileFloatSource.onlyForTesting;
- singleTest(field, "log(\0)");
- // make sure the values were cached
- assertSame(orig, FileFloatSource.onlyForTesting);
- singleTest(field, "sqrt(\0)");
- assertSame(orig, FileFloatSource.onlyForTesting);
-
- makeExternalFile(field, "0=1");
- assertU(h.query("/reloadCache", lrf.makeRequest("", "")));
- singleTest(field, "sqrt(\0)");
- assertNotSame(orig, FileFloatSource.onlyForTesting);
-
- Random r = random();
- for (int i = 0; i < 10; i++) { // do more iterations for a thorough test
- int len = r.nextInt(ids.length + 1);
- boolean sorted = r.nextBoolean();
- // shuffle ids
- for (int j = 0; j < ids.length; j++) {
- int other = r.nextInt(ids.length);
- int v = ids[0];
- ids[0] = ids[other];
- ids[other] = v;
- }
-
- if (sorted) {
- // sort only the first elements
- Arrays.sort(ids, 0, len);
- }
-
- // make random values
- float[] vals = new float[len];
- for (int j = 0; j < len; j++) {
- vals[j] = r.nextInt(200) - 100;
- }
-
- // make and write the external file
- StringBuilder sb = new StringBuilder();
- for (int j = 0; j < len; j++) {
- sb.append("" + ids[j] + "=" + vals[j] + "\n");
- }
- makeExternalFile(field, sb.toString());
-
- // make it visible
- assertU(h.query("/reloadCache", lrf.makeRequest("", "")));
-
- // test it
- float[] answers = new float[ids.length * 2];
- for (int j = 0; j < len; j++) {
- answers[j * 2] = ids[j];
- answers[j * 2 + 1] = Math.max(0, vals[j]);
- }
- for (int j = len; j < ids.length; j++) {
- answers[j * 2] = ids[j];
- answers[j * 2 + 1] = 1; // the default values
- }
-
- singleTest(field, "\0", answers);
- // System.out.println("Done test "+i);
- }
- }
-
- @Test
- public void testExternalFileFieldStringKeys() throws IOException {
- clearIndex();
-
- final String extField = "foo_extfs";
- final String keyField = "sfile_s";
- assertU(adoc("id", "991", keyField, "AAA=AAA"));
- assertU(adoc("id", "992", keyField, "BBB"));
- assertU(adoc("id", "993", keyField, "CCC=CCC"));
- assertU(commit());
- makeExternalFile(extField, "AAA=AAA=543210\nBBB=-8\nCCC=CCC=250");
- singleTest(extField, "\0", 991, 543210, 992, 0, 993, 250);
- }
-
- @Test
- public void testExternalFileFieldNumericKey() throws IOException {
- clearIndex();
-
- final String extField = "eff_trie";
- final String keyField = "eff_tint";
- assertU(adoc("id", "991", keyField, "91"));
- assertU(adoc("id", "992", keyField, "92"));
- assertU(adoc("id", "993", keyField, "93"));
- assertU(commit());
- makeExternalFile(extField, "91=543210\n92=-8\n93=250\n=67");
- singleTest(extField, "\0", 991, 543210, 992, 0, 993, 250);
- }
-
@Test
public void testOrdAndRordOverPointsField() {
assumeTrue("Skipping test when points=false",
Boolean.getBoolean(NUMERIC_POINTS_SYSPROP));
@@ -1014,46 +895,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
}
/**
- * verify that both the field("...") value source parser as well as
ExternalFileField work with
- * esoteric field names
- */
- @Test
- public void testExternalFieldValueSourceParser() throws IOException {
- clearIndex();
-
- String field = "CoMpleX fieldName _extf";
- String fieldAsFunc = "field(\"CoMpleX fieldName _extf\")";
-
- int[] ids = {
- 100, -4, 0, 10, 25, 5, 77, 23, 55, -78, -45, -24, 63, 78, 94, 22, 34,
54321, 261, -627
- };
-
- createIndex(null, ids);
-
- // Unsorted field, the largest first
- makeExternalFile(field, "54321=543210\n0=-999\n25=250");
- // test identity (straight field value)
- singleTest(fieldAsFunc, "\0", 54321, 543210, 0, 0, 25, 250, 100, 1);
- Object orig = FileFloatSource.onlyForTesting;
- singleTest(fieldAsFunc, "log(\0)");
- // make sure the values were cached
- assertSame(orig, FileFloatSource.onlyForTesting);
- singleTest(fieldAsFunc, "sqrt(\0)");
- assertSame(orig, FileFloatSource.onlyForTesting);
-
- makeExternalFile(field, "0=1");
- assertU(adoc("id", "10000")); // will get same reader if no index change
- assertU(commit());
- singleTest(fieldAsFunc, "sqrt(\0)");
- assertNotSame(orig, FileFloatSource.onlyForTesting);
- }
-
- /**
- * some platforms don't allow quote characters in filenames, so in addition
to
- * testExternalFieldValueSourceParser above, test a field name with quotes
in it that does NOT use
- * ExternalFileField
- *
- * @see #testExternalFieldValueSourceParser
+ * some platforms don't allow quote characters in filenames, test a field
name with quotes in it.
*/
@Test
public void testFieldValueSourceParser() {
diff --git
a/solr/core/src/test/org/apache/solr/update/processor/AbstractAtomicUpdatesMultivalueTestBase.java
b/solr/core/src/test/org/apache/solr/update/processor/AbstractAtomicUpdatesMultivalueTestBase.java
index e851088b99b..80240570045 100644
---
a/solr/core/src/test/org/apache/solr/update/processor/AbstractAtomicUpdatesMultivalueTestBase.java
+++
b/solr/core/src/test/org/apache/solr/update/processor/AbstractAtomicUpdatesMultivalueTestBase.java
@@ -399,12 +399,6 @@ public abstract class
AbstractAtomicUpdatesMultivalueTestBase extends EmbeddedSo
});
}
- @Test
- public void testMultivalueExternalFileField() throws SolrServerException,
IOException {
- runTestForFieldWithoutQuery(
- "externalFileRemove", new String[] {"file1.txt", "file2.txt",
"file3.txt", "file4.txt"});
- }
-
@Test
public void testMultivalueFloatPointField() throws SolrServerException,
IOException {
runTestForFieldWithQuery("floatPointRemove", new Float[] {1.0f, 2.0f,
3.0f, 4.0f});
diff --git
a/solr/solr-ref-guide/modules/indexing-guide/pages/external-files-processes.adoc
b/solr/solr-ref-guide/modules/indexing-guide/pages/external-files-processes.adoc
index 9d8ef590413..77029ceaac0 100644
---
a/solr/solr-ref-guide/modules/indexing-guide/pages/external-files-processes.adoc
+++
b/solr/solr-ref-guide/modules/indexing-guide/pages/external-files-processes.adoc
@@ -16,74 +16,7 @@
// specific language governing permissions and limitations
// under the License.
-Solr supports storing field values in an external file with a field type
called the `ExternalFileFieldType`.
-It can also consume a stream of tokens that have already undergone analysis
with a field type called the `PreAnalyzedFieldType`.
-
-== The ExternalFileField Type
-
-The `ExternalFileField` type makes it possible to specify the values for a
field in a file outside the Solr index.
-For such a field, the file contains mappings from a key field to the field
value.
-Another way to think of this is that, instead of specifying the field in
documents as they are indexed, Solr finds values for this field in the external
file.
-
-[IMPORTANT]
-====
-External fields are not searchable.
-They can be used only for function queries or display.
-For more information on function queries, see the section on
xref:query-guide:function-queries.adoc[].
-====
-
-The `ExternalFileField` type is handy for cases where you want to update a
particular field in many documents more often than you want to update the rest
of the documents.
-For example, suppose you have implemented a document rank based on the number
of views.
-You might want to update the rank of all the documents daily or hourly, while
the rest of the contents of the documents might be updated much less frequently.
-Without `ExternalFileField`, you would need to update each document just to
change the rank.
-Using `ExternalFileField` is much more efficient because all document values
for a particular field are stored in an external file that can be updated as
frequently as you wish.
-
-In the xref:schema-elements.adoc[schema], the definition of this field type
might look like this:
-
-[source,xml]
-----
-<fieldType name="entryRankFile" keyField="pkId" defVal="0" stored="false"
indexed="false" class="solr.ExternalFileField"/>
-----
-
-The `keyField` attribute defines the key that will be defined in the external
file.
-It is usually the unique key for the index, but it doesn't need to be as long
as the `keyField` can be used to identify documents in the index.
-A `defVal` defines a default value that will be used if there is no entry in
the external file for a particular document.
-
-=== Format of the External File
-
-The file itself is located in Solr's index directory, which by default is
`$SOLR_HOME/data`.
-The name of the file should be `external_fieldname` or `external_fieldname.*`.
-For the example above, then, the file could be named `external_entryRankFile`
or `external_entryRankFile.txt`.
-
-[TIP]
-====
-If any files using the name pattern `.*` (such as `.txt`) appear, the last
(after being sorted by name) will be used and previous versions will be deleted.
-This behavior supports implementations on systems where one may not be able to
overwrite a file (for example, on Windows, if the file is in use).
-====
-
-The file contains entries that map a key field, on the left of the equals
sign, to a value, on the right.
-Here are a few example entries:
-
-[source,text]
-----
-doc33=1.414
-doc34=3.14159
-doc40=42
-----
-
-The keys listed in this file do not need to be unique.
-The file does not need to be sorted, but Solr will be able to perform the
lookup faster if it is.
-
-=== Reloading an External File
-
-It's possible to define an event listener to reload an external file when
either a searcher is reloaded or when a new searcher is started.
-See the section
xref:configuration-guide:caches-warming.adoc#query-related-listeners[Query-Related
Listeners] for more information, but a sample definition in `solrconfig.xml`
might look like this:
-
-[source,xml]
-----
-<listener event="newSearcher"
class="org.apache.solr.schema.ExternalFileFieldReloader"/>
-<listener event="firstSearcher"
class="org.apache.solr.schema.ExternalFileFieldReloader"/>
-----
+Solr can consume a stream of tokens that have already undergone analysis with
a field type called the `PreAnalyzedFieldType`.
== The PreAnalyzedField Type
diff --git
a/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc
b/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc
index 81c0601fb6e..d29f74cc7b3 100644
---
a/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc
+++
b/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc
@@ -43,8 +43,6 @@ The
{solr-javadocs}/core/org/apache/solr/schema/package-summary.html[`org.apache
|DoublePointField |Double field (64-bit IEEE floating point). This class
encodes double values using a "Dimensional Points" based data structure that
allows for very efficient searches for specific values, or ranges of values.
For single valued fields, `docValues="true"` must be used to enable sorting.
-|ExternalFileField |Pulls values from a file on disk. See the section
xref:external-files-processes.adoc[] for more information.
-
|EnumFieldType |Allows defining an enumerated set of values which may not be
easily sorted by either alphabetic or numeric order (such as a list of
severities, for example). This field type takes a configuration file, which
lists the proper order of the field values. See the section
xref:enum-fields.adoc[] for more information.
|FloatPointField |Floating point field (32-bit IEEE floating point). This
class encodes float values using a "Dimensional Points" based data structure
that allows for very efficient searches for specific values, or ranges of
values. For single valued fields, `docValues="true"` must be used to enable
sorting.
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/result-grouping.adoc
b/solr/solr-ref-guide/modules/query-guide/pages/result-grouping.adoc
index e258fb02999..cc3b824fdce 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/result-grouping.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/result-grouping.adoc
@@ -81,7 +81,7 @@ If `true`, query results will be grouped.
|===
+
The name of the field by which to group results.
-The field must be single-valued, and either be indexed or a field type that
has a value source and works in a function query, such as `ExternalFileField`.
+The field must be single-valued, and either be indexed.
It must also be a string-based field, such as `StrField` or `TextField`.
`group.func`::
diff --git
a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc
b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc
index 113515bdad2..972106a63fb 100644
---
a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc
+++
b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc
@@ -104,6 +104,8 @@ Users who that don't need to vary JAR access on a per-core
basis have several op
* Storing indexes and snapshots in HDFS has been removed. This results in
changes to solrconfig.xml and related configuration files and removal of the
`hdfs` module.
+* ExternalFileField field type has been removed.
+
=== Security
-* There is no longer a distinction between trusted and untrusted configSets;
all configSets are now considered trusted. To ensure security, Solr should be
properly protected using authentication and authorization mechanisms, allowing
only authorized users with administrative privileges to publish them.
\ No newline at end of file
+* There is no longer a distinction between trusted and untrusted configSets;
all configSets are now considered trusted. To ensure security, Solr should be
properly protected using authentication and authorization mechanisms, allowing
only authorized users with administrative privileges to publish them.
diff --git
a/solr/test-framework/src/test-files/solr/collection1/conf/schema.xml
b/solr/test-framework/src/test-files/solr/collection1/conf/schema.xml
index e65df97ed61..b4174377b26 100644
--- a/solr/test-framework/src/test-files/solr/collection1/conf/schema.xml
+++ b/solr/test-framework/src/test-files/solr/collection1/conf/schema.xml
@@ -532,7 +532,6 @@
<fieldType name="binary" class="solr.BinaryField" />
<fieldType name="collation" class="solr.CollationField" language="en" />
- <fieldType name="externalFile" class="solr.ExternalFileField" />
<fieldType name="icuCollation" class="solr.ICUCollationField" locale="en" />
<fieldType name="latLonPointSpatial" class="solr.LatLonPointSpatialField" />
<fieldType name="randomSort" class="solr.RandomSortField" />
@@ -668,7 +667,6 @@
<field name="datePointRemove" type="pdate" indexed="true" stored="true"
multiValued="true"/>
<field name="dateRangeRemove" type="dateRange" indexed="true" stored="true"
multiValued="true"/>
<field name="doublePointRemove" type="pdouble" indexed="true" stored="true"
multiValued="true"/>
- <field name="externalFileRemove" type="externalFile" indexed="true"
stored="true" multiValued="true"/>
<field name="floatPointRemove" type="pfloat" indexed="true" stored="true"
multiValued="true"/>
<field name="icuCollationRemove" type="icuCollation" indexed="true"
stored="true" multiValued="true"/>
<field name="intPointRemove" type="pint" indexed="true" stored="true"
multiValued="true"/>
