This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch branch_9_0
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9_0 by this push:
new 09df6e1 SOLR-15914 Make it super simple to add a (contrib) module to
shared classpath (#557)
09df6e1 is described below
commit 09df6e1000e7c58e044a530c25a043b270409bc9
Author: Jan Høydahl <[email protected]>
AuthorDate: Sat Jan 29 18:33:42 2022 +0100
SOLR-15914 Make it super simple to add a (contrib) module to shared
classpath (#557)
(cherry picked from commit 086b4dc56bdb5bbfaccf9b828c989c3cb9c5e2a5)
---
solr/CHANGES.txt | 4 +
solr/bin/solr.in.cmd | 3 +
solr/bin/solr.in.sh | 3 +
.../java/org/apache/solr/core/CoreContainer.java | 29 -----
.../src/java/org/apache/solr/core/NodeConfig.java | 119 +++++++++++++++++++-
.../java/org/apache/solr/core/SolrXmlConfig.java | 3 +
.../src/java/org/apache/solr/util/ModuleUtils.java | 125 +++++++++++++++++++++
.../org/apache/solr/core/TestCoreContainer.java | 40 ++++++-
.../test/org/apache/solr/util/ModuleUtilsTest.java | 85 ++++++++++++++
solr/server/solr/solr.xml | 1 +
solr/solr-ref-guide/src/configuring-solr-xml.adoc | 14 +++
.../src/major-changes-in-solr-9.adoc | 2 +
solr/solr-ref-guide/src/solr-modules.adoc | 41 +++++++
solr/solr-ref-guide/src/solr-plugins.adoc | 8 +-
14 files changed, 440 insertions(+), 37 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 0bf5353..7d94a2c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -46,6 +46,10 @@ New Features
* SOLR-15197: Support temporal graph queries with DAY and WEEKDAY windows
(Joel Bernstein)
+* SOLR-15914: Official Solr modules (contribs) can now easily be added to
shared class path by environment variable
+ 'SOLR_MODULES' or system property 'solr.modules'. E.g:
SOLR_MODULES=extracting,langid
+ (janhoy, David Smiley, Houston Putman)
+
* SOLR-15880: Introduce support for k nearest neighbors search (Alessandro
Benedetti, Elia Porciani)
Improvements
diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd
index c554be2..72c535b 100755
--- a/solr/bin/solr.in.cmd
+++ b/solr/bin/solr.in.cmd
@@ -222,3 +222,6 @@ REM set SOLR_SOLRXML_REQUIRED=false
REM Some previous versions of Solr use an outdated log4j dependency. If you
are unable to use at least log4j version 2.15.0
REM then enable the following setting to address CVE-2021-44228
REM set SOLR_OPTS=%SOLR_OPTS% -Dlog4j2.formatMsgNoLookups=true
+
+REM The bundled plugins in the "modules" folder can easily be enabled as a
comma-separated list in SOLR_MODULES variable
+REM set SOLR_MODULES=extraction,ltr
\ No newline at end of file
diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh
index aaea9c8..5ec8e9e 100644
--- a/solr/bin/solr.in.sh
+++ b/solr/bin/solr.in.sh
@@ -266,3 +266,6 @@
# Some previous versions of Solr use an outdated log4j dependency. If you are
unable to use at least log4j version 2.15.0
# then enable the following setting to address CVE-2021-44228
# SOLR_OPTS="$SOLR_OPTS -Dlog4j2.formatMsgNoLookups=true"
+
+# The bundled plugins in the "modules" folder can easily be enabled as a
comma-separated list in SOLR_MODULES variable
+# SOLR_MODULES=extraction,ltr
\ No newline at end of file
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 65ad8ad..747dc2e 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -116,7 +116,6 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
@@ -127,7 +126,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -669,32 +667,6 @@ public class CoreContainer {
log.debug("Loading cores into CoreContainer [instanceDir={}]",
getSolrHome());
}
- // Always add $SOLR_HOME/lib to the shared resource loader
- Set<String> libDirs = new LinkedHashSet<>();
- libDirs.add("lib");
-
- if (!StringUtils.isBlank(cfg.getSharedLibDirectory())) {
- List<String> sharedLibs =
Arrays.asList(cfg.getSharedLibDirectory().split("\\s*,\\s*"));
- libDirs.addAll(sharedLibs);
- }
-
- boolean modified = false;
- // add the sharedLib to the shared resource loader before initializing cfg
based plugins
- for (String libDir : libDirs) {
- Path libPath = Paths.get(getSolrHome()).resolve(libDir);
- if (Files.exists(libPath)) {
- try {
- loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
- modified = true;
- } catch (IOException e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't load libs:
" + e, e);
- }
- }
- }
- if (modified) {
- loader.reloadLuceneSPI();
- }
-
ClusterEventProducerFactory clusterEventProducerFactory = new
ClusterEventProducerFactory(this);
clusterEventProducer = clusterEventProducerFactory;
@@ -1714,7 +1686,6 @@ public class CoreContainer {
// CoreDescriptor and we need to reload it from the disk files
CoreDescriptor cd = reloadCoreDescriptor(core.getCoreDescriptor());
solrCores.addCoreDescriptor(cd);
- Closeable oldCore = null;
boolean success = false;
try {
solrCores.waitAddPendingCoreOps(cd.getName());
diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java
b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
index 03195c7..9b87d2a 100644
--- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
@@ -22,17 +22,23 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.logging.LogWatcherConfig;
-
+import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.update.UpdateShardHandlerConfig;
+import org.apache.solr.util.ModuleUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.net.URL;
+import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
@@ -59,6 +65,8 @@ public class NodeConfig {
private final String sharedLibDirectory;
+ private final String modules;
+
private final PluginInfo shardHandlerFactoryConfig;
private final UpdateShardHandlerConfig updateShardHandlerConfig;
@@ -116,7 +124,7 @@ public class NodeConfig {
Properties solrProperties, PluginInfo[]
backupRepositoryPlugins,
MetricsConfig metricsConfig, PluginInfo
transientCacheConfig, PluginInfo tracerConfig,
boolean fromZookeeper, String defaultZkHost, Set<Path>
allowPaths, List<String> allowUrls,
- String configSetServiceClass) {
+ String configSetServiceClass, String modules) {
// all Path params here are absolute and normalized.
this.nodeName = nodeName;
this.coreRootDirectory = coreRootDirectory;
@@ -150,6 +158,7 @@ public class NodeConfig {
this.allowPaths = allowPaths;
this.allowUrls = allowUrls;
this.configSetServiceClass = configSetServiceClass;
+ this.modules = modules;
if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@@ -157,6 +166,9 @@ public class NodeConfig {
}
if (null == this.solrHome) throw new NullPointerException("solrHome");
if (null == this.loader) throw new NullPointerException("loader");
+
+ setupSharedLib();
+ initModules();
}
/**
@@ -206,7 +218,20 @@ public class NodeConfig {
return solrDataHome;
}
- /**
+ /**
+ * Obtain the path of solr's binary installation directory, e.g.
<code>/opt/solr</code>
+ * @return path to install dir
+ * @throws SolrException if property 'solr.install.dir' has not been
initialized
+ */
+ public Path getSolrInstallDir() {
+ String prop =
System.getProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE);
+ if (prop == null || prop.isBlank()) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "solr.install.dir
property not initialized");
+ }
+ return Paths.get(prop);
+ }
+
+ /**
* If null, the lucene default will not be overridden
*
* @see IndexSearcher#setMaxClauseCount
@@ -360,6 +385,81 @@ public class NodeConfig {
return allowUrls;
}
+ // Configures SOLR_HOME/lib to the shared class loader
+ private void setupSharedLib() {
+ // Always add $SOLR_HOME/lib to the shared resource loader
+ Set<String> libDirs = new LinkedHashSet<>();
+ libDirs.add("lib");
+
+ if (!StringUtils.isBlank(getSharedLibDirectory())) {
+ List<String> sharedLibs =
Arrays.asList(getSharedLibDirectory().split("\\s*,\\s*"));
+ libDirs.addAll(sharedLibs);
+ }
+
+ addFoldersToSharedLib(libDirs);
+ }
+
+ /**
+ * Returns the modules as configured in solr.xml. Comma separated list. May
be null if not defined
+ */
+ public String getModules() {
+ return modules;
+ }
+
+ // Finds every jar in each folder and adds it to shardLib, then reloads
Lucene SPI
+ private void addFoldersToSharedLib(Set<String> libDirs) {
+ boolean modified = false;
+ // add the sharedLib to the shared resource loader before initializing cfg
based plugins
+ for (String libDir : libDirs) {
+ Path libPath = getSolrHome().resolve(libDir);
+ if (Files.exists(libPath)) {
+ try {
+ loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
+ modified = true;
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't load libs:
" + e, e);
+ }
+ }
+ }
+ if (modified) {
+ loader.reloadLuceneSPI();
+ }
+ }
+
+ // Adds modules to shared classpath
+ private void initModules() {
+ var moduleNames =
ModuleUtils.resolveModulesFromStringOrSyspropOrEnv(getModules());
+ boolean modified = false;
+ for (String m : moduleNames) {
+ if (!ModuleUtils.moduleExists(getSolrInstallDir(), m)) {
+ log.error("No module with name {}, available modules are {}", m,
ModuleUtils.listAvailableModules(getSolrInstallDir()));
+ // Fail-fast if user requests a non-existing module
+ throw new SolrException(ErrorCode.SERVER_ERROR, "No module with name "
+ m);
+ }
+ Path moduleLibPath = ModuleUtils.getModuleLibPath(getSolrInstallDir(),
m);
+ if (Files.exists(moduleLibPath)) {
+ try {
+ List<URL> urls = SolrResourceLoader.getURLs(moduleLibPath);
+ loader.addToClassLoader(urls);
+ if (log.isInfoEnabled()) {
+ log.info("Added module {}. libPath={} with {} libs", m,
moduleLibPath, urls.size());
+ }
+ if (log.isDebugEnabled()) {
+ log.debug("Libs loaded from {}: {}", moduleLibPath, urls);
+ }
+ modified = true;
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't load libs
for module " + m + ": " + e, e);
+ }
+ } else {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Module lib folder " +
moduleLibPath + " not found.");
+ }
+ }
+ if (modified) {
+ loader.reloadLuceneSPI();
+ }
+ }
+
public static class NodeConfigBuilder {
// all Path fields here are absolute and normalized.
private SolrResourceLoader loader;
@@ -368,6 +468,7 @@ public class NodeConfig {
private Integer booleanQueryMaxClauseCount;
private Path configSetBaseDirectory;
private String sharedLibDirectory;
+ private String modules;
private PluginInfo shardHandlerFactoryConfig;
private UpdateShardHandlerConfig updateShardHandlerConfig =
UpdateShardHandlerConfig.DEFAULT;
private String configSetServiceClass;
@@ -581,6 +682,15 @@ public class NodeConfig {
return this;
}
+ /**
+ * Set list of modules to add to class path
+ * @param moduleNames comma separated list of module names to add to class
loader, e.g. "extracting,ltr,langid"
+ */
+ public NodeConfigBuilder setModules(String moduleNames) {
+ this.modules = moduleNames;
+ return this;
+ }
+
public NodeConfig build() {
// if some things weren't set then set them now. Simple primitives are
set on the field declaration
if (loader == null) {
@@ -595,7 +705,8 @@ public class NodeConfig {
transientCacheSize, useSchemaCache, managementPath,
solrHome, loader, solrProperties,
backupRepositoryPlugins, metricsConfig, transientCacheConfig,
tracerConfig,
- fromZookeeper, defaultZkHost, allowPaths, allowUrls,
configSetServiceClass);
+ fromZookeeper, defaultZkHost, allowPaths, allowUrls,
configSetServiceClass,
+ modules);
}
public NodeConfigBuilder setSolrResourceLoader(SolrResourceLoader
resourceLoader) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
index 48f23be..abc065c 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
@@ -335,6 +335,9 @@ public class SolrXmlConfig {
case "sharedLib":
builder.setSharedLibDirectory(value);
break;
+ case "modules":
+ builder.setModules(value);
+ break;
case "allowPaths":
builder.setAllowPaths(separatePaths(value));
break;
diff --git a/solr/core/src/java/org/apache/solr/util/ModuleUtils.java
b/solr/core/src/java/org/apache/solr/util/ModuleUtils.java
new file mode 100644
index 0000000..434f96b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/ModuleUtils.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import org.apache.solr.common.StringUtils;
+import org.apache.solr.common.util.StrUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Parses the list of modules the user has requested in solr.xml, property
solr.modules or environment SOLR_MODULES.
+ * Then resolves the lib folder for each, so they can be added to class path.
+ */
+public class ModuleUtils {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final String MODULES_FOLDER_NAME = "modules";
+ private static final Pattern validModNamesPattern =
Pattern.compile("[\\w\\d-_]+");
+
+ /**
+ * Returns a path to a module's lib folder
+ * @param moduleName name of module
+ * @return the path to the module's lib folder
+ */
+ public static Path getModuleLibPath(Path solrInstallDirPath, String
moduleName) {
+ return
getModulesPath(solrInstallDirPath).resolve(moduleName).resolve("lib");
+ }
+
+ /**
+ * Finds list of module names requested by system property or environment
variable
+ * @return set of raw volume names from sysprop and/or env.var
+ */
+ static Set<String> resolveFromSyspropOrEnv() {
+ // Fall back to sysprop and env.var if nothing configured through solr.xml
+ Set<String> mods = new HashSet<>();
+ String modulesFromProps = System.getProperty("solr.modules");
+ if (!StringUtils.isEmpty(modulesFromProps)) {
+ mods.addAll(StrUtils.splitSmart(modulesFromProps, ',', true));
+ }
+ String modulesFromEnv = System.getenv("SOLR_MODULES");
+ if (!StringUtils.isEmpty(modulesFromEnv)) {
+ mods.addAll(StrUtils.splitSmart(modulesFromEnv, ',', true));
+ }
+ return mods.stream().map(String::trim).collect(Collectors.toSet());
+ }
+
+ /**
+ * Returns true if a module name is valid and exists in the system
+ */
+ public static boolean moduleExists(Path solrInstallDirPath, String
moduleName) {
+ if (!isValidName(moduleName)) return false;
+ Path modPath = getModulesPath(solrInstallDirPath).resolve(moduleName);
+ return Files.isDirectory(modPath);
+ }
+
+ /**
+ * Returns nam of all existing modules
+ */
+ public static Set<String> listAvailableModules(Path solrInstallDirPath) {
+ try (var moduleFilesStream =
Files.list(getModulesPath(solrInstallDirPath))) {
+ return moduleFilesStream.filter(Files::isDirectory)
+ .map(p -> p.getFileName().toString()).collect(Collectors.toSet());
+ } catch (IOException e) {
+ log.warn("Found no modules in {}", getModulesPath(solrInstallDirPath),
e);
+ return Collections.emptySet();
+ }
+ }
+
+ /**
+ * Parses comma separated string of module names, in practice found in
solr.xml.
+ * If input string is empty (nothing configured) or null (e.g. tag not
present in solr.xml),
+ * we continue to resolve from system property <code>-Dsolr.modules</code>
and if still empty,
+ * fall back to environment variable <code>SOLR_MODULES</code>.
+ * @param modulesFromString raw string of comma-separated module names
+ * @return a set of module
+ */
+ public static Collection<String>
resolveModulesFromStringOrSyspropOrEnv(String modulesFromString) {
+ Collection<String> moduleNames;
+ if (modulesFromString != null && !modulesFromString.isBlank()) {
+ moduleNames = StrUtils.splitSmart(modulesFromString, ',', true);
+ } else {
+ // If nothing configured in solr.xml, check sysprop and environment
+ moduleNames = resolveFromSyspropOrEnv();
+ }
+ return moduleNames.stream().map(String::trim).collect(Collectors.toSet());
+ }
+
+ /**
+ * Returns true if module name is valid
+ */
+ public static boolean isValidName(String moduleName) {
+ return validModNamesPattern.matcher(moduleName).matches();
+ }
+
+ /**
+ * Returns path for modules directory, given the solr install dir path
+ */
+ public static Path getModulesPath(Path solrInstallDirPath) {
+ return solrInstallDirPath.resolve(MODULES_FOLDER_NAME);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
index 7c5e47c..6a8c0f0 100644
--- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
+++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
@@ -19,6 +19,7 @@ package org.apache.solr.core;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -42,7 +43,10 @@ import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.admin.ConfigSetsHandler;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.admin.InfoHandler;
+import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.ModuleUtils;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -88,7 +92,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
public void testSolrHomeAndResourceLoader() throws Exception {
// regardless of what sys prop may be set, the CoreContainer's init arg
should be the definitive
- // solr home -- and nothing i nthe call stack should be "setting" the sys
prop to make that work...
+ // solr home -- and nothing in the call stack should be "setting" the sys
prop to make that work...
final Path fakeSolrHome = createTempDir().toAbsolutePath();
System.setProperty(SOLR_HOME_PROP, fakeSolrHome.toString());
final Path realSolrHome = createTempDir().toAbsolutePath();
@@ -420,6 +424,40 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
}
}
+ @Test
+ public void testModuleLibs() throws Exception {
+ Path tmpRoot = createTempDir("testModLib");
+
+ File lib = Files.createDirectories(ModuleUtils.getModuleLibPath(tmpRoot,
"mod1")).toFile();;
+
+ try (JarOutputStream jar1 = new JarOutputStream(new FileOutputStream(new
File(lib, "jar1.jar")))) {
+ jar1.putNextEntry(new JarEntry("moduleLibFile"));
+ jar1.closeEntry();
+ }
+
+ System.setProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE,
tmpRoot.toAbsolutePath().toString());
+ final CoreContainer cc1 = init(tmpRoot, "<solr></solr>");
+ try {
+ Assert.assertThrows(SolrResourceNotFoundException.class, () ->
cc1.loader.openResource("moduleLibFile").close());
+ } finally {
+ cc1.shutdown();
+ }
+
+ // Explicitly declaring 'lib' makes no change compared to the default
+ final CoreContainer cc2 = init(tmpRoot, "<solr><str
name=\"modules\">mod1</str></solr>");
+ try {
+ cc2.loader.openResource("moduleLibFile").close();
+ } finally {
+ cc2.shutdown();
+ }
+
+ SolrException ex = Assert.assertThrows(SolrException.class, () ->
+ init(tmpRoot, "<solr><str name=\"modules\">nope</str></solr>"));
+ assertEquals("No module with name nope", ex.getMessage());
+
+ System.clearProperty(SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE);
+ }
+
private static final String CONFIGSETS_SOLR_XML ="<?xml version=\"1.0\"
encoding=\"UTF-8\" ?>\n" +
"<solr>\n" +
"<str name=\"configSetBaseDir\">${configsets:configsets}</str>\n" +
diff --git a/solr/core/src/test/org/apache/solr/util/ModuleUtilsTest.java
b/solr/core/src/test/org/apache/solr/util/ModuleUtilsTest.java
new file mode 100644
index 0000000..d2c2db2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/ModuleUtilsTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Set;
+
+public class ModuleUtilsTest extends TestCase {
+ private Path mockRootDir;
+ private final Set<String> expectedMods = Set.of("mod1", "mod2");
+
+ @Override
+ public void setUp() throws Exception {
+ mockRootDir = setupMockInstallDir(expectedMods);
+ }
+
+ public void testModuleExists() {
+ assertTrue(ModuleUtils.moduleExists(mockRootDir, "mod1"));
+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "mod3"));
+ }
+
+ public void testIsValidName() {
+ assertTrue(ModuleUtils.isValidName("mod1-foo_bar-123"));
+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "not valid"));
+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "not/valid"));
+ assertFalse(ModuleUtils.moduleExists(mockRootDir, "not>valid"));
+ }
+
+ public void testGetModuleLibPath() {
+ assertEquals(mockRootDir.resolve("modules")
+ .resolve("mod1").resolve("lib"),
ModuleUtils.getModuleLibPath(mockRootDir, "mod1"));
+ }
+
+ public void testResolveFromSyspropOrEnv() {
+ assertEquals(Collections.emptySet(),
ModuleUtils.resolveFromSyspropOrEnv());
+ System.setProperty("solr.modules", "foo ,bar, baz,mod1");
+ assertEquals(Set.of("foo", "bar", "baz", "mod1"),
ModuleUtils.resolveFromSyspropOrEnv());
+ System.clearProperty("solr.modules");
+ }
+
+ public void testListAvailableModules() {
+ assertEquals(expectedMods, ModuleUtils.listAvailableModules(mockRootDir));
+ }
+
+ public void testResolveModules() {
+ assertEquals(Set.of("foo", "bar", "baz", "mod1"),
ModuleUtils.resolveModulesFromStringOrSyspropOrEnv("foo ,bar, baz,mod1"));
+ assertEquals(Collections.emptySet(),
ModuleUtils.resolveModulesFromStringOrSyspropOrEnv(""));
+ System.setProperty("solr.modules", "foo ,bar, baz,mod1");
+ assertEquals(Set.of("foo", "bar", "baz", "mod1"),
ModuleUtils.resolveModulesFromStringOrSyspropOrEnv(null));
+ System.clearProperty("solr.modules");
+ }
+
+ private Path setupMockInstallDir(Set<String> modules) throws IOException {
+ Path root = Files.createTempDirectory("moduleUtilsTest");
+ Path modPath = root.resolve("modules");
+ Files.createDirectories(modPath);
+ for (var m : modules) {
+ Path libPath = modPath.resolve(m).resolve("lib");
+ Files.createDirectories(libPath);
+ Files.createFile(libPath.resolve("jar1.jar"));
+ Files.createFile(libPath.resolve("jar2.jar"));
+ Files.createFile(libPath.resolve("README.md"));
+ }
+ return root;
+ }
+}
\ No newline at end of file
diff --git a/solr/server/solr/solr.xml b/solr/server/solr/solr.xml
index 8e919c6..6003153 100644
--- a/solr/server/solr/solr.xml
+++ b/solr/server/solr/solr.xml
@@ -30,6 +30,7 @@
<int name="maxBooleanClauses">${solr.max.booleanClauses:1024}</int>
<str name="sharedLib">${solr.sharedLib:}</str>
+ <str name="modules">${solr.modules:}</str>
<str name="allowPaths">${solr.allowPaths:}</str>
<str name="allowUrls">${solr.allowUrls:}</str>
diff --git a/solr/solr-ref-guide/src/configuring-solr-xml.adoc
b/solr/solr-ref-guide/src/configuring-solr-xml.adoc
index 84246e2..64b8d2a5 100644
--- a/solr/solr-ref-guide/src/configuring-solr-xml.adoc
+++ b/solr/solr-ref-guide/src/configuring-solr-xml.adoc
@@ -33,6 +33,7 @@ The default `solr.xml` file is found in
`$SOLR_TIP/server/solr/solr.xml` and loo
<int name="maxBooleanClauses">${solr.max.booleanClauses:1024}</int>
<str name="sharedLib">${solr.sharedLib:}</str>
+ <str name="modules">${solr.modules:}</str>
<str name="allowPaths">${solr.allowPaths:}</str>
<str name="allowUrls">${solr.allowUrls:}</str>
@@ -175,6 +176,19 @@ If the specified path is not absolute, it will be relative
to `$SOLR_HOME`.
Custom handlers may be placed in this directory.
Note that specifying `sharedLib` will not remove `$SOLR_HOME/lib` from Solr's
class path.
+`modules`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: none
+|===
++
+Takes a list of bundled <<solr-modules.adoc#,solr modules>> to add to Solr's
class path
+on startup. This way of adding modules will add them to the shared class
loader, making them
+available for every collection in Solr, unlike `<lib>` tag in `solrconfig.xml`
which is only
+for that one collection. Example value: `extracting,ltr`. See chapter
+<<solr-modules.adoc#,Solr Modules>> for more details.
+
`allowPaths`::
+
[%autowidth,frame=none]
diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
index 0f03ed0..6bde713 100644
--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
@@ -245,6 +245,8 @@ Instead, these libraries will be included with all other
module dependencies in
* SOLR-15954: The prometheus-exporter is no longer packaged as a Solr module.
It can be found under `solr/prometheus-exporter/`.
+* SOLR-15914: Solr modules (formerly known as contribs) can now easily be
enabled by an environment variable (e.g. in `solr.in.sh` or `solr.in.cmd`) or
as a system property (e.g. in `SOLR_OPTS`). Example:
`SOLR_MODULES=extraction,ltr`.
+
== Deprecations & Removed Features
The following list of features have been permanently removed from Solr:
diff --git a/solr/solr-ref-guide/src/solr-modules.adoc
b/solr/solr-ref-guide/src/solr-modules.adoc
new file mode 100644
index 0000000..b9683b5
--- /dev/null
+++ b/solr/solr-ref-guide/src/solr-modules.adoc
@@ -0,0 +1,41 @@
+= Solr Modules
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+Solr modules are addon Solr plugins that are not part of solr-core, but
officially maintained
+by the Solr project. They provide well-defined features such as the
"extracting" module which lets
+users index rich text documents with Apache Tika. A single module can contain
multiple Plugins.
+Modules were earlier known as "contribs".
+
+Each module produces a separate `.jar` file in the build, and additional
dependencies required by
+each module are also packaged with the module. This helps keep the main core
of Solr small and lean.
+
+== Installing a module
+
+The easiest way to enable a module is to list the modules you intend to use
either in the
+system property `solr.modules` or in the environment variable `SOLR_MODULES`
(e.g. in`solr.in.sh`
+or `solr.in.cmd`). You can also add a `<str name="modules">` tag to
+<<configuring-solr-xml.adoc#,solr.xml>>. The expected value is a comma
separated list
+of module names, e.g. `SOLR_MODULES=extracting,ltr`. This way of adding
modules will add
+them to the shared class loader, making them available for every collection in
Solr.
+
+If you only wish to enable a module for a single collection, you may add
`<lib>` tags to `solrconfig.xml`
+as explained in <<libs.adoc#,Lib Directories>>.
+
+Some modules may have been made available as packages for the
<<package-manager.adoc#,Package Manager>>,
+check by listing available pacakges.
\ No newline at end of file
diff --git a/solr/solr-ref-guide/src/solr-plugins.adoc
b/solr/solr-ref-guide/src/solr-plugins.adoc
index 210106e..dd2ba2c 100644
--- a/solr/solr-ref-guide/src/solr-plugins.adoc
+++ b/solr/solr-ref-guide/src/solr-plugins.adoc
@@ -2,7 +2,8 @@
:page-children: libs, \
package-manager, \
cluster-plugins, \
- replica-placement-plugins
+ replica-placement-plugins, \
+ solr-modules
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -45,8 +46,8 @@ Examples of these are
<<authentication-and-authorization-plugins.adoc#,authentic
== Installing Plugins ==
-Most plugins are built-in to Solr and there is nothing to install.
-The subject here is how to make other plugins available to Solr, including
those in modules.
+Most plugins are built-in to Solr core and there is nothing to install.
+The subject here is how to make the bundled modules and other plugins
available to Solr.
Plugins are packaged into a Java jar file and may have other dependent jar
files to function.
The next sections describe some options:
@@ -57,6 +58,7 @@ The next sections describe some options:
[cols="1,1",frame=none,grid=none,stripes=none]
|===
| <<libs.adoc#,Lib Directories>>: Plugins as libraries on the filesystem.
+| <<solr-modules.adoc#,Solr Modules>>: Loading bundled modules.
| <<package-manager.adoc#,Package Management>>: Package-based plugins.
| <<cluster-plugins.adoc#,Cluster Plugins>>: Cluster-level plugins.
| <<replica-placement-plugins.adoc#,Replica Placement Plugins>>: Plugins
specifically for replica placement.
