This is an automated email from the ASF dual-hosted git repository. houston pushed a commit to branch jira/solr-15556-antora in repository https://gitbox.apache.org/repos/asf/solr.git
commit 618367ad962fa6cd01448faf1c99f69362daf3cd Merge: 3e8c4fc cb2e58f Author: Houston Putman <[email protected]> AuthorDate: Tue Feb 1 12:14:24 2022 -0500 Merge remote-tracking branch 'apache/main' into jira/solr-15556-antora gradle/documentation/render-javadoc.gradle | 17 + gradle/validation/forbidden-apis.gradle | 7 +- gradle/validation/rat-sources.gradle | 6 + gradle/validation/spotless.gradle | 1 + gradle/validation/validate-source-patterns.gradle | 1 - settings.gradle | 1 + solr/CHANGES.txt | 2 + solr/core/build.gradle | 29 +- .../org/apache/solr/core/DirectoryFactory.java | 10 +- .../java/org/apache/solr/update/UpdateHandler.java | 12 +- solr/core/src/test-files/core-site.xml | 2 +- .../org/apache/solr/TestSolrCoreProperties.java | 2 +- .../apache/solr/cloud/BasicDistributedZk2Test.java | 430 +------ .../apache/solr/cloud/BasicDistributedZkTest.java | 1321 +------------------- .../solr/cloud/ChaosMonkeyNothingIsSafeTest.java | 312 +---- .../solr/cloud/ChaosMonkeySafeLeaderTest.java | 192 +-- .../org/apache/solr/cloud/MoveReplicaTest.java | 346 +---- .../test/org/apache/solr/cloud/RecoveryZkTest.java | 133 +- .../solr/cloud/RestartWhileUpdatingTest.java | 173 +-- .../test/org/apache/solr/cloud/SyncSliceTest.java | 282 +---- .../TestSolrCloudWithSecureImpersonation.java | 4 +- .../cloud/TlogReplayBufferedWhileIndexingTest.java | 118 +- .../apache/solr/cloud/UnloadDistributedZkTest.java | 339 +---- .../CollectionsAPIDistributedZkTest.java | 619 +-------- .../apache/solr/core/DirectoryFactoriesTest.java | 2 - .../org/apache/solr/core/DirectoryFactoryTest.java | 1 - .../solr/core/TestBackupRepositoryFactory.java | 11 +- .../org/apache/solr/handler/TestRestoreCore.java | 34 +- .../hadoop/HadoopAuthFakeGroupMapping.java} | 4 +- .../solr/security/hadoop/HadoopTestUtil.java | 58 + .../hadoop/TestDelegationWithHadoopAuth.java | 3 +- .../hadoop/TestImpersonationWithHadoopAuth.java | 3 +- .../hadoop/TestSolrCloudWithHadoopAuthPlugin.java | 3 +- .../security/hadoop/TestZkAclsWithHadoopAuth.java | 3 +- .../apache/solr/update/SolrIndexConfigTest.java | 3 +- solr/licenses/checker-qual-3.10.0.jar.sha1 | 1 - solr/licenses/checker-qual-3.19.0.jar.sha1 | 1 + solr/modules/hdfs/README.md | 29 + solr/modules/hdfs/build.gradle | 99 ++ .../org/apache/solr/core/HdfsDirectoryFactory.java | 22 +- .../backup/repository/HdfsBackupRepository.java | 6 - .../org/apache/solr/index/hdfs/CheckHdfsIndex.java | 0 .../org/apache/solr/index/hdfs/package-info.java | 0 .../apache/solr/store/blockcache/BlockCache.java | 0 .../solr/store/blockcache/BlockCacheKey.java | 0 .../solr/store/blockcache/BlockCacheLocation.java | 0 .../solr/store/blockcache/BlockDirectory.java | 0 .../solr/store/blockcache/BlockDirectoryCache.java | 0 .../apache/solr/store/blockcache/BlockLocks.java | 0 .../apache/solr/store/blockcache/BufferStore.java | 0 .../org/apache/solr/store/blockcache/Cache.java | 0 .../solr/store/blockcache/CachedIndexOutput.java | 0 .../store/blockcache/CustomBufferedIndexInput.java | 0 .../org/apache/solr/store/blockcache/Metrics.java | 0 .../blockcache/ReusedBufferedIndexOutput.java | 0 .../org/apache/solr/store/blockcache/Store.java | 0 .../apache/solr/store/blockcache/package-info.java | 0 .../org/apache/solr/store/hdfs/HdfsDirectory.java | 6 - .../org/apache/solr/store/hdfs/HdfsFileWriter.java | 2 - .../solr/store/hdfs/HdfsLocalityReporter.java | 4 - .../apache/solr/store/hdfs/HdfsLockFactory.java | 4 - .../org/apache/solr/store/hdfs/package-info.java | 0 .../org/apache/solr/update/HdfsTransactionLog.java | 2 - .../java/org/apache/solr/update/HdfsUpdateLog.java | 2 - .../src/java/org/apache/solr/util/FSHDFSUtils.java | 0 .../src/java/org/apache/solr/util/HdfsUtil.java | 0 .../hdfs}/src/test-files/core-site.xml | 2 +- .../src/test/org/apache/hadoop/fs/FileUtil.java | 2 +- .../src/test/org/apache/hadoop/fs/HardLink.java | 0 .../org/apache/hadoop/fs/RawLocalFileSystem.java | 0 .../datanode/fsdataset/impl/BlockPoolSlice.java | 2 +- .../server/namenode/NameNodeResourceChecker.java | 0 .../test/org/apache/hadoop/http/HttpServer2.java | 2 +- .../src/test/org/apache/hadoop/package-info.java | 0 .../test/org/apache/hadoop/util/DiskChecker.java | 2 +- .../solr/cloud/MoveReplicaHDFSFailoverTest.java | 0 .../org/apache/solr/cloud/MoveReplicaHDFSTest.java | 15 +- .../cloud/SharedFSAutoReplicaFailoverTest.java | 0 .../HdfsCloudIncrementalBackupTest.java | 0 .../HdfsCollectionsAPIDistributedZkTest.java | 3 +- .../collections/TestHdfsCloudBackupRestore.java | 0 .../solr/cloud/hdfs/HDFSCollectionsAPITest.java | 0 .../cloud/hdfs/HdfsBasicDistributedZk2Test.java | 6 +- .../cloud/hdfs/HdfsBasicDistributedZkTest.java | 16 +- .../hdfs/HdfsChaosMonkeyNothingIsSafeTest.java | 12 +- .../cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java | 10 +- .../solr/cloud/hdfs/HdfsFakeGroupMapping.java} | 6 +- .../apache/solr/cloud/hdfs/HdfsNNFailoverTest.java | 5 +- .../solr/cloud/hdfs/HdfsRecoverLeaseTest.java | 0 .../apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java | 7 +- .../cloud/hdfs/HdfsRestartWhileUpdatingTest.java | 7 +- .../apache/solr/cloud/hdfs/HdfsSyncSliceTest.java | 4 +- .../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 32 +- .../apache/solr/cloud/hdfs/HdfsThreadLeakTest.java | 0 .../HdfsTlogReplayBufferedWhileIndexingTest.java | 8 +- .../cloud/hdfs/HdfsUnloadDistributedZkTest.java | 4 +- .../hdfs/HdfsWriteToMultipleCollectionsTest.java | 5 +- .../org/apache/solr/cloud/hdfs/StressHdfsTest.java | 5 +- .../apache/solr/core/HdfsDirectoryFactoryTest.java | 0 .../HdfsBackupRepositoryIntegrationTest.java | 0 .../repository/HdfsBackupRepositoryTest.java | 0 .../solr/handler/TestHdfsBackupRestoreCore.java | 2 +- .../apache/solr/index/hdfs/CheckHdfsIndexTest.java | 0 .../org/apache/solr/search/TestRecoveryHdfs.java | 0 .../solr/store/blockcache/BlockCacheTest.java | 0 .../solr/store/blockcache/BlockDirectoryTest.java | 8 +- .../solr/store/blockcache/BufferStoreTest.java | 2 +- .../apache/solr/store/hdfs/HdfsDirectoryTest.java | 0 .../solr/store/hdfs/HdfsLockFactoryTest.java | 0 .../org/apache/solr/update/TestHdfsUpdateLog.java | 0 solr/packaging/build.gradle | 1 + .../pages/index-location-format.adoc | 2 +- .../deployment-guide/pages/backup-restore.adoc | 9 +- .../deployment-guide/pages/solr-on-hdfs.adoc | 19 +- .../pages/major-changes-in-solr-9.adoc | 5 +- .../AbstractBasicDistributedZk2TestBase.java} | 188 ++- .../cloud/AbstractBasicDistributedZkTestBase.java} | 236 ++-- .../AbstractChaosMonkeyNothingIsSafeTestBase.java} | 110 +- .../AbstractChaosMonkeySafeLeaderTestBase.java} | 65 +- .../solr/cloud/AbstractMoveReplicaTestBase.java} | 2 +- .../solr/cloud/AbstractRecoveryZkTestBase.java} | 24 +- .../AbstractRestartWhileUpdatingTestBase.java} | 69 +- .../solr/cloud/AbstractSyncSliceTestBase.java} | 6 +- ...ctTlogReplayBufferedWhileIndexingTestBase.java} | 48 +- .../AbstractUnloadDistributedZkTestBase.java} | 25 +- .../cloud/FullThrottleStoppableIndexingThread.java | 0 ...stractCollectionsAPIDistributedZkTestBase.java} | 6 +- .../apache/solr/handler/BackupRestoreUtils.java | 0 .../apache/solr/handler/BackupStatusChecker.java | 6 +- .../apache/solr/handler/TestRestoreCoreUtil.java | 54 + .../src/java/org/apache/solr/handler/package.html | 24 + .../solr/schema/MockExchangeRateProvider.java | 0 .../org/apache/solr/search/facet/DebugAgg.java | 0 .../solr/search/function/NvlValueSourceParser.java | 0 .../similarities/CustomSimilarityFactory.java | 0 .../similarities/MockConfigurableSimilarity.java | 0 .../org/apache/solr/util/MockCoreContainer.java | 0 versions.lock | 4 +- 138 files changed, 912 insertions(+), 4820 deletions(-) diff --cc solr/solr-ref-guide/modules/configuration-guide/pages/index-location-format.adoc index 652f198,0000000..f577db4 mode 100644,000000..100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-location-format.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-location-format.adoc @@@ -1,65 -1,0 +1,65 @@@ += Index Location and Format +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Where and how Solr stores its indexes are configurable options. + +== Specifying a Location for Index Data with the dataDir Parameter + +By default, Solr stores its index data in a directory called `/data` under the core's instance directory (`instanceDir`). +If you would like to specify a different directory for storing index data, you can configure the `dataDir` in the `core.properties` file for the core, or use the `<dataDir>` parameter in the `solrconfig.xml` file. +You can specify another directory either with an absolute path or a pathname relative to the instanceDir of the SolrCore. +For example: + +[source,xml] +---- +<dataDir>/solr/data/${solr.core.name}</dataDir> +---- + +The `${solr.core.name}` substitution will cause the name of the current core to be substituted, which results in each core's data being kept in a separate subdirectory. + +If you are using xref:deployment-guide:user-managed-index-replication.adoc[] to replicate the Solr index, then the `<dataDir>` directory should correspond to the index directory used in the replication configuration. + +NOTE: If the environment variable `SOLR_DATA_HOME` is defined, or if `solr.data.home` is configured for your DirectoryFactory, or if `solr.xml` contains an +element `<solrDataHome>` then the location of data directory will be `<SOLR_DATA_HOME>/<instance_name>/data`. + +== Specifying the DirectoryFactory For Your Index + +The default {solr-javadocs}/core/org/apache/solr/core/NRTCachingDirectoryFactory.html[`solr.NRTCachingDirectoryFactory`] is filesystem based, and tries to pick the best implementation for the current JVM and platform. +You can force a particular implementation and/or configuration options by specifying {solr-javadocs}/core/org/apache/solr/core/MMapDirectoryFactory.html[`solr.MMapDirectoryFactory`] or {solr-javadocs}/core/org/apache/solr/core/NIOFSDirectoryFactory.html[`solr.NIOFSDirectoryFactory`]. + +[source,xml] +---- +<directoryFactory name="DirectoryFactory" + class="solr.MMapDirectoryFactory"> + <bool name="preload">true</bool> +</directoryFactory> +---- + +The {solr-javadocs}/core/org/apache/solr/core/RAMDirectoryFactory.html[`solr.RAMDirectoryFactory`] is memory based, not persistent, and does not work with replication. +Use this DirectoryFactory to store your index in RAM. + +[source,xml] +---- +<directoryFactory class="org.apache.solr.core.RAMDirectoryFactory"/> +---- + +[NOTE] +==== - If you are using Hadoop and would like to store your indexes in HDFS, you should use the {solr-javadocs}/core/org/apache/solr/core/HdfsDirectoryFactory.html[`solr.HdfsDirectoryFactory`] instead of either of the above implementations. ++If you are using Hadoop and would like to store your indexes in HDFS, you should use the {solr-javadocs}/modules/hdfs/org/apache/solr/core/HdfsDirectoryFactory.html[`solr.HdfsDirectoryFactory`] instead of either of the above implementations. +For more details, see the section xref:deployment-guide:solr-on-hdfs.adoc[]. +==== diff --cc solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc index 165d74f,0000000..c6eef7a mode 100644,000000..100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc @@@ -1,793 -1,0 +1,796 @@@ += Backup and Restore +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +If you are worried about data loss, and of course you _should_ be, you need a way to back up your Solr indexes so that you can recover quickly in case of catastrophic failure. + +Solr provides two approaches to backing up and restoring Solr cores or collections, depending on how you are running Solr. +If you run a SolrCloud cluster, you will use the Collections API. +If you run a user-managed cluster or a single-node installation, you will use the replication handler. + +[NOTE] +==== +Backups (and Snapshots) capture data that has been xref:configuration-guide:commits-transaction-logs.adoc#hard-commits-vs-soft-commits[hard committed]. +Committing changes using `softCommit=true` may result in changes that are visible in search results but not included in subsequent backups. + +Likewise, committing changes using `openSearcher=false` may result in changes committed to disk and included in subsequent backups, even if they are not currently visible in search results. +==== + +== SolrCloud Clusters + +Support for backups in SolrCloud is provided with the xref:collection-management.adoc#backup[Collections API]. +This allows the backups to be generated across multiple shards, and restored to the same number of shards and replicas as the original collection. + +NOTE: SolrCloud Backup/Restore requires a shared file system mounted at the same path on all nodes, or HDFS. + +Four different API commands are supported: + +* `action=BACKUP`: This command backs up Solr indexes and configurations. +More information is available in the section xref:collection-management.adoc#backup[Backup Collection]. +* `action=RESTORE`: This command restores Solr indexes and configurations. +More information is available in the section xref:collection-management.adoc#restore[Restore Collection]. +* `action=LISTBACKUP`: This command lists the backup points available at a specified location, displaying metadata for each. +More information is available in the section xref:collection-management.adoc#listbackup[List Backups]. +* `action=DELETEBACKUP`: This command allows deletion of backup files or whole backups. +More information is available in the section xref:collection-management.adoc#deletebackup[Delete Backups]. + +== User-Managed Clusters and Single-Node Installations + +Backups and restoration uses Solr's replication handler. +Out of the box, Solr includes implicit support for replication so this API can be used. +Configuration of the replication handler can, however, be customized by defining your own replication handler in `solrconfig.xml`. +For details on configuring the replication handler, see the section xref:user-managed-index-replication.adoc#configuring-the-replicationhandler[Configuring the ReplicationHandler]. + +=== Backup API + +The `backup` API requires sending a command to the `/replication` handler to back up the system. + +You can trigger a back-up with an HTTP command like this (replace "gettingstarted" with the name of the core you are working with): + +.Backup API Example +[source,text] +---- +http://localhost:8983/solr/gettingstarted/replication?command=backup +---- + +The `backup` command is an asynchronous call, and it will represent data from the latest index commit point. +All indexing and search operations will continue to be executed against the index as usual. + +Only one backup call can be made against a core at one time. +While an ongoing backup operation is happening subsequent calls for restoring will throw an exception. + +The backup request can also take the following additional parameters: + +`location`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The path where the backup will be created. +If the path is not absolute then the backup path will be relative to Solr's instance directory. + +`name`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The snapshot will be created in a directory called `snapshot.<name>`. +If a name is not specified then the directory name will have the following format: `snapshot.<_yyyyMMddHHmmssSSS_>`. + +`numberToKeep`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The number of backups to keep. +If `maxNumberOfBackups` has been specified on the replication handler in `solrconfig.xml`, `maxNumberOfBackups` is always used and attempts to use `numberToKeep` will cause an error. +Also, this parameter is not taken into consideration if the backup name is specified. +More information about `maxNumberOfBackups` can be found in the section xref:user-managed-index-replication.adoc#configuring-the-replicationhandler[Configuring the ReplicationHandler]. + +`repository`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The name of the repository to be used for the backup. +If no repository is specified then the local filesystem repository will be used automatically. + +`commitName`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The name of the commit which was used while taking a snapshot using the CREATESNAPSHOT command. + +=== Backup Status + +The `backup` operation can be monitored to see if it has completed by sending the `details` command to the `/replication` handler, as in this example: + +.Status API Example +[source,text] +---- +http://localhost:8983/solr/gettingstarted/replication?command=details&wt=xml +---- + +.Output Snippet +[source,xml] +---- +<lst name="backup"> + <str name="startTime">Sun Apr 12 16:22:50 DAVT 2015</str> + <int name="fileCount">10</int> + <str name="status">success</str> + <str name="snapshotCompletedAt">Sun Apr 12 16:22:50 DAVT 2015</str> + <str name="snapshotName">my_backup</str> +</lst> +---- + +If it failed then a `snapShootException` will be sent in the response. + +=== Restore API + +Restoring the backup requires sending the `restore` command to the `/replication` handler, followed by the name of the backup to restore. + +You can restore from a backup with a command like this: + +.Example Usage +[source,text] +---- +http://localhost:8983/solr/gettingstarted/replication?command=restore&name=backup_name +---- + +This will restore the named index snapshot into the current core. +Searches will start reflecting the snapshot data once the restore is complete. + +The `restore` request can take these additional parameters: + +`location`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The location of the backup snapshot file. +If not specified, it looks for backups in Solr's data directory. + +`name`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The name of the backup index snapshot to be restored. +If the name is not provided it looks for backups with `snapshot.<timestamp>` format in the location directory. +It picks the latest timestamp backup in that case. + +`repository`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The name of the repository to be used for the backup. +If no repository is specified then the local filesystem repository will be used automatically. + +The `restore` command is an asynchronous call. +Once the restore is complete the data reflected will be of the backed up index which was restored. + +Only one `restore` call can can be made against a core at one point in time. +While an ongoing restore operation is happening subsequent calls for restoring will throw an exception. + +=== Restore Status API + +You can also check the status of a `restore` operation by sending the `restorestatus` command to the `/replication` handler, as in this example: + +.Status API Example +[source,text] +---- +http://localhost:8983/solr/gettingstarted/replication?command=restorestatus&wt=xml +---- + +.Status API Output +[source,xml] +---- +<response> + <lst name="responseHeader"> + <int name="status">0</int> + <int name="QTime">0</int> + </lst> + <lst name="restorestatus"> + <str name="snapshotName">snapshot.<name></str> + <str name="status">success</str> + </lst> +</response> +---- + +The status value can be "In Progress", "success" or "failed". +If it failed then an "exception" will also be sent in the response. + +=== Create Snapshot API + +The snapshot functionality is different from the backup functionality as the index files aren't copied anywhere. +The index files are snapshotted in the same index directory and can be referenced while taking backups. + +You can trigger a snapshot command with an HTTP command like this (replace "techproducts" with the name of the core you are working with): + +.Create Snapshot API Example +[source,text] +---- +http://localhost:8983/solr/admin/cores?action=CREATESNAPSHOT&core=techproducts&commitName=commit1 +---- + +The `CREATESNAPSHOT` request parameters are: + +`commitName`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The name to store the snapshot as. + +`core`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The name of the core to perform the snapshot on. + +`async`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Request ID to track this action which will be processed asynchronously. + +=== List Snapshot API + +The `LISTSNAPSHOTS` command lists all the taken snapshots for a particular core. + +You can trigger a list snapshot command with an HTTP command like this (replace "techproducts" with the name of the core you are working with): + +.List Snapshot API +[source,text] +---- +http://localhost:8983/solr/admin/cores?action=LISTSNAPSHOTS&core=techproducts&commitName=commit1 +---- + +The list snapshot request parameters are: + +`core`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The name of the core to whose snapshots we want to list. + +`async`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Request ID to track this action which will be processed asynchronously. + +=== Delete Snapshot API + +The `DELETESNAPSHOT` command deletes a snapshot for a particular core. + +You can trigger a delete snapshot with an HTTP command like this (replace "techproducts" with the name of the core you are working with): + +.Delete Snapshot API Example +[source,text] +---- +http://localhost:8983/solr/admin/cores?action=DELETESNAPSHOT&core=techproducts&commitName=commit1 +---- + +The delete snapshot request parameters are: + +`commitName`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Specify the commit name to be deleted. + +`core`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The name of the core whose snapshot we want to delete. + +`async`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Request ID to track this action which will be processed asynchronously. + +== Backup/Restore Storage Repositories + +Solr provides a repository abstraction to allow users to backup and restore their data to a variety of different storage systems. +For example, a Solr cluster running on a local filesystem (e.g., EXT3) can store backup data on the same disk, on a remote network-mounted drive, in HDFS, or even in some popular "cloud storage" providers, depending on the 'repository' implementation chosen. +Solr offers three different repository implementations out of the box (`LocalFileSystemRepository`, `HdfsBackupRepository`, and `GCSBackupRepository`), and allows users to create plugins for their own storage systems as needed. + +Users can define any number of repositories in their `solr.xml` file. +The backup and restore APIs described above allow users to select which of these definitions they want to use at runtime via the `repository` parameter. +When no `repository` parameter is specified, the local filesystem repository is used as a default. + +Repositories are defined by a `<repository>` tag nested under a `<backup>` parent tag. +All `<repository>` tags must have a `name` attribute (defines the identifier that users can reference later to select this repository) and a `class` attribute (containing the full Java classname that implements the repository). +They may also have a boolean `default` attribute, which may be `true` on at most one repository definition. +Any children under the `<repository>` tag are passed as additional configuration to the repository, allowing repositories to read their own implementation-specific configuration. + +Information on each of the repository implementations provided with Solr is provided below. + +=== LocalFileSystemRepository + +LocalFileSystemRepository stores and retrieves backup files anywhere on the accessible filesystem. +Files can be stored on "local" disk, or on network-mounted drives that appear local to the filesystem. + +WARNING: SolrCloud administrators looking to use LocalFileSystemRepository in tandem with network drives should be careful to make the drive available at the same location on each Solr node. +Strictly speaking, the mount only needs to be present on the node doing the backup (or restore), and on the node currently serving as the "Overseer". +However since the "overseer" role often moves from node to node in a cluster, it is generally recommended that backup drives be added to all nodes uniformly. + +A LocalFileSystemRepository instance is used as a default by any backup and restore commands that don't explicitly provide a `repository` parameter or have a default specified in `solr.xml`. + +LocalFileSystemRepository accepts the following configuration option: + +`location`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A valid file path (accessible to Solr locally) to use for backup storage and retrieval. +Used as a fallback when user's don't provide a `location` parameter in their Backup or Restore API commands + +An example configuration using this property can be found below. + +[source,xml] +---- +<backup> + <repository name="local_repo" class="org.apache.solr.core.backup.repository.LocalFileSytemRepository"> + <str name="location">/solr/backup_data</str> + </repository> +</backup> +---- + + +=== HdfsBackupRepository + +Stores and retrieves backup files from HDFS directories. - - WARNING: HdfsBackupRepository is deprecated and may be removed or relocated in a subsequent version of Solr. ++This is provided in the `hdfs` xref:configuration-guide:solr-modules.adoc[Solr Module]. ++This plugin must first be xref:configuration-guide:solr-plugins.adoc#installing-plugins[installed] before using. + +HdfsBackupRepository accepts the following configuration options: + +`solr.hdfs.buffer.size`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `4096` kilobytes +|=== ++ +The size, in bytes, of the buffer used to transfer data to and from HDFS. +Better throughput is often attainable with a larger buffer, where memory allows. + +`solr.hdfs.home`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +A HDFS URI in the format `hdfs://<host>:<port>/<hdfsBaseFilePath>` that points Solr to the HDFS cluster to store (or retrieve) backup files on. + +`solr.hdfs.permissions.umask-mode`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A permission umask used when creating files in HDFS. + +`location`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A valid directory path on the HDFS cluster to use for backup storage and retrieval. +Used as a fallback when users don't provide a `location` parameter in their Backup or Restore API commands. + +An example configuration using these properties can be found below: + +[source,xml] +---- +<backup> + <repository name="hdfs" class="org.apache.solr.core.backup.repository.HdfsBackupRepository" default="false"> + <str name="solr.hdfs.home">hdfs://some_hdfs_host:1234/solr/backup/data</str> + <int name="solr.hdfs.buffer.size">8192</int> + <str name="solr.hdfs.permissions.umask-mode">0022</str> + <str name="location">/default/hdfs/backup/location</str> + </repository> +</backup> +---- + +=== GCSBackupRepository + - Stores and retrieves backup files in a Google Cloud Storage ("GCS") bucket. This plugin must first be xref:configuration-guide:solr-plugins.adoc#installing-plugins[installed] before using. ++Stores and retrieves backup files in a Google Cloud Storage ("GCS") bucket. ++This is provided in the `gcs-repository` xref:configuration-guide:solr-modules.adoc[Solr Module]. ++This plugin must first be xref:configuration-guide:solr-plugins.adoc#installing-plugins[installed] before using. + +GCSBackupRepository accepts the following options for overall configuration: + +`gcsBucket`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: _see description_ +|=== ++ +The GCS bucket to read and write all backup files to. +If not specified, GCSBackupRepository will use the value of the `GCS_BUCKET` environment variable. +If both values are absent, the value `solrBackupsBucket` will be used as a default. + +`gcsCredentialPath`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: _see description_ +|=== ++ +A path on the local filesystem (accessible by Solr) to a https://cloud.google.com/iam/docs/creating-managing-service-account-keys[Google Cloud service account key] file. +If not specified, GCSBackupRepository will use the value of the `GCS_CREDENTIAL_PATH` environment variable. +If both values are absent and Solr is running inside GCP, the GCS client will attempt to authenticate using GCP's "Compute Engine Metadata Serve"r or https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity[Workload Identity] features. +If both values are absent and Solr is running outside of GCP, it will be unable to authenticate and any backup or restore operations will fail. + +`location`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A valid "directory" path in the given GCS bucket to us for backup storage and retrieval. +(GCS uses a flat storage model, but Solr's backup functionality names blobs in a way that approximates hierarchical directory storage.) +Used as a fallback when user's don't provide a `location` parameter in their Backup or Restore API commands. + +In addition to these properties for overall configuration, GCSBackupRepository gives users detailed control over the client used to communicate with GCS. +These properties are unlikely to interest most users, but may be valuable for those looking to micromanage performance or subject to a flaky network. + +GCSBackupRepository accepts the following advanced client-configuration options: + +`gcsWriteBufferSizeBytes`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `16777216` bytes (16 MB) +|=== ++ +The buffer size, in bytes, to use when sending data to GCS. + +`gcsReadBufferSizeBytes`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `2097152` bytes (2 MB) +|=== ++ +The buffer size, in bytes, to use when copying data from GCS. + +`gcsClientHttpConnectTimeoutMillis`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `2000` milliseconds +|=== ++ +The connection timeout, in milliseconds, for all HTTP requests made by the GCS client. +`0` may be used to request an infinite timeout. +A negative integer, or not specifying a value at all, will result in the default value. + +`gcsClientHttpReadTimeoutMillis`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `20000` milliseconds +|=== ++ +The read timeout, in milliseconds, for reading data on an established connection. +`0` may be used to request an infinite timeout. +A negative integer, or not specifying a value at all, will result in the default value. + +`gcsClientMaxRetries`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `10` +|=== ++ +The maximum number of times to retry an operation upon failure. +The GCS client will retry operations until this value is reached, or the time spent across all attempts exceeds `gcsClientMaxRequestTimeoutMillis`. +`0` may be used to specify that no retries should be done. + +`gcsClientMaxRequestTimeoutMillis`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `30000` milliseconds +|=== ++ +The maximum amount of time to spend on all retries of an operation that has failed. +The GCS client will retry operations until either this timeout has been reached, or until `gcsClientMaxRetries` attempts have failed. + +`gcsClientHttpInitialRetryDelayMillis`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `1000` milliseconds +|=== ++ +The time, in milliseconds, to delay before the first retry of a HTTP request that has failed. +This value also factors in to subsequent retries - see the `gcsClientHttpRetryDelayMultiplier` description below for more information. +If `gcsClientMaxRetries` is `0`, this property is ignored as no retries are attempted. + +`gcsClientHttpRetryDelayMultiplier`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `1.0` +|=== ++ +A floating-point multiplier used to scale the delay between each successive retry of a failed HTTP request.. +The greater this number is, the more quickly the retry delay compounds and scales. ++ +Under the covers, the GSC client uses an exponential backoff strategy between retries, governed by the formula: stem:[gcsClientH\t\tpInitialRetryDelayMillis*(gcsClientH\t\tpRetryDelayM\u\l\tiplier)^(retryNum-1)]. +The first retry will have a delay of stem:[gcsClientH\t\tpInitialRetryDelayMillis], the second a delay of stem:[gcsClientH\t\tpInitialRetryDelayMillis * gcsClientH\t\tpRetryDelayM\u\l\tiplier], the third a delay of stem:[gcsClientH\t\tpInitialRetryDelayMillis * gcsClientH\t\tpRetryDelayM\u\l\tiplier^2], and so on. ++ +If not specified the value `1.0` is used by default, ensuring that `gcsClientHttpInitialRetryDelayMillis` is used between each retry attempt. + +`gcsClientHttpMaxRetryDelayMillis`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `30000` milliseconds +|=== ++ +The maximum delay, in milliseconds, between retry attempts on a failed HTTP request. +This is commonly used to cap the exponential growth in retry-delay that occurs over multiple attempts. +See the `gcsClientHttpRetryDelayMultiplier` description above for more information on how each delay is calculated when not subject to this maximum. + +`gcsClientRpcInitialTimeoutMillis`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `10000` milliseconds +|=== ++ +The time, in milliseconds, to wait on a RPC request before timing out. +This value also factors in to subsequent retries - see the `gcsClientRpcTimeoutMultiplier` description below for more information. +If `gcsClientMaxRetries` is `0`, this property is ignored as no retries are attempted. + +`gcsClientRpcTimeoutMultiplier`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `1.0` +|=== ++ +A floating-point multiplier used to scale the timeout on each successive attempt of a failed RPC request. +The greater this number is, the more quickly the timeout compounds and scales. ++ +Under the covers, the GSC client uses an exponential backoff strategy for RPC timeouts, governed by the formula: stem:[gcsClientRpcInitialTimeoutMillis*(gcsClientRpcTimeoutM\u\l\tiplier)^(retryNum-1)]. +The first retry will have a delay of stem:[gcsClientRpcInitialTimeoutMillis], the second a delay of stem:[gcsClientRpcInitialTimeoutMillis * gcsClientRpcTimeoutM\u\l\tiplier], the third a delay of stem:[gcsClientRpcInitialTimeoutMillis * gcsClientRpcTimeoutM\u\l\tiplier^2], and so on. ++ +If not specified the value `1.0` is used by default, ensuring that `gcsClientRpcInitialTimeoutMillis` is used on each RPC attempt. + +`gcsClientRpcMaxTimeoutMillis`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `30000` milliseconds +|=== ++ +The maximum timeout, in milliseconds, for retry attempts of a failed RPC request. +This is commonly used to cap the exponential growth in timeout that occurs over multiple attempts. +See the `gcsClientRpcTimeoutMultiplier` description above for more information on how each timeout is calculated when not subject to this maximum. + +An example configuration using the overall and GCS-client properties can be seen below: + +[source,xml] +---- +<backup> + <repository name="gcs_backup" class="org.apache.solr.gcs.GCSBackupRepository" default="false"> + <str name="gcsBucket">solrBackups</str> + <str name="gcsCredentialPath">/local/path/to/credential/file</str> + <str name="location">/default/gcs/backup/location</str> + + <int name="gcsClientMaxRetries">5</int> + <int name="gcsClientHttpInitialHttpDelayMillis">1500</int> + <double name="gcsClientHttpRetryDelayMultiplier">1.5</double> + <int name="gcsClientMaxHttpRetryDelayMillis">10000</int> + </repository> +</backup> +---- + +=== S3BackupRepository + +Stores and retrieves backup files in an Amazon S3 bucket. ++This is provided in the `s3-repository` xref:configuration-guide:solr-modules.adoc[Solr Module]. +This plugin must first be xref:configuration-guide:solr-plugins.adoc#installing-plugins[installed] before using. + +This plugin uses the https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/credentials.html[default AWS credentials provider chain], so ensure that your credentials are set appropriately (e.g., via env var, or in `~/.aws/credentials`, etc.). + +[NOTE] +==== +When using the Backup & Restore Collections API Calls, you can provide a **location** that either starts with `s3://` or not. +Either way, if your **location** (or s3 object prefix) starts with a `/`, it will be removed automatically. +The repository does not allow backup locations that begin with a `/`. +==== + +An example configuration to enable S3 backups and restore can be seen below: + +[source,xml] +---- +<backup> + <repository name="s3" class="org.apache.solr.s3.S3BackupRepository" default="false"> + <str name="s3.bucket.name">my-s3-bucket</str> + <str name="s3.region">us-west-2</str> + </repository> +</backup> +---- + +S3BackupRepository accepts the following options (in `solr.xml`) for overall configuration: + +`s3.bucket.name`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The S3 bucket to read and write all backup files to. Can be overridden by setting `S3_BUCKET_NAME` environment variable. + +`s3.profile`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A profile to load AWS settings for from config files. +Profiles allow for independent settings for multiple S3Repositories. +Can be overridden by setting `AWS_PROFILE` environment variable or `-Daws.profile` system property. +For more information on setting configuration per-profile, refer to the https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html#file-format-config[AWS Java SDK documentation] + +`s3.region`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +A valid Amazon S3 region string where your bucket is provisioned. You must have read and write permissions for this bucket. +For a full list of regions, please reference the https://docs.aws.amazon.com/general/latest/gr/s3.html[S3 documentation]. +Can be overridden by setting `S3_REGION` environment variable, or setting the region in the AWS Configuration file. + +`s3.endpoint`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Explicit S3 endpoint. Not needed under normal operations when using AWS S3 (the S3 client can infer the endpoint from the `s3.region`). +This parameter is helpful if using a mock S3 framework and want to explicitly override where S3 requests are routed, such as when using S3Mock. +Can be overridden by setting `S3_ENDPOINT` environment variable. + +[NOTE] +==== +You can use the `s3.endpoint` option to use this BackupRepository with _s3-compatible_ endpoints. +Beware that not all _s3-compatible_ endpoints will work with the S3BackupRepository. +Minio is an example of an _s3-compatible_ endpoint that does not work with the S3BackupRepository. +The S3BackupRepository is only guaranteed to be compatible with AWS S3 and S3Mock. +==== + +`s3.proxy.url`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Proxy url for the S3 client to route requests through, if desired. +The url should include `<scheme>://<hostname>:<port>`, however port and scheme _may_ be inferred if missing. ++ +If used, this will override any system proxy settings that are set. +There is no need to disable the `s3.proxy.useSystemSettings` option. +If you need to use a proxy `username`, `password` or `nonProxyHosts`, please use the system properties listed below. + +`s3.proxy.useSystemSettings`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: true +|=== ++ +By default use the system proxy settings if they are set when communicating with the S3 server. +The supported proxy system properties are: ++ +* `http.proxyHost` +* `http.proxyPort` +* `http.nonProxyHosts` +* `http.proxyUser` +* `http.proxyPassword` + +`s3.retries.disable`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: false +|=== ++ +Disable retries for all S3 operations. This is not recommended. + +==== S3 Client Configuration + +The AWS Java SDKs provide many ways of setting the configuration for an S3 Client. +The Solr S3Repository allows these configurations to be set via: + +* Environment Variables +* Java System Properties +* AWS Configuration File (possibly per-profile) + +https://docs.aws.amazon.com/sdkref/latest/guide/settings-global.html[These options] include: + +* Region +* Access Keys +* Retries +** RetryMode (`LEGACY`, `STANDARD`, `ADAPTIVE`) +** Max Attempts diff --cc solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc index 131d62b,0000000..c02b89e mode 100644,000000..100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc @@@ -1,277 -1,0 +1,290 @@@ += Solr on HDFS +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + - WARNING: Storing indexes in HDFS is deprecated and may be be removed in 9.0. - This functionality may be moved to a 3rd-party plugin in the future. + - Solr has support for writing and reading its index and transaction log files to the HDFS distributed filesystem. ++The Solr HDFS module has support for writing and reading its index and transaction log files to the HDFS distributed filesystem. ++This plugin must first be xref:configuration-guide:solr-plugins.adoc#installing-plugins[installed] before using. ++This module needs to be installed as lib, it is planned to evolve to a Solr package in a later release. + +This does not use Hadoop MapReduce to process Solr data, rather it only uses the HDFS filesystem for index and transaction log file storage. + +To use HDFS rather than a local filesystem, you must be using Hadoop 2.x and you will need to instruct Solr to use the `HdfsDirectoryFactory`. +There are also several additional parameters to define. +These can be set in one of three ways: + +* Pass JVM arguments to the `bin/solr` script. +These would need to be passed every time you start Solr with `bin/solr`. +* Modify `solr.in.sh` (or `solr.in.cmd` on Windows) to pass the JVM arguments automatically when using `bin/solr` without having to set them manually. +* Define the properties in `solrconfig.xml`. +These configuration changes would need to be repeated for every collection, so is a good option if you only want some of your collections stored in HDFS. + +== Starting Solr on HDFS + +=== User-Managed Cluters and Single-Node Installations + +For user-managed clusters or single-node Solr installations, there are a few parameters you should modify before starting Solr. +These can be set in `solrconfig.xml` (more on that <<HdfsDirectoryFactory Parameters,below>>), or passed to the `bin/solr` script at startup. + +* You need to use an `HdfsDirectoryFactory` and a data directory in the form `hdfs://host:port/path` +* You need to specify an `updateLog` location in the form `hdfs://host:port/path` +* You should specify a lock factory type of `'hdfs'` or none. + +If you do not modify `solrconfig.xml`, you can instead start Solr on HDFS with the following command: + +[source,bash] +---- +bin/solr start -Dsolr.directoryFactory=HdfsDirectoryFactory + -Dsolr.lock.type=hdfs + -Dsolr.data.dir=hdfs://host:port/path + -Dsolr.updatelog=hdfs://host:port/path +---- + +This example will start Solr using the defined JVM properties (explained in more detail <<HdfsDirectoryFactory Parameters,below>>). + +=== SolrCloud Instances + +In SolrCloud mode, it's best to leave the data and update log directories as the defaults Solr comes with and simply specify the `solr.hdfs.home`. +All dynamically created collections will create the appropriate directories automatically under the `solr.hdfs.home` root directory. + +* Set `solr.hdfs.home` in the form `hdfs://host:port/path` +* You should specify a lock factory type of `'hdfs'` or none. + +[source,bash] +---- +bin/solr start -c -Dsolr.directoryFactory=HdfsDirectoryFactory + -Dsolr.lock.type=hdfs + -Dsolr.hdfs.home=hdfs://host:port/path +---- + +This command starts Solr using the defined JVM properties. + + +=== Modifying solr.in.sh (*nix) or solr.in.cmd (Windows) + +The examples above assume you will pass JVM arguments as part of the start command every time you use `bin/solr` to start Solr. +However, `bin/solr` looks for an include file named `solr.in.sh` (`solr.in.cmd` on Windows) to set environment variables. +By default, this file is found in the `bin` directory, and you can modify it to permanently add the `HdfsDirectoryFactory` settings and ensure they are used every time Solr is started. + +For example, to set JVM arguments to always use HDFS when running in SolrCloud mode (as shown above), you would add a section such as this: + +[source,bash] +---- +# Set HDFS DirectoryFactory & Settings +-Dsolr.directoryFactory=HdfsDirectoryFactory \ +-Dsolr.lock.type=hdfs \ +-Dsolr.hdfs.home=hdfs://host:port/path \ +---- + +== The Block Cache + +For performance, the `HdfsDirectoryFactory` uses a Directory that will cache HDFS blocks. +This caching mechanism replaces the standard file system cache that Solr utilizes. +By default, this cache is allocated off-heap. +This cache will often need to be quite large and you may need to raise the off-heap memory limit for the specific JVM you are running Solr in. +For the Oracle/OpenJDK JVMs, the following is an example command-line parameter that you can use to raise the limit when starting Solr: + +[source,bash] +---- +-XX:MaxDirectMemorySize=20g +---- + +== HdfsDirectoryFactory Parameters + +The `HdfsDirectoryFactory` has a number of settings defined as part of the `directoryFactory` configuration. + +=== Solr HDFS Settings + +`solr.hdfs.home`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +A root location in HDFS for Solr to write collection data to. +Rather than specifying an HDFS location for the data directory or update log directory, use this to specify one root location and have everything automatically created within this HDFS location. +The structure of this parameter is `hdfs://host:port/path/solr`. + +=== Block Cache Settings + +`solr.hdfs.blockcache.enabled`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `true` +|=== ++ +Enable the blockcache. + +`solr.hdfs.blockcache.read.enabled`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `true` +|=== ++ +Enable the read cache. + +`solr.hdfs.blockcache.direct.memory.allocation`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `true` +|=== ++ +Enable direct memory allocation. +If this is `false`, heap is used. + +`solr.hdfs.blockcache.slab.count`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `1` +|=== ++ +Number of memory slabs to allocate. +Each slab is 128 MB in size. + +`solr.hdfs.blockcache.global`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `true` +|=== ++ +Enable/Disable using one global cache for all SolrCores. +The settings used will be from the first HdfsDirectoryFactory created. + +=== NRTCachingDirectory Settings + +`solr.hdfs.nrtcachingdirectory.enable`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `true` +|=== ++ +Enable the use of NRTCachingDirectory. + +`solr.hdfs.nrtcachingdirectory.maxmergesizemb`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `16` +|=== ++ +NRTCachingDirectory max segment size for merges. + +`solr.hdfs.nrtcachingdirectory.maxcachedmb`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `192` +|=== ++ +NRTCachingDirectory max cache size. + +=== HDFS Client Configuration Settings + +`solr.hdfs.confdir`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Pass the location of HDFS client configuration files - needed for HDFS HA for example. + +=== Kerberos Authentication Settings + +Hadoop can be configured to use the Kerberos protocol to verify user identity when trying to access core services like HDFS. +If your HDFS directories are protected using Kerberos, then you need to configure Solr's HdfsDirectoryFactory to authenticate using Kerberos in order to read and write to HDFS. +To enable Kerberos authentication from Solr, you need to set the following parameters: + +`solr.hdfs.security.kerberos.enabled`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `false` +|=== ++ +Set to `true` to enable Kerberos authentication. + +`solr.hdfs.security.kerberos.keytabfile`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +A keytab file contains pairs of Kerberos principals and encrypted keys which allows for password-less authentication when Solr attempts to authenticate with secure Hadoop. ++ +This file will need to be present on all Solr servers at the same path provided in this parameter. + +`solr.hdfs.security.kerberos.principal`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The Kerberos principal that Solr should use to authenticate to secure Hadoop; the format of a typical Kerberos V5 principal is: `primary/instance@realm`. + ++== Update Log settings ++When using HDFS to store Solr indexes, it is recommended to also store the transaction logs on HDFS. This can be done by using the `solr.HdfsUpdateLog` update log hander class. ++The solrconfig.xml is often used to define an update log handler class name either using a variable reference or direct specification, for example: ++ ++[source,xml] ++---- ++<updateLog class="${solr.ulog:solr.UpdateLog}"> ++---- ++ ++When specifying a class like this, it needs to be ensured that the correct class name is specified. ++When no class name is specified, Solr automatically picks the correct update log handler class `solr.HdfsUpdateLog` for collections which are configured to use the HdfsDirectory Factory. ++ ++ +== Example solrconfig.xml for HDFS + +Here is a sample `solrconfig.xml` configuration for storing Solr indexes on HDFS: + +[source,xml] +---- +<directoryFactory name="DirectoryFactory" class="solr.HdfsDirectoryFactory"> + <str name="solr.hdfs.home">hdfs://host:port/solr</str> + <bool name="solr.hdfs.blockcache.enabled">true</bool> + <int name="solr.hdfs.blockcache.slab.count">1</int> + <bool name="solr.hdfs.blockcache.direct.memory.allocation">true</bool> + <int name="solr.hdfs.blockcache.blocksperbank">16384</int> + <bool name="solr.hdfs.blockcache.read.enabled">true</bool> + <bool name="solr.hdfs.nrtcachingdirectory.enable">true</bool> + <int name="solr.hdfs.nrtcachingdirectory.maxmergesizemb">16</int> + <int name="solr.hdfs.nrtcachingdirectory.maxcachedmb">192</int> +</directoryFactory> +---- + +If using Kerberos, you will need to add the three Kerberos related properties to the `<directoryFactory>` element in `solrconfig.xml`, such as: + +[source,xml] +---- +<directoryFactory name="DirectoryFactory" class="solr.HdfsDirectoryFactory"> + ... + <bool name="solr.hdfs.security.kerberos.enabled">true</bool> + <str name="solr.hdfs.security.kerberos.keytabfile">/etc/krb5.keytab</str> + <str name="solr.hdfs.security.kerberos.principal">solr/[email protected]</str> +</directoryFactory> +---- diff --cc solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc index 6516845,0000000..e98bf0a mode 100644,000000..100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc @@@ -1,303 -1,0 +1,304 @@@ += Major Changes in Solr 9 +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Solr 9.0 is a major new release of Solr. + +This page highlights the biggest changes, including new features you may want to be aware of, and changes in default behavior and deprecated features that have been removed. + +== Solr 9 Upgrade Planning + +Before starting an upgrade to Solr 9, please take the time to review all information about changes from the version you are currently on up to Solr 9. + +You should also consider all changes that have been made to Solr in any version you have not upgraded to already. For example, if you are currently using Solr 8.1, you should review changes made in all subsequent 8.x releases in addition to changes for 9.0. + +A thorough review of the list in Major Changes in Earlier 8.x Versions as well as the {solr-javadocs}/changes//Changes.html[CHANGES.txt] in your Solr instance will help you plan your migration to Solr 9. + +=== Upgrade Prerequisites + +* LUCENE-8738: Move to Java 11 as minimum Java version. + +* Upgrade all collections in `stateFormat=1` to `stateFormat=2` *before* upgrading to Solr 9, as Solr 9 does not support the older format and no longer supports migrating collections from the older format to the current format (previously known as stateFormat=2). +Upgrade is to be done using Collection API MIGRATESTATEFORMAT action using a previous version of Solr. +See for example https://solr.apache.org/guide/8_8/cluster-node-management.html#migratestateforma[Solr 8.8 Ref Guide]. +// Can't link directly to .adoc file, need to link to 8.something ref guide as MIGRATESTATEFORMAT no longer exists in 9.0. + +* SOLR-12823: Remove `/clusterstate.json` support, i.e., support for collections created with `stateFormat=1` as well as support for Collection API MIGRATESTATEFORMAT action. +Also removes support for cluster property `legacyCloud` (as if always false now). + +* If you're using a SolrJ CloudClient to connect to your Solr cluster, you must be using SolrJ version 8.10 or higher (8.x) when upgrading your SolrCloud from 8.x to 9.x. +Otherwise, SolrJ will not be able to connect to the cluster once it has upgraded to Solr 9. +Once you have upgraded all Solr clusters that the client is connecting to, you can upgrade the SolrJ client to 9.x. + +* If you're using Solr in standalone mode with the xref:query-guide:query-elevation-component.adoc[] with it's elevation file in the data directory, you'll have to move it to the xref:configuration-guide:config-sets.adoc[Configset] instead. +The only reason QEC supported the data directory was to support loading its changes on a commit instead of a more expensive core reload. +That feature now works from the configset dir too. +SolrCloud doesn't support that but may sometime. + +=== Rolling Upgrades + +* SOLR-14702: All references to "master" and "slave" were replaced in the code with "leader" and "follower". +This includes API calls for the replication handler and metrics. +For rolling upgrades into 9.0, you need to be on Solr version 8.7 or greater. Some metrics also changed, alerts and monitors on Solr KPIs that mention "master" or "slave" will also now be "leader" and "follower" + +=== Reindexing After Upgrade + +In Solr 8, it's possible to add docValues to a schema without re-indexing via `UninvertDocValuesMergePolicy`, an advanced/expert utility. +Due to changes in Lucene 9, that isn't possible any more; the component was removed. +If re-indexing is too onerous, use this mechanism in Solr 8. + +=== Solr Binary Release `dist/` Directory + +The `dist/` directory in the Solr binary release has been removed as of Solr 9.0. + +* The `solr-core` and `solr-solrj` jars can be found under `server/solr-webapp/webapp/WEB-INF/lib/`. +* The Solr module jars can be found in `modules/<module-name>/lib`, packaged individually for each module. +* The `solrj-deps` (SolrJ Dependencies) are no longer separated out from the other Server jars. +Please refer to the SolrJ Maven artifact to see the exact dependencies you need to include from `server/solr-webapp/webapp/WEB-INF/lib/` and `server/lib/ext/` if you are loading in SolrJ manually. +If you plan on using SolrJ as a JDBC driver, please refer to the xref:query-guide:sql-query.adoc#generic-clients[JDBC documentation] + +== Solr 9.0 Raw Notes + +_(raw; not yet edited)_ + + +* SOLR-13671: Allow 'var' keyword in Java sources + +* SOLR-12055 introduces async logging by default. There's a small window where log messages may be lost in the event of some hard crash. +Switch back to synchronous logging if this is unacceptable, see comments in the log4j2 configuration files (log4j2.xml by default). + +* SOLR-13323: The unused package org.apache.solr.internal.csv.writer and associated classes/tests that were easily confused with but not used by org.apache.solr.response.CSVWriter (or any other code) have been removed + +* SOLR-13854, SOLR-13858: SolrMetricProducer / SolrInfoBean APIs have changed and third-party components that implement these APIs need to be updated. + +* SOLR-14344: Remove Deprecated HttpSolrClient.RemoteSolrException and HttpSolrClient.RemoteExcecutionException. +All the usages are replaced by BaseHttpSolrClient.RemoteSolrException and BaseHttpSolrClient.RemoteExcecutionException. + +* SOLR-15409: Zookeeper client libraries upgraded to 3.7.0, which may not be compatible with your existing server installations + +* SOLR-15809: Get rid of blacklist/whitelist terminology. JWTAuthPlugin parameter `algWhitelist` is now `algAllowlist`. The old parameter will still + work in 9.x. Environment variables `SOLR_IP_WHITELIST` and `SOLR_IP_BLACKLIST` are no longer supported, but replaced with `SOLR_IP_ALLOWLIST` and `SOLR_IP_DENYLIST`. + +* SOLR-11623: Every request handler in Solr now implements PermissionNameProvider. Any custom or 3rd party request handler must also do this + +* SOLR-14142: Jetty low level request-logging in NCSA format is now enabled by default, with a retention of 3 days worth of logs. + This may require some more disk space for logs than was the case in 8.x. See Reference Guide chapter "Configuring Logging" for how to change this. + +* SOLR-15944: The Tagger's JSON response format now always uses an object/map to represent each tag instead of an array. + +* SOLR-15842: Async responses for backups now correctly aggregate and return information. +In previous versions there was a field returned in async backup status responses, `Response`. This has now been renamed to `msg`, to better fit other collections API responses. +The `response` field is now a map, containing information about the backup (`startTime`, `indexSizeMB`, `indexFileCount`, etc.). + +* SOLR-15884: In Backup request responses, the `response` key now uses a map to return information instead of a list. +This is only applicable for users returning information in JSON format, which is the default behavior. + ++* SOLR-14660: HDFS storage support has been moved to a module. Existing Solr configurations do not need any HDFS-related ++changes, however the module needs to be installed - see the section xref:deployment-guide:solr-on-hdfs.adoc[]. ++ +== New Features & Enhancements + +* Replica placement plugins + +* Rate limiting and task management + +* Certificate Auth Plugin + +* SQL Query interface in UI + +== Configuration and Default Parameter Changes + +* SOLR-7530: TermsComponent's JSON response format was changed so that "terms" property carries per field arrays by default regardless of distrib, terms.list, terms.ttf parameters. +This affects JSON based response format but not others + +* SOLR-14036: Implicit /terms handler now returns terms across all shards in SolrCloud instead of only the local core. +Users/apps may be assuming the old behavior. +A request can be modified via the standard distrib=false param to only use the local core receiving the request. + +* SOLR-13783: In situations where a NamedList must be output as plain text, commas between key-value pairs will now be followed by a space (e.g., {shape=square, color=yellow} rather than {shape=square,color=yellow}) for consistency with other `java.util.Map` implementations based on `AbstractMap`. + +* SOLR-11725: JSON aggregations uses corrected sample formula to compute standard deviation and variance. +The computation of stdDev and variance in JSON aggregation is same as StatsComponent. + +* SOLR-14012: unique and hll aggregations always returns long value irrespective of standalone or solcloud + +* SOLR-11775: Return long value for facet count in Json Facet module irrespective of number of shards + +* SOLR-15276: V2 API call to look up async request status restful style of "/cluster/command-status/1000" instead of "/cluster/command-status?requestid=1000". + +* SOLR-14972: The default port of prometheus exporter has changed from 9983 to 8989, so you may need to adjust your configuration after upgrade. + +* SOLR-15471: The language identification "whitelist" configuration is now an "allowlist" to better convey the meaning of the property + +* SOLR-12891: MacroExpander will no longer will expand URL parameters inside of the 'expr' parameter (used by streaming expressions). +Additionally, users are advised to use the 'InjectionDefense' class when constructing streaming expressions that include user supplied data to avoid risks similar to SQL injection. +The legacy behavior of expanding the 'expr' parameter can be reinstated with -DStreamingExpressionMacros=true passed to the JVM at startup + +* SOLR-13324: URLClassifyProcessor#getCanonicalUrl now throws MalformedURLException rather than hiding it. +Although the present code is unlikely to produce such an exception it may be possible in future changes or in subclasses. +Currently this change should only effect compatibility of custom code overriding this method. + +* SOLR-14510: The `writeStartDocumentList` in `TextResponseWriter` now receives an extra boolean parameter representing the "exactness" of the `numFound` value (exact vs approximation). +Any custom response writer extending `TextResponseWriter` will need to implement this abstract method now (instead previous with the same name but without the new boolean parameter). + +* SOLR-9376: The response format for field values serialized as raw XML (via the `[xml]` raw value DocTransformer +and `wt=xml`) has changed. Previously, values were dropped in directly as top-level child elements of each `<doc>`, +obscuring associated field names and yielding inconsistent `<doc>` structure. As of version 9.0, raw values are +wrapped in a `<raw name="field_name">[...]</raw>` element at the top level of each `<doc>` (or within an enclosing +`<arr name="field_name"><raw>[...]</raw></arr>` element for multi-valued fields). Existing clients that parse field +values serialized in this way will need to be updated accordingly. + +* SOLR-9575: Solr no longer requires a `solr.xml` in `$SOLR_HOME`. If one is not found, Solr will instead use the default one from `$SOLR_TIP/server/solr/solr.xml`. You can revert to the pre-9.0 behaviour by setting environment variable `SOLR_SOLRXML_REQUIRED=true` or system property `-Dsolr.solrxml.required=true`. Solr also does not require a `zoo.cfg` in `$SOLR_HOME` if started with embedded zookeeper. + +=== solr.xml maxBooleanClauses now enforced recursively + +Lucene 9.0 has additional safety checks over previous versions that impact how the `solr.xml` global xref:configuration-guide:configuring-solr-xml#global-maxbooleanclauses[`maxBooleanClauses`] option is enforced. + +In previous versions of Solr, this option was a hard limit on the number of clauses in any `BooleanQuery` object - but it was only enforced for the _direct_ clauses. +Starting with Solr 9, this global limit is now also enforced against the total number of clauses in a _nested_ query structure. + +Users who upgrade from prior versions of Solr may find that some requests involving complex internal query structures (Example: long query strings using `edismax` with many `qf` and `pf` fields that include query time synonym expansion) which worked in the past now hit this limit and fail. + +User's in this situation are advised to consider the complexity f their queries/configuration, and increase the value of xref:configuration-guide:configuring-solr-xml#global-maxbooleanclauses[`maxBooleanClauses`] if warranted. + +=== Log4J configuration & Solr MDC values + +link:http://www.slf4j.org/apidocs/org/slf4j/MDC.html[MDC] values that Solr sets for use by Logging calls (such as the collection name, shard name, replica name, etc...) have been modified to now be "bare" values, with out the special single character prefixes that were included in past version. +For example: In 8.x Log messages for a collection named "gettingstarted" would have an MDC value with a key `collection` mapped to a value of `c:gettingstarted`, in 9.x the value will simply be `gettingstarted`. + +Solr's default `log4j2.xml` configuration file has been modified to prepend these same prefixes to MDC values when included in Log messages as part of the `<PatternLayout/>`. +Users who have custom logging configurations that wish to ensure Solr 9.x logs are consistently formatted after upgrading will need to make similar changes to their logging configuration files. See link:https://issues.apache.org/jira/browse/SOLR-15630[SOLR-15630] for more details. + + +=== base_url removed from stored state + +If you're able to upgrade SolrJ to 8.8.x for all of your client applications, then you can set `-Dsolr.storeBaseUrl=false` (introduced in Solr 8.8.1) to better align the stored state in Zookeeper with future versions of Solr; as of Solr 9.x, the `base_url` will no longer be persisted in stored state. +However, if you are not able to upgrade SolrJ to 8.8.x for all client applications, then you should set `-Dsolr.storeBaseUrl=true` so that Solr will continue to store the `base_url` in Zookeeper. +For background, see: SOLR-12182 and SOLR-15145. + +Support for the `solr.storeBaseUrl` system property will be removed in Solr 10.x and `base_url` will no longer be stored. + +* Solr's distributed tracing no longer incorporates a special `samplePercentage` SolrCloud cluster property. +Instead, consult the documentation for the tracing system you use on how to sample the traces. +Consequently, if you use a Tracer at all, you will always have traces and thus trace IDs in logs. +What percentage of them get reported to a tracing server is up to you. + +* JaegerTracerConfigurator no longer recognizes any configuration in solr.xml. + It is now completely configured via System properties and/or Environment variables as documented by Jaeger. + +=== Schema Changes + +* `LegacyBM25SimilarityFactory` has been removed. + +* SOLR-13593 SOLR-13690 SOLR-13691: Allow to look up analyzer components by their SPI names in field type configuration. + +=== Authentication & Security Changes + +* The property `blockUnknown` in the BasicAuthPlugin and the JWTAuthPlugin now defaults to `true`. +This change is backward incompatible. +If you need the pre-9.0 default behavior, you need to explicitly set `blockUnknown:false` in `security.json`. + +* The allow-list defining allowed URLs for the `shards` parameter is not in the `shardHandler` configuration anymore. It is defined by the `allowUrls` top-level property of the `solr.xml` file. For more information, see xref:configuration-guide:configuring-solr-xml.adoc#allow-urls[Format of solr.allowUrls] documentation. + +* SOLR-13985: Solr's Jetty now binds to localhost network interface by default for better out of the box security. +Administrators that need Solr exposed more broadly can change the SOLR_JETTY_HOST property in their Solr include (solr.in.sh/solr.in.cmd) file. + +* SOLR-14147: Solr now runs with the java security manager enabled by default. Administrators that need to run Solr with Hadoop will need to disable this feature by setting SOLR_SECURITY_MANAGER_ENABLED=false in the environment or in one of the Solr init scripts. Other features in Solr could also break. (Robert Muir, marcussorealheis) + +* SOLR-14118: Solr embedded zookeeper only binds to localhost by default. +This embedded zookeeper should not be used in production. +If you rely upon the previous behavior, then you can change the clientPortAddress in solr/server/solr/zoo.cfg + +=== Module Changes + +* **SOLR-15917: "Contrib modules" have been renamed to "Modules", and have been moved from the `contrib/` to `modules/`.** +Use of these modules remains the same, except for the changes listed below. + +* SOLR-15916: `dist/` is no longer provided in the binary release. +All module jars are now provided under `modules/<name>/lib`, including the module jar and all dependency jars. +Please update your `<lib>` entries in your `solrconfig.xml` to use this new location. +More information can be found in the xref:configuration-guide:libs.adoc#lib-directives-in-solrconfig[Libs documentation]. + +* SOLR-14067: `StatelessScriptUpdateProcessorFactory` moved to `modules/scripting` package instead of shipping as part of Solr, due to security concerns. +Renamed to ScriptUpdateProcessorFactory for simpler name. + +* SOLR-15121: `XSLTResponseWriter` moved to `modules/scripting` package instead +of shipping as part of Solr, due to security concerns. + +* SOLR-14926: `modules/clustering` back and rewritten + +* SOLR-14912: Cleaned up solr-extraction module to produce solr-extraction-* jar (instead of solr-cell-*). (Dawid Weiss) + +* SOLR-15924: Extra lucene libraries used in modules are no longer packaged in `lucene-libs/` under module directories in the binary release. +Instead, these libraries will be included with all other module dependencies in `lib/`. + +* SOLR-15954: The prometheus-exporter is no longer packaged as a Solr module. It can be found under `solr/prometheus-exporter/`. + +* SOLR-15914: Solr modules (formerly known as contribs) can now easily be enabled by an environment variable (e.g. in `solr.in.sh` or `solr.in.cmd`) or as a system property (e.g. in `SOLR_OPTS`). Example: `SOLR_MODULES=extraction,ltr`. + +== Deprecations & Removed Features + +The following list of features have been permanently removed from Solr: + +* SOLR-14656: Autoscaling framework removed. +This includes: +** Autoscaling, policy, triggers etc. +** withCollection handling (SOLR-14964) +** UTILIZENODE command +** Sim framework +** Suggestions tab in UI +** Reference guide pages for autoscaling +** autoAddReplicas feature + +* SOLR-14783: Data Import Handler (DIH) has been removed from Solr. +The community package is available at: https://github.com/rohitbemax/dataimporthandler + +* SOLR-14792: VelocityResponseWriter has been removed from Solr. +This encompasses all previous included `/browse` and `wt=velocity` examples. +This feature has been migrated to an installable package at https://github.com/erikhatcher/solr-velocity + +* SOLR-13817: Legacy SolrCache implementations (LRUCache, LFUCache, FastLRUCache) have been removed. +Users have to modify their existing configurations to use CaffeineCache instead. (ab) + +* CDCR + - * Storing indexes and backups in HDFS - +* Solr's blob store +** SOLR-14654: plugins cannot be loaded using "runtimeLib=true" option. Use the package manager to use and load plugins + +* Metrics History + +* SOLR-15470: The binary distribution no longer contains test-framework jars. + +* SOLR-15203: Remove the deprecated `jwkUrl` in favour of `jwksUrl` when configuring JWT authentication. + +* SOLR-12847: maxShardsPerNode parameter has been removed because it was broken and inconsistent with other replica placement strategies. +Other relevant placement strategies should be used instead, such as autoscaling policy or rules-based placement. + +* SOLR-14092: Deprecated BlockJoinFacetComponent and BlockJoinDocSetFacetComponent are removed. +Users are encouraged to migrate to uniqueBlock() in JSON Facet API. (Mikhail Khludnev) + +* SOLR-13596: Deprecated GroupingSpecification methods are removed. + +* SOLR-11266: default Content-Type override for JSONResponseWriter from `_default` configSet is removed. +Example has been provided in `sample_techproducts_configs` to override content-type. + +* `min_rf` deprecated in 7.x + +* hl.method=postings highlighter, deprecated in 7.0 + +* SOLR-15124: Removed three core level admin API endpoints because they are already registered at the node level +where they really belong: /admin/threads, /admin/properties, /admin/logging
