This is an automated email from the ASF dual-hosted git repository. gerlowskija pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit b342464f4a4c509d285fea26adf83bb4a155210e Author: John Durham <[email protected]> AuthorDate: Mon Apr 17 18:45:22 2023 -0500 SOLR-15737: Add v2 equivalent of collection "snapshot" APIs (#1471) Snapshots can now be created or deleted at `/api/collections/collName/snapshots/snapshotName` (`POST` and `DELETE` respectively), and listed at `GET /api/collections/collName/snapshots`. --------- Co-authored-by: Jason Gerlowski <[email protected]> --- solr/CHANGES.txt | 4 + .../solr/handler/admin/CollectionsHandler.java | 120 +++----- .../solr/handler/admin/api/AdminAPIBase.java | 20 ++ .../admin/api/CreateCollectionSnapshotAPI.java | 162 ++++++++++ .../admin/api/DeleteCollectionSnapshotAPI.java | 139 +++++++++ .../admin/api/ListCollectionSnapshotsAPI.java | 90 ++++++ .../admin/api/CreateCollectionSnapshotAPITest.java | 65 ++++ .../admin/api/DeleteCollectionSnapshotAPITest.java | 65 ++++ .../pages/collection-management.adoc | 330 ++++++++++++++++++++- 9 files changed, 914 insertions(+), 81 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 4f35b5c1fcd..d0f39ee0d60 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -58,6 +58,10 @@ Improvements * SOLR-15493: Throw an error message if the feature store doesn't exist. (Ilaria Petreti via Alessandro Benedetti) +* SOLR-15737: Solr's collection-level "snapshot" APIs now have v2 equivalents. Snapshots can be created at `POST + /api/collections/collName/snapshots/snapshotName`, listed at `GET /api/collections/collName/snapshots`, and deleted at + `DELETE /api/collections/collName/snapshots/snapshotName`. (John Durham via Jason Gerlowski) + Optimizations --------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index f4e121db01b..965ed4b67de 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -214,9 +214,11 @@ import org.apache.solr.handler.admin.api.CollectionStatusAPI; import org.apache.solr.handler.admin.api.CreateAliasAPI; import org.apache.solr.handler.admin.api.CreateCollectionAPI; import org.apache.solr.handler.admin.api.CreateCollectionBackupAPI; +import org.apache.solr.handler.admin.api.CreateCollectionSnapshotAPI; import org.apache.solr.handler.admin.api.CreateShardAPI; import org.apache.solr.handler.admin.api.DeleteAliasAPI; import org.apache.solr.handler.admin.api.DeleteCollectionAPI; +import org.apache.solr.handler.admin.api.DeleteCollectionSnapshotAPI; import org.apache.solr.handler.admin.api.DeleteNodeAPI; import org.apache.solr.handler.admin.api.DeleteReplicaAPI; import org.apache.solr.handler.admin.api.DeleteReplicaPropertyAPI; @@ -224,6 +226,7 @@ import org.apache.solr.handler.admin.api.DeleteShardAPI; import org.apache.solr.handler.admin.api.ForceLeaderAPI; import org.apache.solr.handler.admin.api.InstallShardDataAPI; import org.apache.solr.handler.admin.api.ListAliasesAPI; +import org.apache.solr.handler.admin.api.ListCollectionSnapshotsAPI; import org.apache.solr.handler.admin.api.ListCollectionsAPI; import org.apache.solr.handler.admin.api.MigrateDocsAPI; import org.apache.solr.handler.admin.api.ModifyCollectionAPI; @@ -1649,96 +1652,60 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission (req, rsp, h) -> { req.getParams().required().check(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME); - String extCollectionName = req.getParams().get(COLLECTION_PROP); - boolean followAliases = req.getParams().getBool(FOLLOW_ALIASES, false); - String collectionName = - followAliases - ? h.coreContainer - .getZkController() - .getZkStateReader() - .getAliases() - .resolveSimpleAlias(extCollectionName) - : extCollectionName; - String commitName = req.getParams().get(CoreAdminParams.COMMIT_NAME); - ClusterState clusterState = h.coreContainer.getZkController().getClusterState(); - if (!clusterState.hasCollection(collectionName)) { - throw new SolrException( - ErrorCode.BAD_REQUEST, - "Collection '" + collectionName + "' does not exist, no action taken."); - } + final String extCollectionName = req.getParams().get(COLLECTION_PROP); + final boolean followAliases = req.getParams().getBool(FOLLOW_ALIASES, false); + final String commitName = req.getParams().get(CoreAdminParams.COMMIT_NAME); + final String asyncId = req.getParams().get(ASYNC); - SolrZkClient client = h.coreContainer.getZkController().getZkClient(); - if (SolrSnapshotManager.snapshotExists(client, collectionName, commitName)) { - throw new SolrException( - ErrorCode.BAD_REQUEST, - "Snapshot with name '" - + commitName - + "' already exists for collection '" - + collectionName - + "', no action taken."); - } + final CreateCollectionSnapshotAPI createCollectionSnapshotAPI = + new CreateCollectionSnapshotAPI(h.coreContainer, req, rsp); - Map<String, Object> params = - copy( - req.getParams(), - null, - COLLECTION_PROP, - FOLLOW_ALIASES, - CoreAdminParams.COMMIT_NAME); - return params; + final CreateCollectionSnapshotAPI.CreateSnapshotRequestBody requestBody = + new CreateCollectionSnapshotAPI.CreateSnapshotRequestBody(); + requestBody.followAliases = followAliases; + requestBody.asyncId = asyncId; + + final CreateCollectionSnapshotAPI.CreateSnapshotResponse createSnapshotResponse = + createCollectionSnapshotAPI.createSnapshot( + extCollectionName, commitName, requestBody); + + V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, createSnapshotResponse); + + return null; }), DELETESNAPSHOT_OP( DELETESNAPSHOT, (req, rsp, h) -> { req.getParams().required().check(COLLECTION_PROP, CoreAdminParams.COMMIT_NAME); - String extCollectionName = req.getParams().get(COLLECTION_PROP); - String collectionName = - h.coreContainer - .getZkController() - .getZkStateReader() - .getAliases() - .resolveSimpleAlias(extCollectionName); - ClusterState clusterState = h.coreContainer.getZkController().getClusterState(); - if (!clusterState.hasCollection(collectionName)) { - throw new SolrException( - ErrorCode.BAD_REQUEST, - "Collection '" + collectionName + "' does not exist, no action taken."); - } + final String extCollectionName = req.getParams().get(COLLECTION_PROP); + final String commitName = req.getParams().get(CoreAdminParams.COMMIT_NAME); + final boolean followAliases = req.getParams().getBool(FOLLOW_ALIASES, false); + final String asyncId = req.getParams().get(ASYNC); - Map<String, Object> params = - copy( - req.getParams(), - null, - COLLECTION_PROP, - FOLLOW_ALIASES, - CoreAdminParams.COMMIT_NAME); - return params; + final DeleteCollectionSnapshotAPI deleteCollectionSnapshotAPI = + new DeleteCollectionSnapshotAPI(h.coreContainer, req, rsp); + + final DeleteCollectionSnapshotAPI.DeleteSnapshotResponse deleteSnapshotResponse = + deleteCollectionSnapshotAPI.deleteSnapshot( + extCollectionName, commitName, followAliases, asyncId); + + V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, deleteSnapshotResponse); + return null; }), LISTSNAPSHOTS_OP( LISTSNAPSHOTS, (req, rsp, h) -> { req.getParams().required().check(COLLECTION_PROP); - String extCollectionName = req.getParams().get(COLLECTION_PROP); - String collectionName = - h.coreContainer - .getZkController() - .getZkStateReader() - .getAliases() - .resolveSimpleAlias(extCollectionName); - ClusterState clusterState = h.coreContainer.getZkController().getClusterState(); - if (!clusterState.hasCollection(collectionName)) { - throw new SolrException( - ErrorCode.BAD_REQUEST, - "Collection '" + collectionName + "' does not exist, no action taken."); - } + final ListCollectionSnapshotsAPI listCollectionSnapshotsAPI = + new ListCollectionSnapshotsAPI(h.coreContainer, req, rsp); + + final ListCollectionSnapshotsAPI.ListSnapshotsResponse response = + listCollectionSnapshotsAPI.listSnapshots(req.getParams().get(COLLECTION_PROP)); - NamedList<Object> snapshots = new NamedList<Object>(); - SolrZkClient client = h.coreContainer.getZkController().getZkClient(); - Collection<CollectionSnapshotMetaData> m = - SolrSnapshotManager.listSnapshots(client, collectionName); - for (CollectionSnapshotMetaData meta : m) { + NamedList<Object> snapshots = new NamedList<>(); + for (CollectionSnapshotMetaData meta : response.snapshots.values()) { snapshots.add(meta.getName(), meta.toNamedList()); } @@ -2044,7 +2011,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission CollectionPropertyAPI.class, DeleteNodeAPI.class, ListAliasesAPI.class, - AliasPropertyAPI.class); + AliasPropertyAPI.class, + ListCollectionSnapshotsAPI.class, + CreateCollectionSnapshotAPI.class, + DeleteCollectionSnapshotAPI.class); } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java index 4ffe8b83bbc..0931ee7dfa7 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/AdminAPIBase.java @@ -82,6 +82,26 @@ public abstract class AdminAPIBase extends JerseyResource { } } + protected String resolveCollectionName(String collName, boolean followAliases) { + final String collectionName = + followAliases + ? coreContainer + .getZkController() + .getZkStateReader() + .getAliases() + .resolveSimpleAlias(collName) + : collName; + + final ClusterState clusterState = coreContainer.getZkController().getClusterState(); + if (!clusterState.hasCollection(collectionName)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Collection '" + collectionName + "' does not exist, no action taken."); + } + + return collectionName; + } + /** * TODO Taken from CollectionsHandler.handleRequestBody, but its unclear where (if ever) this gets * cleared. diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPI.java new file mode 100644 index 00000000000..b331af1aa7c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPI.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; +import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; +import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import java.util.HashMap; +import java.util.Map; +import javax.inject.Inject; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.handler.admin.CollectionsHandler; +import org.apache.solr.jersey.AsyncJerseyResponse; +import org.apache.solr.jersey.JacksonReflectMapWriter; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** V2 API for Creating Collection Snapshots. */ +@Path("/collections/{collName}/snapshots") +public class CreateCollectionSnapshotAPI extends AdminAPIBase { + + @Inject + public CreateCollectionSnapshotAPI( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + super(coreContainer, solrQueryRequest, solrQueryResponse); + } + + /** This API is analogous to V1's (POST /solr/admin/collections?action=CREATESNAPSHOT) */ + @POST + @Path("/{snapshotName}") + @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @PermissionName(COLL_EDIT_PERM) + public CreateSnapshotResponse createSnapshot( + @Parameter(description = "The name of the collection.", required = true) + @PathParam("collName") + String collName, + @Parameter(description = "The name of the snapshot to be created.", required = true) + @PathParam("snapshotName") + String snapshotName, + @RequestBody(description = "Contains user provided parameters", required = true) + CreateSnapshotRequestBody requestBody) + throws Exception { + + final CreateSnapshotResponse response = instantiateJerseyResponse(CreateSnapshotResponse.class); + final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(collName, solrQueryRequest); + + final String collectionName = resolveCollectionName(collName, requestBody.followAliases); + + final SolrZkClient client = coreContainer.getZkController().getZkClient(); + if (SolrSnapshotManager.snapshotExists(client, collectionName, snapshotName)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Snapshot with name '" + + snapshotName + + "' already exists for collection '" + + collectionName + + "', no action taken."); + } + + final ZkNodeProps remoteMessage = + createRemoteMessage(collName, requestBody.followAliases, snapshotName, requestBody.asyncId); + final SolrResponse remoteResponse = + CollectionsHandler.submitCollectionApiCommand( + coreContainer, + coreContainer.getDistributedCollectionCommandRunner(), + remoteMessage, + CollectionParams.CollectionAction.CREATESNAPSHOT, + DEFAULT_COLLECTION_OP_TIMEOUT); + + if (remoteResponse.getException() != null) { + throw remoteResponse.getException(); + } + + response.collection = collName; + response.followAliases = requestBody.followAliases; + response.snapshotName = snapshotName; + response.requestId = requestBody.asyncId; + + return response; + } + + /** + * The RequestBody for {@link CreateCollectionSnapshotAPI}'s {@link #createSnapshot(String, + * String, CreateSnapshotRequestBody)} + */ + public static class CreateSnapshotRequestBody implements JacksonReflectMapWriter { + @JsonProperty(value = "followAliases", defaultValue = "false") + public boolean followAliases; + + @JsonProperty("async") + public String asyncId; + } + + /** + * The Response for {@link CreateCollectionSnapshotAPI}'s {@link #createSnapshot(String, String, + * CreateSnapshotRequestBody)} + */ + public static class CreateSnapshotResponse extends AsyncJerseyResponse { + @Schema(description = "The name of the collection.") + @JsonProperty(COLLECTION_PROP) + String collection; + + @Schema(description = "The name of the snapshot to be created.") + @JsonProperty("snapshot") + String snapshotName; + + @Schema(description = "A flag that treats the collName parameter as a collection alias.") + @JsonProperty("followAliases") + boolean followAliases; + } + + public static ZkNodeProps createRemoteMessage( + String collectionName, boolean followAliases, String snapshotName, String asyncId) { + final Map<String, Object> remoteMessage = new HashMap<>(); + + remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.CREATESNAPSHOT.toLower()); + remoteMessage.put(COLLECTION_PROP, collectionName); + remoteMessage.put(CoreAdminParams.COMMIT_NAME, snapshotName); + remoteMessage.put(FOLLOW_ALIASES, followAliases); + if (asyncId != null) remoteMessage.put(ASYNC, asyncId); + + return new ZkNodeProps(remoteMessage); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPI.java new file mode 100644 index 00000000000..e591e7aed7e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPI.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; +import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; +import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.HashMap; +import java.util.Map; +import javax.inject.Inject; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CollectionParams; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.CollectionsHandler; +import org.apache.solr.jersey.AsyncJerseyResponse; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** V2 API for Deleting Collection Snapshots. */ +@Path("/collections/{collName}/snapshots") +public class DeleteCollectionSnapshotAPI extends AdminAPIBase { + + @Inject + public DeleteCollectionSnapshotAPI( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + super(coreContainer, solrQueryRequest, solrQueryResponse); + } + + /** This API is analogous to V1's (POST /solr/admin/collections?action=DELETESNAPSHOT) */ + @DELETE + @Path("/{snapshotName}") + @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @PermissionName(COLL_EDIT_PERM) + public DeleteSnapshotResponse deleteSnapshot( + @Parameter(description = "The name of the collection.", required = true) + @PathParam("collName") + String collName, + @Parameter(description = "The name of the snapshot to be deleted.", required = true) + @PathParam("snapshotName") + String snapshotName, + @Parameter(description = "A flag that treats the collName parameter as a collection alias.") + @DefaultValue("false") + @QueryParam("followAliases") + boolean followAliases, + @QueryParam("async") String asyncId) + throws Exception { + final DeleteSnapshotResponse response = instantiateJerseyResponse(DeleteSnapshotResponse.class); + final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(collName, solrQueryRequest); + + final String collectionName = resolveCollectionName(collName, followAliases); + + final ZkNodeProps remoteMessage = + createRemoteMessage(collectionName, followAliases, snapshotName, asyncId); + final SolrResponse remoteResponse = + CollectionsHandler.submitCollectionApiCommand( + coreContainer, + coreContainer.getDistributedCollectionCommandRunner(), + remoteMessage, + CollectionParams.CollectionAction.DELETESNAPSHOT, + DEFAULT_COLLECTION_OP_TIMEOUT); + + if (remoteResponse.getException() != null) { + throw remoteResponse.getException(); + } + + response.collection = collName; + response.snapshotName = snapshotName; + response.followAliases = followAliases; + response.requestId = asyncId; + + return response; + } + + /** + * The Response for {@link DeleteCollectionSnapshotAPI}'s {@link #deleteSnapshot(String, String, + * boolean, String)} + */ + public static class DeleteSnapshotResponse extends AsyncJerseyResponse { + @Schema(description = "The name of the collection.") + @JsonProperty(COLLECTION_PROP) + String collection; + + @Schema(description = "The name of the snapshot to be deleted.") + @JsonProperty("snapshot") + String snapshotName; + + @Schema(description = "A flag that treats the collName parameter as a collection alias.") + @JsonProperty("followAliases") + boolean followAliases; + } + + public static ZkNodeProps createRemoteMessage( + String collectionName, boolean followAliases, String snapshotName, String asyncId) { + final Map<String, Object> remoteMessage = new HashMap<>(); + + remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.DELETESNAPSHOT.toLower()); + remoteMessage.put(COLLECTION_PROP, collectionName); + remoteMessage.put(CoreAdminParams.COMMIT_NAME, snapshotName); + remoteMessage.put(FOLLOW_ALIASES, followAliases); + + if (asyncId != null) remoteMessage.put(ASYNC, asyncId); + + return new ZkNodeProps(remoteMessage); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollectionSnapshotsAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollectionSnapshotsAPI.java new file mode 100644 index 00000000000..b3fa81fb2c0 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollectionSnapshotsAPI.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; +import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.Collection; +import java.util.Map; +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.CollectionUtil; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.snapshots.CollectionSnapshotMetaData; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.jersey.AsyncJerseyResponse; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** V2 API for Listing Collection Snapshots. */ +@Path("/collections/{collName}/snapshots") +public class ListCollectionSnapshotsAPI extends AdminAPIBase { + + @Inject + public ListCollectionSnapshotsAPI( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + super(coreContainer, solrQueryRequest, solrQueryResponse); + } + + /** This API is analogous to V1's (POST /solr/admin/collections?action=LISTSNAPSHOTS) */ + @GET + @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @PermissionName(COLL_READ_PERM) + public ListSnapshotsResponse listSnapshots( + @Parameter(description = "The name of the collection.", required = true) + @PathParam("collName") + String collName) + throws Exception { + + final ListSnapshotsResponse response = instantiateJerseyResponse(ListSnapshotsResponse.class); + final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); + recordCollectionForLogAndTracing(collName, solrQueryRequest); + + final String collectionName = resolveCollectionName(collName, true); + + SolrZkClient client = coreContainer.getZkController().getZkClient(); + Collection<CollectionSnapshotMetaData> m = + SolrSnapshotManager.listSnapshots(client, collectionName); + + Map<String, CollectionSnapshotMetaData> snapshots = CollectionUtil.newHashMap(m.size()); + for (CollectionSnapshotMetaData metaData : m) { + snapshots.put(metaData.getName(), metaData); + } + + response.snapshots = snapshots; + + return response; + } + + /** The Response for {@link ListCollectionSnapshotsAPI}'s {@link #listSnapshots(String)} */ + public static class ListSnapshotsResponse extends AsyncJerseyResponse { + @Schema(description = "The snapshots for the collection.") + @JsonProperty(SolrSnapshotManager.SNAPSHOTS_INFO) + public Map<String, CollectionSnapshotMetaData> snapshots; + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java new file mode 100644 index 00000000000..c5f31fbdf45 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateCollectionSnapshotAPITest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.hamcrest.Matchers.containsInAnyOrder; + +import java.util.Map; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CoreAdminParams; +import org.hamcrest.MatcherAssert; +import org.junit.Test; + +public class CreateCollectionSnapshotAPITest extends SolrTestCaseJ4 { + + @Test + public void testConstructsValidOverseerMessage() { + final ZkNodeProps messageOne = + CreateCollectionSnapshotAPI.createRemoteMessage( + "myCollName", false, "mySnapshotName", null); + final Map<String, Object> rawMessageOne = messageOne.getProperties(); + assertEquals(4, rawMessageOne.size()); + MatcherAssert.assertThat( + rawMessageOne.keySet(), + containsInAnyOrder( + QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); + assertEquals("createsnapshot", rawMessageOne.get(QUEUE_OPERATION)); + assertEquals("myCollName", rawMessageOne.get(COLLECTION_PROP)); + assertEquals("mySnapshotName", rawMessageOne.get(CoreAdminParams.COMMIT_NAME)); + assertEquals(false, rawMessageOne.get(FOLLOW_ALIASES)); + + final ZkNodeProps messageTwo = + CreateCollectionSnapshotAPI.createRemoteMessage( + "myCollName", true, "mySnapshotName", "myAsyncId"); + final Map<String, Object> rawMessageTwo = messageTwo.getProperties(); + assertEquals(5, rawMessageTwo.size()); + MatcherAssert.assertThat( + rawMessageTwo.keySet(), + containsInAnyOrder( + QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES, ASYNC)); + assertEquals("createsnapshot", rawMessageTwo.get(QUEUE_OPERATION)); + assertEquals("myCollName", rawMessageTwo.get(COLLECTION_PROP)); + assertEquals("mySnapshotName", rawMessageTwo.get(CoreAdminParams.COMMIT_NAME)); + assertEquals(true, rawMessageTwo.get(FOLLOW_ALIASES)); + assertEquals("myAsyncId", rawMessageTwo.get(ASYNC)); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java new file mode 100644 index 00000000000..ac363a02bae --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/DeleteCollectionSnapshotAPITest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; +import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; +import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; +import static org.apache.solr.common.params.CommonAdminParams.ASYNC; +import static org.hamcrest.Matchers.containsInAnyOrder; + +import java.util.Map; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.params.CoreAdminParams; +import org.hamcrest.MatcherAssert; +import org.junit.Test; + +public class DeleteCollectionSnapshotAPITest extends SolrTestCaseJ4 { + + @Test + public void testConstructsValidOverseerMessage() { + final ZkNodeProps messageOne = + DeleteCollectionSnapshotAPI.createRemoteMessage( + "myCollName", false, "mySnapshotName", null); + final Map<String, Object> rawMessageOne = messageOne.getProperties(); + assertEquals(4, rawMessageOne.size()); + MatcherAssert.assertThat( + rawMessageOne.keySet(), + containsInAnyOrder( + QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES)); + assertEquals("deletesnapshot", rawMessageOne.get(QUEUE_OPERATION)); + assertEquals("myCollName", rawMessageOne.get(COLLECTION_PROP)); + assertEquals("mySnapshotName", rawMessageOne.get(CoreAdminParams.COMMIT_NAME)); + assertEquals(false, rawMessageOne.get(FOLLOW_ALIASES)); + + final ZkNodeProps messageTwo = + DeleteCollectionSnapshotAPI.createRemoteMessage( + "myCollName", true, "mySnapshotName", "myAsyncId"); + final Map<String, Object> rawMessageTwo = messageTwo.getProperties(); + assertEquals(5, rawMessageTwo.size()); + MatcherAssert.assertThat( + rawMessageTwo.keySet(), + containsInAnyOrder( + QUEUE_OPERATION, COLLECTION_PROP, CoreAdminParams.COMMIT_NAME, FOLLOW_ALIASES, ASYNC)); + assertEquals("deletesnapshot", rawMessageTwo.get(QUEUE_OPERATION)); + assertEquals("myCollName", rawMessageTwo.get(COLLECTION_PROP)); + assertEquals("mySnapshotName", rawMessageTwo.get(CoreAdminParams.COMMIT_NAME)); + assertEquals(true, rawMessageTwo.get(FOLLOW_ALIASES)); + assertEquals("myAsyncId", rawMessageTwo.get(ASYNC)); + } +} diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc index 3cfb11536aa..60ba5fc2e59 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc @@ -1586,8 +1586,8 @@ Multiple collections cannot be backed up to the same location. [NOTE] ==== -Previous versions of Solr supported a different snapshot-based backup method without the incremental support described above. -Solr can still restore from backups that use this old format, but creating new backups of this format is not recommended and snapshot-based backups are officially deprecated. +Previous versions of Solr supported a different backup file format that lacked the incremental support described above. +Solr can still restore from backups that use this old format, but creating new backups of this format is not recommended and is officially deprecated. See the `incremental` parameter below for more information. ==== @@ -1669,9 +1669,9 @@ This parameter has no effect if `incremental=false` is specified. |Optional |Default: `true` |=== + -A boolean parameter allowing users to choose whether to create an incremental (`incremental=true`) or a "snapshot" (`incremental=false`) backup. +A boolean parameter allowing users to choose whether to create an incremental (`incremental=true`) or a "full" (`incremental=false`) backup. If unspecified, backups are done incrementally by default. -Incremental backups are preferred in all known circumstances and snapshot backups are deprecated, so this parameter should only be used after much consideration. +Incremental backups are preferred in all known circumstances and "full" (i.e. non-incremental) backups are deprecated, so this parameter should only be used after much consideration. `indexBackup` (v1), `backupStrategy` (v2):: + @@ -1728,7 +1728,7 @@ Basic metadata is returned about each backup including: the timestamp the backup [NOTE] ==== -Previous versions of Solr supported a different snapshot-based backup file structure that did not support the storage of multiple backups at the same location. +Previous versions of Solr supported a different backup file structure that did not support the storage of multiple backups at the same location. Solr can still restore backups stored in this old format, but it is deprecated and will be removed in subsequent versions of Solr. The LISTBACKUP API does not support the deprecated format and attempts to use this API on a location holding an older backup will result in an error message. ==== @@ -1983,7 +1983,7 @@ Deletes backup files stored at the specified repository location. [NOTE] ==== -Previous versions of Solr supported a different snapshot-based backup file structure that did not support the storage of multiple backups at the same location. +Previous versions of Solr supported a different backup file structure that did not support the storage of multiple backups at the same location. Solr can still restore backups stored in this old format, but it is deprecated and will be removed in subsequent versions of Solr. The DELETEBACKUP API does not support the deprecated format and attempts to use this API on a location holding an older backup will result in an error message. ==== @@ -2257,3 +2257,321 @@ The primary use-case is to redistribute the leader role if there are a large num Rebalancing will likely not improve performance unless the imbalance of leadership roles is measured in multiples of 10. NOTE: The BALANCESHARDUNIQUE command that distributes the preferredLeader property does not guarantee perfect distribution and in some collection topologies it is impossible to make that guarantee. + +[[createsnapshot]] +== CREATESNAPSHOT: Create a snapshot of a collection + +Solr has support for creating collection "snapshots", which "checkpoint" the collection state in a way that allows users to revert to that point if needed later on. +This is particularly useful prior to reindexing or making config changes to a collection. + +Unlike backups, which copy collection data off-disk, snapshots themselves don't provide disaster recovery in case of disk or hardware failure. +They provide less protection than backups, at a much cheaper cost. + +=== CREATESNAPSHOT Example + +*Input* + +The following API command creates a snapshot of a specified collection. + +[.dynamic-tabs] +-- +[example.tab-pane#v1createsnapshot] +==== +[.tab-label]*V1 API* + +[source,bash] +---- +http://localhost:8983/solr/admin/collections?action=CREATESNAPSHOT&collection=techproducts&commitName=snapshot0&followAliases=true&async=someAsyncId +---- +==== + +[example.tab-pane#v2createsnapshot] +==== +[.tab-label]*V2 API* + +[source,bash] +---- +curl -X POST http://localhost:8983/api/collections/techproducts/snapshots/snapshot0 -H 'Content-Type: application/json' -d ' + { + "followAliases": true, + "async": "someAsyncId" + } +' +---- +==== +-- + +*Output* + +[source,json] +---- +{ + "responseHeader": { + "status": 0, + "QTime": 214 + }, + "requestid": "someAsyncId" + "collection": "techproducts", + "snapshot": "snapshot0", + "followAliases": true +} +---- + +=== CREATESNAPSHOT Parameters + +`collection`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The name of the collection to create a snapshot for. + +`snapshot`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The name of the snapshot to create for the collection. + +`followAliases`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: false +|=== ++ +A flag that treats the collection parameter as an alias for the actual collection name to be resolved. + +`async`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Request ID to track this action which will be xref:configuration-guide:collections-api.adoc#asynchronous-calls[processed asynchronously]. + +[[listsnapshots]] +== LISTSNAPSHOTS: List all snapshots for a collection + +Lists all the snapshots taken of a collection. + +=== LISTSNAPSHOTS Example + +*Input* + +The following API command lists all the snapshots taken of a collection. + +[.dynamic-tabs] +-- +[example.tab-pane#v1listsnapshots] +==== +[.tab-label]*V1 API* + +[source,bash] +---- +http://localhost:8983/solr/admin/collections?action=LISTSNAPSHOTS&collection=techproducts +---- +==== + +[example.tab-pane#v2listsnapshots] +==== +[.tab-label]*V2 API* + +[source,bash] +---- +curl -X GET http://localhost:8983/api/collections/techproducts/snapshots +---- +==== +-- + +*Output* + +[source,json] +---- +{ + "responseHeader": { + "status": 0, + "QTime": 2 + }, + "snapshots": { + "snapshot0": { + "name": "snapshot0", + "status": "Successful", + "creationDate": 1677985318116, + "replicaSnapshots": [ + { + "coreName": "techproducts_shard1_replica_n6", + "indexDirPath": "/path/to/solr/dir/node1/solr/techproducts_shard1_replica_n6/data/index/", + "generationNumber": 2, + "leader": true, + "shardId": "shard1", + "files": [ + "_0.si", + "_0.fdm", + "_0_Lucene90_0.dvd", + "segments_2", + "_0_Lucene90_0.doc", + "_0_Lucene90_0.tim", + "_0.fdx", + "_0.fdt", + "_0_Lucene90_0.dvm", + "_0_Lucene90_0.tip", + "_0_Lucene90_0.tmd", + "_0.fnm" + ] + }, + { + "coreName": "techproducts_shard1_replica_n2", + "indexDirPath": "/path/to/solr/dir/node2/solr/techproducts_shard1_replica_n2/data/index/", + "generationNumber": 2, + "leader": false, + "shardId": "shard1", + "files": [ + "_0.si", + "_0.fdm", + "_0_Lucene90_0.dvd", + "segments_2", + "_0_Lucene90_0.doc", + "_0_Lucene90_0.tim", + "_0.fdx", + "_0.fdt", + "_0_Lucene90_0.dvm", + "_0_Lucene90_0.tip", + "_0_Lucene90_0.tmd", + "_0.fnm" + ] + }, + { + "coreName": "techproducts_shard2_replica_n4", + "indexDirPath": "/path/to/solr/dir/node1/solr/techproducts_shard2_replica_n4/data/index/", + "generationNumber": 6, + "leader": true, + "shardId": "shard2", + "files": [ + "segments_6" + ] + }, + { + "coreName": "techproducts_shard2_replica_n1", + "indexDirPath": "/path/to/solr/dir/node2/solr/techproducts_shard2_replica_n1/data/index/", + "generationNumber": 6, + "leader": false, + "shardId": "shard2", + "files": [ + "segments_6" + ] + } + ], + "shards": [ + "shard2", + "shard1" + ] + } + } +} + +---- + +=== LISTSNAPSHOTS Parameters + +`collection`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The name of the collection to create a snapshot for. + +[[deletesnapshot]] +== DELETESNAPSHOT: Delete a snapshot taken of a collection + +Deletes a snapshot taken of a specified collection. + +=== DELETESNAPSHOT Example + +*Input* + +The following API command deletes a snapshot taken of a collection. + +[.dynamic-tabs] +-- +[example.tab-pane#v1deletesnapshot] +==== +[.tab-label]*V1 API* + +[source,bash] +---- +http://localhost:8983/solr/admin/collections?action=DELETESNAPSHOT&collection=techproducts&commitName=snapshot0&followAliases=true&async=someAsyncId +---- +==== + +[example.tab-pane#v2deletesnapshot] +==== +[.tab-label]*V2 API* + +[source,bash] +---- +curl -X DELETE http://localhost:8983/api/collections/techproducts/snapshots/snapshot0?followAliases=true&async=someAsyncId +---- +==== +-- + +*Output* + +[source,json] +---- +{ + "responseHeader": { + "status": 0, + "QTime": 20 + }, + "requestid": "someAsyncId", + "collection": "techproducts", + "snapshot": "snapshot0", + "followAliases": true +} +---- + +=== DELETESNAPSHOT Parameters + +`collection`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The name of the collection to delete a snapshot from. + +`snapshot`:: ++ +[%autowidth,frame=none] +|=== +s|Required |Default: none +|=== ++ +The name of the snapshot to delete. + +`followAliases`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: false +|=== ++ +A flag that treats the collectionName parameter as an alias for the actual collection name to be resolved. + +`async`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Request ID to track this action which will be xref:configuration-guide:collections-api.adoc#asynchronous-calls[processed asynchronously].
