gerlowskija commented on code in PR #1126: URL: https://github.com/apache/solr/pull/1126#discussion_r1005968009
########## solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java: ########## @@ -278,31 +276,28 @@ public enum CoreAdminOperation implements CoreAdminOp { LISTSNAPSHOTS, it -> { final SolrParams params = it.req.getParams(); - String cname = params.required().get(CoreAdminParams.CORE); + final String coreName = params.required().get(CoreAdminParams.CORE); - CoreContainer cc = it.handler.getCoreContainer(); + final CoreContainer coreContainer = it.handler.getCoreContainer(); + final SnapshotAPI snapshotAPI = new SnapshotAPI(coreContainer); - try (SolrCore core = cc.getCore(cname)) { - if (core == null) { - throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname); - } + final SnapshotAPI.ListSnapshotsResponse response = snapshotAPI.listSnapshots(coreName); - SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); - @SuppressWarnings({"rawtypes"}) - NamedList result = new NamedList(); - for (String name : mgr.listSnapshots()) { - Optional<SnapshotMetaData> metadata = mgr.getSnapshotMetaData(name); - if (metadata.isPresent()) { - NamedList<String> props = new NamedList<>(); - props.add( - SolrSnapshotManager.GENERATION_NUM, - String.valueOf(metadata.get().getGenerationNumber())); - props.add(SolrSnapshotManager.INDEX_DIR_PATH, metadata.get().getIndexDirPath()); - result.add(name, props); - } - } - it.rsp.add(SolrSnapshotManager.SNAPSHOTS_INFO, result); + final NamedList<Object> snapshotsResult = new NamedList<>(); Review Comment: [Q] With some of our other JAX-RS based APIs, we're able to implement the v1 code by taking the strongly-typed response and calling `V2ApiUtils.squashIntoSolrResponseWIthoutHeader`. Elsewhere that's allowed us to simplify the v1 code - we don't have to copy individual fields over to the NamedList, etc. Do you think that's something that would work here, or did you have a reason to intentionally avoid that approach here? At a glance, it seems like it'd work, but there might be something I'm missing. (You can find an example of this "squashing" [here](https://github.com/apache/solr/blob/3e4e1e681a572a1b9b1f56a03ca459e1dc1544d4/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java#L1298), for reference.) ########## solr/core/src/java/org/apache/solr/handler/admin/api/SnapshotAPI.java: ########## @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; +import static org.apache.solr.security.PermissionNameProvider.Name.CORE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.CORE_READ_PERM; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Parameter; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import javax.inject.Inject; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import org.apache.lucene.index.IndexCommit; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.jersey.SolrJerseyResponse; + +@Path("/cores/{coreName}/snapshots") +public class SnapshotAPI extends JerseyResource { + + private final CoreContainer coreContainer; + + @Inject + public SnapshotAPI(CoreContainer coreContainer) { + this.coreContainer = coreContainer; + } + + @POST + @Path("/{snapshotName}") + @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @PermissionName(CORE_EDIT_PERM) + public CreateSnapshotResponse createSnapshot( + @Parameter(description = "The name of the core to snapshot.", required = true) + @PathParam("coreName") + String coreName, + @Parameter(description = "The name to associate with the core snapshot.", required = true) + @PathParam("snapshotName") + String snapshotName) + throws Exception { + final CreateSnapshotResponse response = instantiateJerseyResponse(CreateSnapshotResponse.class); + + try (SolrCore core = coreContainer.getCore(coreName)) { + if (core == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Unable to locate core " + coreName); + } + + final String indexDirPath = core.getIndexDir(); + final IndexDeletionPolicyWrapper delPol = core.getDeletionPolicy(); + final IndexCommit ic = delPol.getAndSaveLatestCommit(); + try { + if (null == ic) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "No index commits to snapshot in core " + coreName); + } + final SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); + mgr.snapshot(snapshotName, indexDirPath, ic.getGeneration()); + + response.core = core.getName(); + response.commitName = snapshotName; + response.indexDirPath = indexDirPath; + response.generation = ic.getGeneration(); + response.files = ic.getFileNames(); + } finally { + delPol.releaseCommitPoint(ic); + } + } + + return response; + } + + public static class CreateSnapshotResponse extends SolrJerseyResponse { + @JsonProperty(CoreAdminParams.CORE) Review Comment: [0] It's not required, especially in cases where the nuances of what a particular field represents are unclear, but if you DO have a good idea for any of these fields and are willing, you can use the `@Schema` annotation to document it. These descriptions end up in the OpenAPI spec generated by the Solr build, which we'll use in subsequent releases to create Solr clients for different languages (e.g. golang, Python). We're also hoping to use them to auto-generate certain sections of our ref-guide. So lots of indirect value there if you're able to document any of these. ########## solr/core/src/java/org/apache/solr/handler/admin/api/SnapshotAPI.java: ########## @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; +import static org.apache.solr.security.PermissionNameProvider.Name.CORE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.CORE_READ_PERM; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Parameter; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import javax.inject.Inject; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import org.apache.lucene.index.IndexCommit; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.jersey.SolrJerseyResponse; + +@Path("/cores/{coreName}/snapshots") +public class SnapshotAPI extends JerseyResource { Review Comment: [-1] Since we have snapshot APIs at both the "core" and "collection" levels, we should probably name this class in a way that makes it easy to distinguish which of the two it refers to. Maybe `CoreSnapshotAPI`? ########## solr/core/src/java/org/apache/solr/handler/admin/api/SnapshotAPI.java: ########## @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; +import static org.apache.solr.security.PermissionNameProvider.Name.CORE_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.CORE_READ_PERM; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.Parameter; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import javax.inject.Inject; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import org.apache.lucene.index.IndexCommit; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.jersey.SolrJerseyResponse; + +@Path("/cores/{coreName}/snapshots") +public class SnapshotAPI extends JerseyResource { + + private final CoreContainer coreContainer; + + @Inject + public SnapshotAPI(CoreContainer coreContainer) { + this.coreContainer = coreContainer; + } + + @POST + @Path("/{snapshotName}") + @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @PermissionName(CORE_EDIT_PERM) + public CreateSnapshotResponse createSnapshot( + @Parameter(description = "The name of the core to snapshot.", required = true) + @PathParam("coreName") + String coreName, + @Parameter(description = "The name to associate with the core snapshot.", required = true) + @PathParam("snapshotName") + String snapshotName) + throws Exception { + final CreateSnapshotResponse response = instantiateJerseyResponse(CreateSnapshotResponse.class); + + try (SolrCore core = coreContainer.getCore(coreName)) { + if (core == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Unable to locate core " + coreName); + } + + final String indexDirPath = core.getIndexDir(); + final IndexDeletionPolicyWrapper delPol = core.getDeletionPolicy(); + final IndexCommit ic = delPol.getAndSaveLatestCommit(); + try { + if (null == ic) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "No index commits to snapshot in core " + coreName); + } + final SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); + mgr.snapshot(snapshotName, indexDirPath, ic.getGeneration()); + + response.core = core.getName(); + response.commitName = snapshotName; + response.indexDirPath = indexDirPath; + response.generation = ic.getGeneration(); + response.files = ic.getFileNames(); + } finally { + delPol.releaseCommitPoint(ic); + } + } + + return response; + } + + public static class CreateSnapshotResponse extends SolrJerseyResponse { + @JsonProperty(CoreAdminParams.CORE) + public String core; + + @JsonProperty(CoreAdminParams.SNAPSHOT_NAME) + public String commitName; + + @JsonProperty(SolrSnapshotManager.INDEX_DIR_PATH) + public String indexDirPath; + + @JsonProperty(SolrSnapshotManager.GENERATION_NUM) + public long generation; + + @JsonProperty(SolrSnapshotManager.FILE_LIST) + public Collection<String> files; + } + + @GET + @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @PermissionName(CORE_READ_PERM) + public ListSnapshotsResponse listSnapshots( + @Parameter( + description = "The name of the core for which to retrieve snapshots.", + required = true) + @PathParam("coreName") + String coreName) { + final ListSnapshotsResponse response = instantiateJerseyResponse(ListSnapshotsResponse.class); + + try (SolrCore core = coreContainer.getCore(coreName)) { + if (core == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Unable to locate core " + coreName); + } + + SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); + + final Map<String, SnapshotInformation> result = new HashMap<>(); + for (String name : mgr.listSnapshots()) { + Optional<SolrSnapshotMetaDataManager.SnapshotMetaData> metadata = + mgr.getSnapshotMetaData(name); + if (metadata.isPresent()) { + final SnapshotInformation snapshotInformation = + new SnapshotInformation( + metadata.get().getGenerationNumber(), metadata.get().getIndexDirPath()); + result.put(name, snapshotInformation); + } + } + + response.snapshots = result; + } + + return response; + } + + public static class ListSnapshotsResponse extends SolrJerseyResponse { + @JsonProperty(SolrSnapshotManager.SNAPSHOTS_INFO) + public Map<String, SnapshotInformation> snapshots; + } + + public static class SnapshotInformation { + @JsonProperty(SolrSnapshotManager.GENERATION_NUM) + public final long generationNumber; + + @JsonProperty(SolrSnapshotManager.INDEX_DIR_PATH) + public final String indexDirPath; + + public SnapshotInformation(long generationNumber, String indexDirPath) { + this.generationNumber = generationNumber; + this.indexDirPath = indexDirPath; + } + } + + @DELETE + @Path("/{snapshotName}") + @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @PermissionName(CORE_EDIT_PERM) + public DeleteSnapshotResponse deleteSnapshot( + @Parameter( + description = "The name of the core for which to delete a snapshot.", + required = true) + @PathParam("coreName") + String coreName, + @Parameter(description = "The name of the core snapshot to delete.", required = true) + @PathParam("snapshotName") + String snapshotName) + throws Exception { + final DeleteSnapshotResponse response = instantiateJerseyResponse(DeleteSnapshotResponse.class); + + final SolrCore core = coreContainer.getCore(coreName); + if (core == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Unable to locate core " + coreName); + } + + try { + core.deleteNamedSnapshot(snapshotName); + // Ideally we shouldn't need this. This is added since the RPC logic in Review Comment: [0] I know this comment was there in the original code that you moved from DeleteSnapshotOp, so this isn't a comment on your PR, but it makes no sense to me. What is the "this" being referred to? Adding the coreName to the response? Anyway, just talking out loud I guess.... ########## solr/core/src/java/org/apache/solr/handler/admin/DeleteSnapshotOp.java: ########## @@ -17,35 +17,26 @@ package org.apache.solr.handler.admin; -import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.admin.api.SnapshotAPI; class DeleteSnapshotOp implements CoreAdminHandler.CoreAdminOp { @Override public void execute(CoreAdminHandler.CallInfo it) throws Exception { final SolrParams params = it.req.getParams(); - String commitName = params.required().get(CoreAdminParams.COMMIT_NAME); - String cname = params.required().get(CoreAdminParams.CORE); + final String commitName = params.required().get(CoreAdminParams.COMMIT_NAME); + final String coreName = params.required().get(CoreAdminParams.CORE); - CoreContainer cc = it.handler.getCoreContainer(); - SolrCore core = cc.getCore(cname); - if (core == null) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Unable to locate core " + cname); - } + final CoreContainer coreContainer = it.handler.getCoreContainer(); + final SnapshotAPI snapshotAPI = new SnapshotAPI(coreContainer); - try { - core.deleteNamedSnapshot(commitName); - // Ideally we shouldn't need this. This is added since the RPC logic in - // OverseerCollectionMessageHandler can not provide the coreName as part of the result. - it.rsp.add(CoreAdminParams.CORE, core.getName()); - it.rsp.add(CoreAdminParams.COMMIT_NAME, commitName); - } finally { - core.close(); - } + final SnapshotAPI.DeleteSnapshotResponse response = + snapshotAPI.deleteSnapshot(coreName, commitName); + + it.rsp.add(CoreAdminParams.CORE, response.coreName); Review Comment: [Q] ditto, re: can we use `V2ApiUtils.squashIntoSolrResponseWithoutHeader` here instead? ########## solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc: ########## @@ -234,38 +234,43 @@ http://localhost:8983/solr/gettingstarted/replication?command=restorestatus&wt=x The status value can be "In Progress", "success" or "failed". If it failed then an "exception" will also be sent in the response. -=== Create Snapshot API +[[create-snapshot-api]] +== CREATE: Create a Snapshot The snapshot functionality is different from the backup functionality as the index files aren't copied anywhere. The index files are snapshotted in the same index directory and can be referenced while taking backups. You can trigger a snapshot command with an HTTP command like this (replace "techproducts" with the name of the core you are working with): -.Create Snapshot API Example -[source,text] +[.dynamic-tabs] +-- +[example.tab-pane#v1createsnapshot] +==== +[.tab-label]*V1 API* + +[source,bash] ---- -http://localhost:8983/solr/admin/cores?action=CREATESNAPSHOT&core=techproducts&commitName=commit1 +curl -X POST http://localhost:8983/solr/admin/cores?action=CREATESNAPSHOT&core=techproducts&commitName=commit1 + ---- +==== -The `CREATESNAPSHOT` request parameters are: +[example.tab-pane#v2createsnapshot] +==== +[.tab-label]*V2 API* -`commitName`:: -+ -[%autowidth,frame=none] -|=== -|Optional |Default: none -|=== -+ -The name to store the snapshot as. +With the v2 API, the core and snapshot names are part of the path, no longer query parameters. Review Comment: [+1] Awesome docs, thanks for adding these! Big fan of the use of "tabs" in the docs to handle v1/v2 examples. Only nitpick I'd make here is to drop or rephrase the "no longer query parameters" bit (maybe use "instead of"?). But even that's probably personal preference. ########## solr/core/src/java/org/apache/solr/handler/admin/CreateSnapshotOp.java: ########## @@ -17,50 +17,29 @@ package org.apache.solr.handler.admin; -import org.apache.lucene.index.IndexCommit; -import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.IndexDeletionPolicyWrapper; -import org.apache.solr.core.SolrCore; import org.apache.solr.core.snapshots.SolrSnapshotManager; -import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.handler.admin.api.SnapshotAPI; class CreateSnapshotOp implements CoreAdminHandler.CoreAdminOp { @Override public void execute(CoreAdminHandler.CallInfo it) throws Exception { final SolrParams params = it.req.getParams(); - String commitName = params.required().get(CoreAdminParams.COMMIT_NAME); - String cname = params.required().get(CoreAdminParams.CORE); + final String coreName = params.required().get(CoreAdminParams.CORE); + final String commitName = params.required().get(CoreAdminParams.COMMIT_NAME); - CoreContainer cc = it.handler.getCoreContainer(); + final CoreContainer coreContainer = it.handler.getCoreContainer(); + final SnapshotAPI snapshotAPI = new SnapshotAPI(coreContainer); - try (SolrCore core = cc.getCore(cname)) { - if (core == null) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Unable to locate core " + cname); - } + final SnapshotAPI.CreateSnapshotResponse response = + snapshotAPI.createSnapshot(coreName, commitName); - final String indexDirPath = core.getIndexDir(); - final IndexDeletionPolicyWrapper delPol = core.getDeletionPolicy(); - final IndexCommit ic = delPol.getAndSaveLatestCommit(); - try { - if (null == ic) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "No index commits to snapshot in core " + cname); - } - final SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); - mgr.snapshot(commitName, indexDirPath, ic.getGeneration()); - - it.rsp.add(CoreAdminParams.CORE, core.getName()); - it.rsp.add(CoreAdminParams.COMMIT_NAME, commitName); - it.rsp.add(SolrSnapshotManager.INDEX_DIR_PATH, indexDirPath); - it.rsp.add(SolrSnapshotManager.GENERATION_NUM, ic.getGeneration()); - it.rsp.add(SolrSnapshotManager.FILE_LIST, ic.getFileNames()); - } finally { - delPol.releaseCommitPoint(ic); - } - } + it.rsp.add(CoreAdminParams.CORE, response.core); Review Comment: [Q] ditto, re: could we use `V2ApiUtils.squashIntoSolrResponseWithoutHeader` here instead of manually copying over values? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@solr.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@solr.apache.org For additional commands, e-mail: issues-h...@solr.apache.org