This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new c5ce8dd3437 SOLR-17351: Decompose filestore "get file" API
c5ce8dd3437 is described below
commit c5ce8dd3437210c6866bc6b197afb998047cc3b2
Author: Jason Gerlowski <[email protected]>
AuthorDate: Mon Feb 17 13:16:54 2025 -0500
SOLR-17351: Decompose filestore "get file" API
This commit splits up the "get file" endpoint into a number of different
APIs. Specifically:
- metadata-fetching has been moved out to the endpoint,
GET/api/cluster/filestore/metadata/some/path.txt
- Filestore commands such as pushing/pulling files are now
available at: POST /api/cluster/filestore/commands
These divisions allow us to generate SolrRequest/SolrResponse classes
representing these APIs, meaning that SolrJ users no longer need to use
GenericSolrRequest/GenericSolrResponse.
(As a 9.x backport this commit retains the original form of these APIs
to retain backwards compatibility, but this support should be removed in
10.0)
---
solr/CHANGES.txt | 6 +
.../client/api/endpoint/ClusterFileStoreApis.java | 108 ++++++++++++-
.../client/api/endpoint/NodeFileStoreApis.java | 12 +-
.../model/FileStoreDirectoryListingResponse.java | 4 -
.../java/org/apache/solr/core/CoreContainer.java | 1 +
.../apache/solr/filestore/ClusterFileStore.java | 178 ++++++++++++++++++++-
.../org/apache/solr/filestore/NodeFileStore.java | 166 +++++--------------
.../pages/package-manager-internals.adoc | 15 +-
.../solr/client/solrj/InputStreamResponse.java | 5 +-
.../apache/solr/client/solrj/util/ClientUtils.java | 2 +
.../solrj/src/resources/java-template/api.mustache | 11 ++
11 files changed, 356 insertions(+), 152 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9c53bd1effe..2f238ea0275 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -25,6 +25,12 @@ Improvements
other v2 APIs. SolrJ now offers (experimental) SolrRequest implementations
for all v2 configset APIs in
`org.apache.solr.client.solrj.request.ConfigsetsApi`. (Jason Gerlowski)
+* SOLR-17351: Solr's filestore "get-file" API has been decomposed into several
separate endpoints. Traditional file-fetching is now available at
+ `GET /api/cluster/filestore/files/some/path.txt` and metadata fetching (and
directory listing) is now available at
+ `GET /api/cluster/filestore/metadata/some/path.txt`. SolrJ now offers
request and response bindings for these APIs in
+ `org.apache.solr.client.solrj.request.FileStoreApi`. The older form of this
endpoint (`GET /api/node/files`) is deprecated and is slated to
+ be removed. (Jason Gerlowski)
+
Optimizations
---------------------
* SOLR-17578: Remove ZkController internal core supplier, for slightly faster
reconnection after Zookeeper session loss. (Pierre Salagnac)
diff --git
a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
index 11ee4e1e26d..8dbc705e7ab 100644
---
a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
+++
b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
@@ -17,6 +17,8 @@
package org.apache.solr.client.api.endpoint;
import static
org.apache.solr.client.api.util.Constants.GENERIC_ENTITY_PROPERTY;
+import static
org.apache.solr.client.api.util.Constants.OMIT_FROM_CODEGEN_PROPERTY;
+import static org.apache.solr.client.api.util.Constants.RAW_OUTPUT_PROPERTY;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
@@ -24,23 +26,26 @@ import io.swagger.v3.oas.annotations.extensions.Extension;
import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
import jakarta.ws.rs.PUT;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.QueryParam;
import java.io.InputStream;
import java.util.List;
+import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;
import org.apache.solr.client.api.model.SolrJerseyResponse;
import org.apache.solr.client.api.model.UploadToFileStoreResponse;
@Path("/cluster")
public interface ClusterFileStoreApis {
- // TODO Better understand the purpose of the 'sig' parameter and improve
docs here.
+
@PUT
@Operation(
summary = "Upload a file to the filestore.",
tags = {"file-store"})
- @Path("/files{filePath:.+}")
+ @Path("/filestore/files{filePath:.+}")
UploadToFileStoreResponse uploadFile(
@Parameter(description = "File store path") @PathParam("filePath")
String filePath,
@Parameter(description = "Signature(s) for the file being uploaded")
@QueryParam("sig")
@@ -60,7 +65,7 @@ public interface ClusterFileStoreApis {
@Operation(
summary = "Delete a file or directory from the filestore.",
tags = {"file-store"})
- @Path("/files{path:.+}")
+ @Path("/filestore/files{path:.+}")
SolrJerseyResponse deleteFile(
@Parameter(description = "Path to a file or directory within the
filestore")
@PathParam("path")
@@ -70,4 +75,101 @@ public interface ClusterFileStoreApis {
"Indicates whether the deletion should only be done on the
receiving node. For internal use only")
@QueryParam("localDelete")
Boolean localDelete);
+
+ @GET
+ @Operation(
+ summary = "Retrieve metadata about a file or directory in the
filestore.",
+ tags = {"file-store"})
+ @Path("/filestore/metadata{path:.+}")
+ FileStoreDirectoryListingResponse getMetadata(
+ @Parameter(description = "Path to a file or directory within the
filestore")
+ @PathParam("path")
+ String path);
+
+ @GET
+ @Operation(
+ summary = "Retrieve raw contents of a file in the filestore.",
+ tags = {"file-store"},
+ extensions = {
+ @Extension(properties = {@ExtensionProperty(name =
RAW_OUTPUT_PROPERTY, value = "true")})
+ })
+ @Path("/filestore/files{filePath:.+}")
+ SolrJerseyResponse getFile(
+ @Parameter(description = "Path to a file or directory within the
filestore")
+ @PathParam("filePath")
+ String path);
+
+ @POST
+ @Operation(
+ summary = "Fetches a filestore entry from other nodes in the cluster.",
+ tags = {"file-store"})
+ @Path("/filestore/commands/fetch{path:.+}")
+ SolrJerseyResponse fetchFile(
+ @Parameter(description = "Path to a file or directory within the
filestore")
+ @PathParam("path")
+ String path,
+ @Parameter(description = "An optional Solr node name to fetch the file
from")
+ @QueryParam("getFrom")
+ String getFrom);
+
+ @POST
+ @Operation(
+ summary = "Syncs a file by pushing it to other nodes in the cluster.",
+ tags = {"file-store"})
+ @Path("/filestore/commands/sync{path:.+}")
+ SolrJerseyResponse syncFile(
+ @Parameter(description = "Path to a file or directory within the
filestore")
+ @PathParam("path")
+ String path);
+
+ ////////////////////////
+ // Deprecated API bindings for 'upload' and 'delete' APIs without the
'/filestore' path segment.
+ // cf. 'uploadFile' and 'deleteFile' bindings above. Remove in 10.0
+ @PUT
+ @Operation(
+ summary = "Upload a file to the filestore. Deprecated: Use 'uploadFile'
instead",
+ tags = {"file-store"},
+ deprecated = true,
+ extensions = {
+ @Extension(
+ properties = {@ExtensionProperty(name =
OMIT_FROM_CODEGEN_PROPERTY, value = "true")})
+ })
+ @Path("/files{filePath:.+}")
+ @Deprecated
+ UploadToFileStoreResponse uploadFileDeprecated(
+ @Parameter(description = "File store path") @PathParam("filePath")
String filePath,
+ @Parameter(description = "Signature(s) for the file being uploaded")
@QueryParam("sig")
+ List<String> sig,
+ @Parameter(description = "File content to be stored in the filestore")
+ @RequestBody(
+ required = true,
+ extensions = {
+ @Extension(
+ properties = {
+ @ExtensionProperty(name = GENERIC_ENTITY_PROPERTY, value
= "true")
+ })
+ })
+ InputStream requestBody);
+
+ @DELETE
+ @Operation(
+ summary =
+ "Delete a file or directory from the filestore. Deprecated: Use
'deleteFile' instead",
+ tags = {"file-store"},
+ deprecated = true,
+ extensions = {
+ @Extension(
+ properties = {@ExtensionProperty(name =
OMIT_FROM_CODEGEN_PROPERTY, value = "true")})
+ })
+ @Path("/files{path:.+}")
+ @Deprecated
+ SolrJerseyResponse deleteFileDeprecated(
+ @Parameter(description = "Path to a file or directory within the
filestore")
+ @PathParam("path")
+ String path,
+ @Parameter(
+ description =
+ "Indicates whether the deletion should only be done on the
receiving node. For internal use only")
+ @QueryParam("localDelete")
+ Boolean localDelete);
}
diff --git
a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java
b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java
index 15f4a73cfb1..5627c385551 100644
---
a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java
+++
b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java
@@ -33,22 +33,28 @@ import org.apache.solr.client.api.model.SolrJerseyResponse;
*/
@Path("/node")
public interface NodeFileStoreApis {
+
+ /**
+ * @deprecated use {@link ClusterFileStoreApis} methods/APIs instead.
+ */
@GET
@Operation(
summary = "Retrieve file contents or metadata from the filestore.",
tags = {"file-store"},
+ deprecated = true,
// The response of this v2 API is highly variable based on the
parameters specified. It can
// return raw (potentially binary) file data, a JSON-ified
representation of that file data,
// metadata regarding one or multiple file store entries, etc. This
variability can be
// handled on the Jersey server side, but would be prohibitively
difficult to accommodate in
- // our code-generation templates. Ideally, cosmetic improvements (e.g.
splitting it up into
- // multiple endpoints) will make this unnecessary in the future. But
for now, the extension
- // property below ensures that this endpoint is ignored entirely when
doing code generation.
+ // our code-generation templates. It has been split up to be more
manageable in 10.0, but the
+ // extension property below is required in the interim to ensure that
the endpoint is ignored
+ // entirely when doing code generation.
extensions = {
@Extension(
properties = {@ExtensionProperty(name =
OMIT_FROM_CODEGEN_PROPERTY, value = "true")})
})
@Path("/files{path:.+}")
+ @Deprecated
SolrJerseyResponse getFile(
@Parameter(description = "Path to a file or directory within the
filestore")
@PathParam("path")
diff --git
a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
index bcbc5f1f728..79e002b6cee 100644
---
a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
+++
b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
@@ -19,10 +19,6 @@ package org.apache.solr.client.api.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Map;
-/**
- * One of several possible responses from {@link
- * org.apache.solr.client.api.endpoint.NodeFileStoreApis#getFile(String,
Boolean, String, Boolean)}
- */
public class FileStoreDirectoryListingResponse extends SolrJerseyResponse {
@JsonProperty public Map<String, Object> files;
}
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index b9f441801b2..1f2767302f4 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1181,6 +1181,7 @@ public class CoreContainer {
protected void configure() {
bindFactory(new
InjectionFactories.SingletonFactory<>(fileStore))
.to(DistribFileStore.class)
+ .to(FileStore.class)
.in(Singleton.class);
}
})
diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
index db422baf6bf..69df495453f 100644
--- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
@@ -18,23 +18,33 @@
package org.apache.solr.filestore;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM;
+import static org.apache.solr.response.RawResponseWriter.CONTENT;
import jakarta.inject.Inject;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.solr.api.JerseyResource;
import org.apache.solr.client.api.endpoint.ClusterFileStoreApis;
+import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;
+import org.apache.solr.client.api.model.FileStoreEntryMetadata;
import org.apache.solr.client.api.model.SolrJerseyResponse;
import org.apache.solr.client.api.model.UploadToFileStoreResponse;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.jersey.PermissionName;
import org.apache.solr.pkg.PackageAPI;
import org.apache.solr.request.SolrQueryRequest;
@@ -61,7 +71,7 @@ public class ClusterFileStore extends JerseyResource
implements ClusterFileStore
@Inject
public ClusterFileStore(
CoreContainer coreContainer,
- DistribFileStore fileStore,
+ FileStore fileStore,
SolrQueryRequest req,
SolrQueryResponse rsp) {
this.coreContainer = coreContainer;
@@ -141,6 +151,115 @@ public class ClusterFileStore extends JerseyResource
implements ClusterFileStore
return response;
}
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_READ_PERM)
+ public SolrJerseyResponse getFile(String path) {
+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+
+ final var type = fileStore.getType(path, false);
+ if (type == FileStore.FileType.NOFILE) {
+ throw new SolrException(
+ SolrException.ErrorCode.NOT_FOUND,
+ "Requested path [" + path + "] not found in filestore");
+ } else if (type == FileStore.FileType.DIRECTORY) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Requested path [" + path + "] is a directory and has no returnable
contents");
+ }
+
+ attachFileToResponse(path, fileStore, req, rsp);
+ return response;
+ }
+
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_READ_PERM)
+ public FileStoreDirectoryListingResponse getMetadata(String path) {
+ if (path == null) {
+ path = "";
+ }
+ FileStore.FileType type = fileStore.getType(path, false);
+ return getMetadata(type, path, fileStore);
+ }
+
+ public static void attachFileToResponse(
+ String path, FileStore fileStore, SolrQueryRequest req,
SolrQueryResponse rsp) {
+ ModifiableSolrParams solrParams = new ModifiableSolrParams();
+ solrParams.add(CommonParams.WT, FILE_STREAM);
+ req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));
+ rsp.add(
+ CONTENT,
+ (SolrCore.RawWriter)
+ os ->
+ fileStore.get(
+ path,
+ it -> {
+ try {
+ InputStream inputStream = it.getInputStream();
+ if (inputStream != null) {
+ inputStream.transferTo(os);
+ }
+ } catch (IOException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Error
reading file " + path);
+ }
+ },
+ false));
+ }
+
+ @SuppressWarnings("fallthrough")
+ public static FileStoreDirectoryListingResponse getMetadata(
+ FileStore.FileType type, String path, FileStore fileStore) {
+ final var dirListingResponse = new FileStoreDirectoryListingResponse();
+ if (path == null) {
+ path = "";
+ }
+
+ switch (type) {
+ case NOFILE:
+ dirListingResponse.files = Collections.singletonMap(path, null);
+ break;
+ case METADATA:
+ case FILE:
+ int idx = path.lastIndexOf('/');
+ String fileName = path.substring(idx + 1);
+ String parentPath = path.substring(0, path.lastIndexOf('/'));
+ List<FileStore.FileDetails> l = fileStore.list(parentPath, s ->
s.equals(fileName));
+
+ dirListingResponse.files =
+ Collections.singletonMap(path, l.isEmpty() ? null :
convertToResponse(l.get(0)));
+ break;
+ case DIRECTORY:
+ final var directoryContents =
+ fileStore.list(path, null).stream()
+ .map(details -> convertToResponse(details))
+ .collect(Collectors.toList());
+ dirListingResponse.files = Collections.singletonMap(path,
directoryContents);
+ break;
+ }
+
+ return dirListingResponse;
+ }
+
+ // TODO Modify the filestore implementation itself to return this object, so
conversion isn't
+ // needed.
+ private static FileStoreEntryMetadata
convertToResponse(FileStore.FileDetails details) {
+ final var entryMetadata = new FileStoreEntryMetadata();
+
+ entryMetadata.name = details.getSimpleName();
+ if (details.isDir()) {
+ entryMetadata.dir = true;
+ return entryMetadata;
+ }
+
+ entryMetadata.size = details.size();
+ entryMetadata.timestamp = details.getTimeStamp();
+ if (details.getMetaData() != null) {
+ details.getMetaData().toMap(entryMetadata.unknownProperties());
+ }
+
+ return entryMetadata;
+ }
+
private void doLocalDelete(String filePath) {
fileStore.deleteLocal(filePath);
}
@@ -194,6 +313,63 @@ public class ClusterFileStore extends JerseyResource
implements ClusterFileStore
return response;
}
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public SolrJerseyResponse fetchFile(String path, String getFrom) {
+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+ if (path == null) {
+ path = "";
+ }
+ pullFileFromNode(coreContainer, fileStore, path, getFrom);
+ return response;
+ }
+
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public SolrJerseyResponse syncFile(String path) {
+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+ syncToAllNodes(fileStore, path);
+ return response;
+ }
+
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public UploadToFileStoreResponse uploadFileDeprecated(
+ String filePath, List<String> sig, InputStream requestBody) {
+ return uploadFile(filePath, sig, requestBody);
+ }
+
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public SolrJerseyResponse deleteFileDeprecated(String path, Boolean
localDelete) {
+ return deleteFile(path, localDelete);
+ }
+
+ public static void syncToAllNodes(FileStore fileStore, String path) {
+ try {
+ fileStore.syncToAllNodes(path);
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error
getting file ", e);
+ }
+ }
+
+ public static void pullFileFromNode(
+ CoreContainer coreContainer, FileStore fileStore, String path, String
getFrom) {
+ coreContainer
+ .getUpdateShardHandler()
+ .getUpdateExecutor()
+ .submit(
+ () -> {
+ log.debug("Downloading file {}", path);
+ try {
+ fileStore.fetch(path, getFrom);
+ } catch (Exception e) {
+ log.error("Failed to download file: {}", path, e);
+ }
+ log.info("downloaded file: {}", path);
+ });
+ }
+
private List<String> readSignatures(List<String> signatures, byte[] buf)
throws SolrException, IOException {
if (signatures == null || signatures.isEmpty()) return null;
diff --git a/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java
b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java
index e6c61419437..babcc0dc551 100644
--- a/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java
@@ -17,29 +17,19 @@
package org.apache.solr.filestore;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM;
-import static org.apache.solr.response.RawResponseWriter.CONTENT;
import static
org.apache.solr.security.PermissionNameProvider.Name.FILESTORE_READ_PERM;
import jakarta.inject.Inject;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
import org.apache.solr.api.JerseyResource;
import org.apache.solr.client.api.endpoint.NodeFileStoreApis;
-import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;
-import org.apache.solr.client.api.model.FileStoreEntryMetadata;
import org.apache.solr.client.api.model.FileStoreJsonFileResponse;
import org.apache.solr.client.api.model.SolrJerseyResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrCore;
import org.apache.solr.jersey.PermissionName;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -58,7 +48,7 @@ public class NodeFileStore extends JerseyResource implements
NodeFileStoreApis {
@Inject
public NodeFileStore(
CoreContainer coreContainer,
- DistribFileStore fileStore,
+ FileStore fileStore,
SolrQueryRequest req,
SolrQueryResponse rsp) {
this.coreContainer = coreContainer;
@@ -67,21 +57,18 @@ public class NodeFileStore extends JerseyResource
implements NodeFileStoreApis {
this.fileStore = fileStore;
}
- // TODO SOLR-17351 - this single "get" operation actually supports several
different chunks of
- // functionality: syncing, directory listing, file-fetching,
metadata-fetching. We should split
- // it up into multiple distinct APIs
+ // Has been replaced by a number of distinct ClusterFileStore endpoints
+ /**
+ * @deprecated use {@link ClusterFileStore} methods instead
+ */
@Override
@PermissionName(FILESTORE_READ_PERM)
+ @Deprecated
public SolrJerseyResponse getFile(String path, Boolean sync, String getFrom,
Boolean meta) {
- final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+ final var clusterFileApi = new ClusterFileStore(coreContainer, fileStore,
req, rsp);
if (Boolean.TRUE.equals(sync)) {
- try {
- fileStore.syncToAllNodes(path);
- return response;
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error
getting file ", e);
- }
+ return clusterFileApi.syncFile(path);
}
if (path == null) {
@@ -89,122 +76,41 @@ public class NodeFileStore extends JerseyResource
implements NodeFileStoreApis {
}
final var pathCopy = path;
if (getFrom != null) {
- coreContainer
- .getUpdateShardHandler()
- .getUpdateExecutor()
- .submit(
- () -> {
- log.debug("Downloading file {}", pathCopy);
- try {
- fileStore.fetch(pathCopy, getFrom);
- } catch (Exception e) {
- log.error("Failed to download file: {}", pathCopy, e);
- }
- log.info("downloaded file: {}", pathCopy);
- });
- return response;
+ return clusterFileApi.fetchFile(path, getFrom);
}
FileStore.FileType type = fileStore.getType(path, false);
- if (type == FileStore.FileType.NOFILE) {
- final var fileMissingResponse =
- instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);
- fileMissingResponse.files = Collections.singletonMap(path, null);
- return fileMissingResponse;
+ if (type == FileStore.FileType.NOFILE
+ || type == FileStore.FileType.DIRECTORY
+ || (Boolean.TRUE.equals(meta) && type == FileStore.FileType.FILE)) {
+ return clusterFileApi.getMetadata(path);
}
- if (type == FileStore.FileType.DIRECTORY) {
- final var directoryListingResponse =
- instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);
- final var directoryContents =
- fileStore.list(path, null).stream()
- .map(details -> convertToResponse(details))
- .collect(Collectors.toList());
- directoryListingResponse.files = Collections.singletonMap(path,
directoryContents);
- return directoryListingResponse;
- }
- if (Boolean.TRUE.equals(meta)) {
- if (type == FileStore.FileType.FILE) {
- int idx = path.lastIndexOf('/');
- String fileName = path.substring(idx + 1);
- String parentPath = path.substring(0, path.lastIndexOf('/'));
- List<FileStore.FileDetails> l = fileStore.list(parentPath, s ->
s.equals(fileName));
- final var fileMetaResponse =
- instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);
- fileMetaResponse.files =
- Collections.singletonMap(path, l.isEmpty() ? null :
convertToResponse(l.get(0)));
- return fileMetaResponse;
- }
- } else { // User wants to get the "raw" file
- // TODO Should we be trying to json-ify otherwise "raw" files in this
way? It seems like a
- // pretty sketchy idea, esp. for code with very little test coverage.
Consider removing
- if ("json".equals(req.getParams().get(CommonParams.WT))) {
- final var jsonResponse =
instantiateJerseyResponse(FileStoreJsonFileResponse.class);
- try {
- fileStore.get(
- pathCopy,
- it -> {
- try {
- InputStream inputStream = it.getInputStream();
- if (inputStream != null) {
- jsonResponse.response = new
String(inputStream.readAllBytes(), UTF_8);
- }
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error reading
file " + pathCopy);
+ // Support for JSON-ified response removed in 10.0, but here for the
remainder of 9.x
+ if ("json".equals(req.getParams().get(CommonParams.WT))) {
+ final var jsonResponse =
instantiateJerseyResponse(FileStoreJsonFileResponse.class);
+ try {
+ fileStore.get(
+ pathCopy,
+ it -> {
+ try {
+ InputStream inputStream = it.getInputStream();
+ if (inputStream != null) {
+ jsonResponse.response = new
String(inputStream.readAllBytes(), UTF_8);
}
- },
- false);
- return jsonResponse;
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error getting file from
path " + path);
- }
- } else {
- ModifiableSolrParams solrParams = new ModifiableSolrParams();
- solrParams.add(CommonParams.WT, FILE_STREAM);
- req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));
- rsp.add(
- CONTENT,
- (SolrCore.RawWriter)
- os ->
- fileStore.get(
- pathCopy,
- it -> {
- try {
- InputStream inputStream = it.getInputStream();
- if (inputStream != null) {
- inputStream.transferTo(os);
- }
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR,
- "Error reading file " + pathCopy);
- }
- },
- false));
+ } catch (IOException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Error reading file
" + pathCopy);
+ }
+ },
+ false);
+ return jsonResponse;
+ } catch (IOException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Error getting file from
path " + path);
}
+ } else {
+ return clusterFileApi.getFile(path);
}
- return response;
- }
-
- // TODO Modify the filestore implementation itself to return this object, so
conversion isn't
- // needed.
- private FileStoreEntryMetadata convertToResponse(FileStore.FileDetails
details) {
- final var entryMetadata = new FileStoreEntryMetadata();
-
- entryMetadata.name = details.getSimpleName();
- if (details.isDir()) {
- entryMetadata.dir = true;
- return entryMetadata;
- }
-
- entryMetadata.size = details.size();
- entryMetadata.timestamp = details.getTimeStamp();
- if (details.getMetaData() != null) {
- details.getMetaData().toMap(entryMetadata.unknownProperties());
- }
-
- return entryMetadata;
}
}
diff --git
a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc
b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc
index aedc7166fd2..fa308238a9d 100644
---
a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc
+++
b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc
@@ -88,10 +88,9 @@ The metadata file stores the sha512 and signatures of the
jar files too.
The end points are:
-* `PUT /api/cluster/files/{full/path/to/file}` in each node
-* `GET /api/node/files/{full/path/to/file}` to download the file
-* `GET /api/node/files/{full/path/to/file}?meta=true` get the metadata of the
file
-* `GET /api/node/files/{full/path/to/}` get the list of files in
`/full/path/to`
+* `PUT /api/cluster/filestore/files/{full/path/to/file}` to upload a new file
and add it to the file store
+* `GET /api/cluster/filestore/files/{full/path/to/file}` to download a file
already in the filestore
+* `GET /api/cluster/filestore/metadata/{full/path/to/file}` to get the
metadata of a particular file, or a list of files available in a directory
=== Signing Your Artifacts
@@ -115,14 +114,14 @@ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar |
openssl enc -base64 | sed
+
[source, bash]
----
-curl --data-binary @runtimelibs.jar -X PUT
http://localhost:8983/api/cluster/files/mypkg/1.0/myplugins.jar?sig=<signature-of-jar>
+curl --data-binary @runtimelibs.jar -X PUT
http://localhost:8983/api/cluster/filestore/files/mypkg/1.0/myplugins.jar?sig=<signature-of-jar>
----
. Verify your jar upload:
+
[source, bash]
----
-curl http://localhost:8983/api/node/files/mypkg/1.0?omitHeader=true
+curl
http://localhost:8983/api/cluster/filestore/metadata/mypkg/1.0?omitHeader=true
----
+
[source, json]
@@ -328,14 +327,14 @@ curl -o runtimelibs3.jar -LO
https://github.com/apache/solr/blob/releases/solr
openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 |
sed 's/+/%2B/g' | tr -d \\n | sed
-curl --data-binary @runtimelibs3.jar -X PUT
http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=<signature>
+curl --data-binary @runtimelibs3.jar -X PUT
http://localhost:8983/api/cluster/filestore/files/mypkg/2.0/myplugins.jar?sig=<signature>
----
. Verify it:
+
[source, bash]
----
-curl http://localhost:8983/api/node/files/mypkg/2.0?omitHeader=true
+curl
http://localhost:8983/api/cluster/filestore/metadata/mypkg/2.0?omitHeader=true
----
+
[source, json]
diff --git
a/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java
index b86e76f13ac..a4cde5b5266 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java
@@ -70,8 +70,8 @@ public class InputStreamResponse extends SimpleSolrResponse {
/**
* Access the server response as an {@link InputStream}, regardless of the
HTTP status code
*
- * <p>Caller is responsible for consuming and closing the stream, and
releasing it from the
- * tracking done by {@link ObjectReleaseTracker}. No validation is done on
the HTTP status code.
+ * <p>Caller is responsible for consuming and closing the stream. No
validation is done on the
+ * HTTP status code.
*/
public InputStream getResponseStream() {
final NamedList<Object> resp = getResponse();
@@ -124,7 +124,6 @@ public class InputStreamResponse extends SimpleSolrResponse
{
inputStream.transferTo(baos);
return baos.toString(Charset.defaultCharset());
} finally {
- ObjectReleaseTracker.release(inputStream);
IOUtils.closeQuietly(baos);
IOUtils.closeQuietly(inputStream);
}
diff --git
a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
index 37af22f8721..82ecb5c68a6 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
@@ -115,6 +115,8 @@ public class ClientUtils {
final var v2Root = SolrRequest.ApiVersion.V2.getApiPrefix();
if (existingPath.contains(v1Root)) {
return existingPath.replaceFirst(v1Root, v2Root);
+ } else if (existingPath.endsWith("/api")) {
+ return existingPath;
} else if (!existingPath.contains(v2Root)) {
return existingPath + v2Root;
} else {
diff --git a/solr/solrj/src/resources/java-template/api.mustache
b/solr/solrj/src/resources/java-template/api.mustache
index f276f3f0a72..0ed28650056 100644
--- a/solr/solrj/src/resources/java-template/api.mustache
+++ b/solr/solrj/src/resources/java-template/api.mustache
@@ -23,10 +23,12 @@ import java.io.OutputStream;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
@@ -239,6 +241,15 @@ public class {{classname}} {
return params;
}
+ @Override
+ public Set<String> getQueryParams() {
+ final var queryParams = new HashSet<String>();
+ {{#queryParams}}
+ queryParams.add("{{baseName}}");
+ {{/queryParams}}
+ return queryParams;
+ }
+
@Override
protected {{operationIdCamelCase}}Response
createResponse(SolrClient client) {
return new {{operationIdCamelCase}}Response();
