This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 2ab12431c04 SOLR-17351: Decompose filestore "get file" API (#3047)
2ab12431c04 is described below
commit 2ab12431c04a3d860d21d10cf45d2434ea50bbe8
Author: Jason Gerlowski <[email protected]>
AuthorDate: Fri Feb 14 08:55:00 2025 -0800
SOLR-17351: Decompose filestore "get file" API (#3047)
This PR splits up the "get file" endpoint into a number of different APIs.
Specifically:
- metadata-fetching has been moved out to the endpoint,
GET/api/cluster/filestore/metadata/some/path.txt
- Filestore commands such as pushing/pulling files are now available at:
POST /api/cluster/filestore/commands
- Support for "JSON-ified" file data has been dropped in this PR (but
will be
retained but deprecated in the eventual 9.x backport)
These divisions allow us to generate SolrRequest/SolrResponse classes
representing these APIs, meaning that SolrJ users no longer need to use
GenericSolrRequest/GenericSolrResponse.
---
solr/CHANGES.txt | 4 +
.../client/api/endpoint/ClusterFileStoreApis.java | 52 ++++-
.../client/api/endpoint/NodeFileStoreApis.java | 67 -------
.../model/FileStoreDirectoryListingResponse.java | 4 -
.../api/model/FileStoreJsonFileResponse.java | 30 ---
.../java/org/apache/solr/core/CoreContainer.java | 2 -
.../apache/solr/filestore/ClusterFileStore.java | 163 ++++++++++++++++
.../apache/solr/filestore/DistribFileStore.java | 79 ++++----
.../org/apache/solr/filestore/NodeFileStore.java | 210 ---------------------
.../apache/solr/packagemanager/PackageManager.java | 2 +-
.../apache/solr/packagemanager/PackageUtils.java | 41 ++--
.../solr/packagemanager/RepositoryManager.java | 12 +-
.../solr/filestore/TestDistribFileStore.java | 25 ++-
.../solr/pkg/PackageStoreSchemaPluginsTest.java | 2 +-
.../pages/package-manager-internals.adoc | 15 +-
.../solr/client/solrj/InputStreamResponse.java | 6 +-
.../apache/solr/client/solrj/util/ClientUtils.java | 2 +
.../solrj/src/resources/java-template/api.mustache | 11 ++
18 files changed, 322 insertions(+), 405 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 439ecef8d7c..b619c37bdeb 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -37,6 +37,10 @@ Improvements
* SOLR-17516: `LBHttp2SolrClient` is now generic, adding support for
`HttpJdkSolrClient`. (James Dyer)
+* SOLR-17351: Solr's filestore "get-file" API has been decomposed into several
separate endpoints. Traditional file-fetching is now available at
+ `GET /api/cluster/filestore/files/some/path.txt` and metadata fetching (and
directory listing) is now available at `GET
/api/cluster/filestore/metadata/some/path.txt`.
+ SolrJ now offers request and response bindings for these APIs in
`org.apache.solr.client.solrj.request.FileStoreApi`. (Jason Gerlowski)
+
Optimizations
---------------------
* SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes
directly for data instead of proxying through one.
diff --git
a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
index 11ee4e1e26d..2445b17e7aa 100644
---
a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
+++
b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
@@ -17,6 +17,7 @@
package org.apache.solr.client.api.endpoint;
import static
org.apache.solr.client.api.util.Constants.GENERIC_ENTITY_PROPERTY;
+import static org.apache.solr.client.api.util.Constants.RAW_OUTPUT_PROPERTY;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
@@ -24,16 +25,19 @@ import io.swagger.v3.oas.annotations.extensions.Extension;
import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
import jakarta.ws.rs.PUT;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.QueryParam;
import java.io.InputStream;
import java.util.List;
+import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;
import org.apache.solr.client.api.model.SolrJerseyResponse;
import org.apache.solr.client.api.model.UploadToFileStoreResponse;
-@Path("/cluster")
+@Path("/cluster/filestore")
public interface ClusterFileStoreApis {
// TODO Better understand the purpose of the 'sig' parameter and improve
docs here.
@PUT
@@ -56,6 +60,29 @@ public interface ClusterFileStoreApis {
})
InputStream requestBody);
+ @GET
+ @Operation(
+ summary = "Retrieve metadata about a file or directory in the
filestore.",
+ tags = {"file-store"})
+ @Path("/metadata{path:.+}")
+ FileStoreDirectoryListingResponse getMetadata(
+ @Parameter(description = "Path to a file or directory within the
filestore")
+ @PathParam("path")
+ String path);
+
+ @GET
+ @Operation(
+ summary = "Retrieve raw contents of a file in the filestore.",
+ tags = {"file-store"},
+ extensions = {
+ @Extension(properties = {@ExtensionProperty(name =
RAW_OUTPUT_PROPERTY, value = "true")})
+ })
+ @Path("/files{filePath:.+}")
+ SolrJerseyResponse getFile(
+ @Parameter(description = "Path to a file or directory within the
filestore")
+ @PathParam("filePath")
+ String path);
+
@DELETE
@Operation(
summary = "Delete a file or directory from the filestore.",
@@ -70,4 +97,27 @@ public interface ClusterFileStoreApis {
"Indicates whether the deletion should only be done on the
receiving node. For internal use only")
@QueryParam("localDelete")
Boolean localDelete);
+
+ @POST
+ @Operation(
+ summary = "Fetches a filestore entry from other nodes in the cluster.",
+ tags = {"file-store"})
+ @Path("/commands/fetch{path:.+}")
+ SolrJerseyResponse fetchFile(
+ @Parameter(description = "Path to a file or directory within the
filestore")
+ @PathParam("path")
+ String path,
+ @Parameter(description = "An optional Solr node name to fetch the file
from")
+ @QueryParam("getFrom")
+ String getFrom);
+
+ @POST
+ @Operation(
+ summary = "Syncs a file by pushing it to other nodes in the cluster.",
+ tags = {"file-store"})
+ @Path("/commands/sync{path:.+}")
+ SolrJerseyResponse syncFile(
+ @Parameter(description = "Path to a file or directory within the
filestore")
+ @PathParam("path")
+ String path);
}
diff --git
a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java
b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java
deleted file mode 100644
index 15f4a73cfb1..00000000000
---
a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.api.endpoint;
-
-import static
org.apache.solr.client.api.util.Constants.OMIT_FROM_CODEGEN_PROPERTY;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.Parameter;
-import io.swagger.v3.oas.annotations.extensions.Extension;
-import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.QueryParam;
-import org.apache.solr.client.api.model.SolrJerseyResponse;
-
-/**
- * V2 APIs for fetching filestore files, syncing them across nodes, or
fetching related metadata.
- */
-@Path("/node")
-public interface NodeFileStoreApis {
- @GET
- @Operation(
- summary = "Retrieve file contents or metadata from the filestore.",
- tags = {"file-store"},
- // The response of this v2 API is highly variable based on the
parameters specified. It can
- // return raw (potentially binary) file data, a JSON-ified
representation of that file data,
- // metadata regarding one or multiple file store entries, etc. This
variability can be
- // handled on the Jersey server side, but would be prohibitively
difficult to accommodate in
- // our code-generation templates. Ideally, cosmetic improvements (e.g.
splitting it up into
- // multiple endpoints) will make this unnecessary in the future. But
for now, the extension
- // property below ensures that this endpoint is ignored entirely when
doing code generation.
- extensions = {
- @Extension(
- properties = {@ExtensionProperty(name =
OMIT_FROM_CODEGEN_PROPERTY, value = "true")})
- })
- @Path("/files{path:.+}")
- SolrJerseyResponse getFile(
- @Parameter(description = "Path to a file or directory within the
filestore")
- @PathParam("path")
- String path,
- @Parameter(
- description =
- "If true, triggers syncing for this file across all nodes in
the filestore")
- @QueryParam("sync")
- Boolean sync,
- @Parameter(description = "An optional Solr node name to fetch the file
from")
- @QueryParam("getFrom")
- String getFrom,
- @Parameter(description = "Indicates that (only) file metadata should be
fetched")
- @QueryParam("meta")
- Boolean meta);
-}
diff --git
a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
index bcbc5f1f728..79e002b6cee 100644
---
a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
+++
b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
@@ -19,10 +19,6 @@ package org.apache.solr.client.api.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Map;
-/**
- * One of several possible responses from {@link
- * org.apache.solr.client.api.endpoint.NodeFileStoreApis#getFile(String,
Boolean, String, Boolean)}
- */
public class FileStoreDirectoryListingResponse extends SolrJerseyResponse {
@JsonProperty public Map<String, Object> files;
}
diff --git
a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java
b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java
deleted file mode 100644
index 321faec01ec..00000000000
---
a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.client.api.model;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * One of several possible responses from {@link
- * org.apache.solr.client.api.endpoint.NodeFileStoreApis#getFile(String,
Boolean, String, Boolean)}
- *
- * <p>Typically used when 'wt=json' is specified while retrieving an
individual file from the
- * filestore
- */
-public class FileStoreJsonFileResponse extends SolrJerseyResponse {
- @JsonProperty public String response;
-}
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index e791b3ca602..1ed72a71a7a 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -110,7 +110,6 @@ import
org.apache.solr.core.backup.repository.BackupRepositoryFactory;
import org.apache.solr.filestore.ClusterFileStore;
import org.apache.solr.filestore.DistribFileStore;
import org.apache.solr.filestore.FileStore;
-import org.apache.solr.filestore.NodeFileStore;
import org.apache.solr.handler.ClusterAPI;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.SnapShooter;
@@ -880,7 +879,6 @@ public class CoreContainer {
fileStore = new DistribFileStore(this);
registerV2ApiIfEnabled(ClusterFileStore.class);
- registerV2ApiIfEnabled(NodeFileStore.class);
packageLoader = new SolrPackageLoader(this);
registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI);
diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
index db422baf6bf..043192aa0d8 100644
--- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
@@ -18,23 +18,33 @@
package org.apache.solr.filestore;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM;
+import static org.apache.solr.response.RawResponseWriter.CONTENT;
import jakarta.inject.Inject;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.solr.api.JerseyResource;
import org.apache.solr.client.api.endpoint.ClusterFileStoreApis;
+import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;
+import org.apache.solr.client.api.model.FileStoreEntryMetadata;
import org.apache.solr.client.api.model.SolrJerseyResponse;
import org.apache.solr.client.api.model.UploadToFileStoreResponse;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.jersey.PermissionName;
import org.apache.solr.pkg.PackageAPI;
import org.apache.solr.request.SolrQueryRequest;
@@ -141,6 +151,115 @@ public class ClusterFileStore extends JerseyResource
implements ClusterFileStore
return response;
}
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_READ_PERM)
+ public SolrJerseyResponse getFile(String path) {
+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+
+ final var type = fileStore.getType(path, false);
+ if (type == FileStore.FileType.NOFILE) {
+ throw new SolrException(
+ SolrException.ErrorCode.NOT_FOUND,
+ "Requested path [" + path + "] not found in filestore");
+ } else if (type == FileStore.FileType.DIRECTORY) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Requested path [" + path + "] is a directory and has no returnable
contents");
+ }
+
+ attachFileToResponse(path, fileStore, req, rsp);
+ return response;
+ }
+
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_READ_PERM)
+ public FileStoreDirectoryListingResponse getMetadata(String path) {
+ if (path == null) {
+ path = "";
+ }
+ FileStore.FileType type = fileStore.getType(path, false);
+ return getMetadata(type, path, fileStore);
+ }
+
+ public static void attachFileToResponse(
+ String path, FileStore fileStore, SolrQueryRequest req,
SolrQueryResponse rsp) {
+ ModifiableSolrParams solrParams = new ModifiableSolrParams();
+ solrParams.add(CommonParams.WT, FILE_STREAM);
+ req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));
+ rsp.add(
+ CONTENT,
+ (SolrCore.RawWriter)
+ os ->
+ fileStore.get(
+ path,
+ it -> {
+ try {
+ InputStream inputStream = it.getInputStream();
+ if (inputStream != null) {
+ inputStream.transferTo(os);
+ }
+ } catch (IOException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Error
reading file " + path);
+ }
+ },
+ false));
+ }
+
+ @SuppressWarnings("fallthrough")
+ public static FileStoreDirectoryListingResponse getMetadata(
+ FileStore.FileType type, String path, FileStore fileStore) {
+ final var dirListingResponse = new FileStoreDirectoryListingResponse();
+ if (path == null) {
+ path = "";
+ }
+
+ switch (type) {
+ case NOFILE:
+ dirListingResponse.files = Collections.singletonMap(path, null);
+ break;
+ case METADATA:
+ case FILE:
+ int idx = path.lastIndexOf('/');
+ String fileName = path.substring(idx + 1);
+ String parentPath = path.substring(0, path.lastIndexOf('/'));
+ List<FileStore.FileDetails> l = fileStore.list(parentPath, s ->
s.equals(fileName));
+
+ dirListingResponse.files =
+ Collections.singletonMap(path, l.isEmpty() ? null :
convertToResponse(l.get(0)));
+ break;
+ case DIRECTORY:
+ final var directoryContents =
+ fileStore.list(path, null).stream()
+ .map(details -> convertToResponse(details))
+ .collect(Collectors.toList());
+ dirListingResponse.files = Collections.singletonMap(path,
directoryContents);
+ break;
+ }
+
+ return dirListingResponse;
+ }
+
+ // TODO Modify the filestore implementation itself to return this object, so
conversion isn't
+ // needed.
+ private static FileStoreEntryMetadata
convertToResponse(FileStore.FileDetails details) {
+ final var entryMetadata = new FileStoreEntryMetadata();
+
+ entryMetadata.name = details.getSimpleName();
+ if (details.isDir()) {
+ entryMetadata.dir = true;
+ return entryMetadata;
+ }
+
+ entryMetadata.size = details.size();
+ entryMetadata.timestamp = details.getTimeStamp();
+ if (details.getMetaData() != null) {
+ details.getMetaData().toMap(entryMetadata.unknownProperties());
+ }
+
+ return entryMetadata;
+ }
+
private void doLocalDelete(String filePath) {
fileStore.deleteLocal(filePath);
}
@@ -194,6 +313,50 @@ public class ClusterFileStore extends JerseyResource
implements ClusterFileStore
return response;
}
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public SolrJerseyResponse fetchFile(String path, String getFrom) {
+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+ if (path == null) {
+ path = "";
+ }
+ pullFileFromNode(coreContainer, fileStore, path, getFrom);
+ return response;
+ }
+
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public SolrJerseyResponse syncFile(String path) {
+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+ syncToAllNodes(fileStore, path);
+ return response;
+ }
+
+ public static void syncToAllNodes(FileStore fileStore, String path) {
+ try {
+ fileStore.syncToAllNodes(path);
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error
getting file ", e);
+ }
+ }
+
+ public static void pullFileFromNode(
+ CoreContainer coreContainer, FileStore fileStore, String path, String
getFrom) {
+ coreContainer
+ .getUpdateShardHandler()
+ .getUpdateExecutor()
+ .submit(
+ () -> {
+ log.debug("Downloading file {}", path);
+ try {
+ fileStore.fetch(path, getFrom);
+ } catch (Exception e) {
+ log.error("Failed to download file: {}", path, e);
+ }
+ log.info("downloaded file: {}", path);
+ });
+ }
+
private List<String> readSignatures(List<String> signatures, byte[] buf)
throws SolrException, IOException {
if (signatures == null || signatures.isEmpty()) return null;
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java
b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java
index 00e71968250..42679fd51ce 100644
--- a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java
@@ -18,8 +18,6 @@
package org.apache.solr.filestore;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.client.solrj.SolrRequest.METHOD.DELETE;
-import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
@@ -36,7 +34,6 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -50,11 +47,9 @@ import net.jcip.annotations.NotThreadSafe;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.FileStoreApi;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrPaths;
@@ -192,26 +187,31 @@ public class DistribFileStore implements FileStore {
var solrClient = coreContainer.getDefaultHttpSolrClient();
try {
- GenericSolrRequest request = new GenericSolrRequest(GET, "/node/files"
+ getMetaPath());
- request.setResponseParser(new InputStreamResponseParser(null));
- var response = solrClient.requestWithBaseUrl(baseUrl,
request::process).getResponse();
- is = (InputStream) response.get("stream");
+ final var metadataRequest = new FileStoreApi.GetFile(getMetaPath());
+ final var client =
coreContainer.getSolrClientCache().getHttpSolrClient(baseUrl);
+ final var response = metadataRequest.process(client);
metadata =
- Utils.newBytesConsumer((int) MAX_PKG_SIZE).accept((InputStream)
response.get("stream"));
+ Utils.newBytesConsumer((int) MAX_PKG_SIZE)
+ .accept(response.getResponseStreamIfSuccessful());
m = (Map<?, ?>) Utils.fromJSON(metadata.array(),
metadata.arrayOffset(), metadata.limit());
- } catch (SolrServerException | IOException e) {
+ } catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error
fetching metadata", e);
} finally {
org.apache.solr.common.util.IOUtils.closeQuietly(is);
}
+ ByteBuffer filedata = null;
+ try {
+ final var fileRequest = new FileStoreApi.GetFile(path);
+ final var fileResponse = solrClient.requestWithBaseUrl(baseUrl, null,
fileRequest);
+ try (final var stream = fileResponse.getResponseStreamIfSuccessful()) {
+ filedata = Utils.newBytesConsumer((int) MAX_PKG_SIZE).accept(stream);
+ }
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error
fetching data", e);
+ }
+
try {
- GenericSolrRequest request = new GenericSolrRequest(GET, "/node/files"
+ path);
- request.setResponseParser(new InputStreamResponseParser(null));
- var response = solrClient.requestWithBaseUrl(baseUrl,
request::process).getResponse();
- is = (InputStream) response.get("stream");
- ByteBuffer filedata =
- Utils.newBytesConsumer((int) MAX_PKG_SIZE).accept((InputStream)
response.get("stream"));
filedata.mark();
String sha512 = DigestUtils.sha512Hex(new
ByteBufferInputStream(filedata));
String expected = (String) m.get("sha512");
@@ -222,8 +222,6 @@ public class DistribFileStore implements FileStore {
filedata.reset();
persistToFile(filedata, metadata);
return true;
- } catch (SolrServerException e) {
- throw new SolrException(SERVER_ERROR, "Error fetching data", e);
} catch (IOException ioe) {
throw new SolrException(SERVER_ERROR, "Error persisting file", ioe);
} finally {
@@ -238,18 +236,12 @@ public class DistribFileStore implements FileStore {
try {
String baseUrl =
coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(liveNode);
- final var solrParams = new ModifiableSolrParams();
- solrParams.add("meta", "true");
- solrParams.add("omitHeader", "true");
+ final var metadataRequest = new FileStoreApi.GetMetadata(path);
+ final var client =
coreContainer.getSolrClientCache().getHttpSolrClient(baseUrl);
+ final var metadataResponse =
metadataRequest.process(client).getParsed();
+ boolean nodeHasBlob =
+ metadataResponse.files != null &&
metadataResponse.files.containsKey(path);
- final var request = new GenericSolrRequest(GET, "/node/files" +
path, solrParams);
- boolean nodeHasBlob = false;
- var solrClient = coreContainer.getDefaultHttpSolrClient();
- var resp = solrClient.requestWithBaseUrl(baseUrl,
request::process).getResponse();
-
- if (Utils.getObjectByPath(resp, false, Arrays.asList("files", path))
!= null) {
- nodeHasBlob = true;
- }
if (nodeHasBlob) {
boolean success = fetchFileFromNodeAndPersist(liveNode);
if (success) return true;
@@ -367,11 +359,13 @@ public class DistribFileStore implements FileStore {
for (String node : nodes) {
String baseUrl =
coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(node);
+
+ String nodeToFetchFrom;
if (i < FETCHFROM_SRC) {
// this is to protect very large clusters from overwhelming a single
node
// the first FETCHFROM_SRC nodes will be asked to fetch from this
node.
// it's there in the memory now. So , it must be served fast
- getFrom = myNodeName;
+ nodeToFetchFrom = myNodeName;
} else {
if (i == FETCHFROM_SRC) {
// This is just an optimization
@@ -382,19 +376,17 @@ public class DistribFileStore implements FileStore {
} catch (Exception e) {
}
}
- // trying to avoid the thundering herd problem when there are a very
large no:of nodes
- // others should try to fetch it from any node where it is
available. By now,
+ // trying to avoid the thundering herd problem when there are a very
large number of
+ // nodes others should try to fetch it from any node where it is
available. By now,
// almost FETCHFROM_SRC other nodes may have it
- getFrom = "*";
+ nodeToFetchFrom = "*";
}
try {
- var solrClient = coreContainer.getDefaultHttpSolrClient();
- var solrParams = new ModifiableSolrParams();
- solrParams.set("getFrom", getFrom);
-
- var request = new GenericSolrRequest(GET, "/node/files" + info.path,
solrParams);
+ final var pullFileRequest = new FileStoreApi.FetchFile(info.path);
+ pullFileRequest.setGetFrom(nodeToFetchFrom);
+ final var client =
coreContainer.getSolrClientCache().getHttpSolrClient(baseUrl);
// fire and forget
- solrClient.requestWithBaseUrl(baseUrl, request::process);
+ pullFileRequest.process(client);
} catch (Exception e) {
log.info("Node: {} failed to respond for file fetch notification",
node, e);
// ignore the exception
@@ -510,9 +502,8 @@ public class DistribFileStore implements FileStore {
deleteLocal(path);
List<String> nodes =
FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer);
- final var solrParams = new ModifiableSolrParams();
- solrParams.add("localDelete", "true");
- final var solrRequest = new GenericSolrRequest(DELETE, "/cluster/files" +
path, solrParams);
+ final var solrRequest = new FileStoreApi.DeleteFile(path);
+ solrRequest.setLocalDelete(Boolean.TRUE);
for (String node : nodes) {
String baseUrl =
diff --git a/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java
b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java
deleted file mode 100644
index e6c61419437..00000000000
--- a/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.filestore;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM;
-import static org.apache.solr.response.RawResponseWriter.CONTENT;
-import static
org.apache.solr.security.PermissionNameProvider.Name.FILESTORE_READ_PERM;
-
-import jakarta.inject.Inject;
-import java.io.IOException;
-import java.io.InputStream;
-import java.lang.invoke.MethodHandles;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-import org.apache.solr.api.JerseyResource;
-import org.apache.solr.client.api.endpoint.NodeFileStoreApis;
-import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;
-import org.apache.solr.client.api.model.FileStoreEntryMetadata;
-import org.apache.solr.client.api.model.FileStoreJsonFileResponse;
-import org.apache.solr.client.api.model.SolrJerseyResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.jersey.PermissionName;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** Implementation for {@link NodeFileStoreApis} */
-public class NodeFileStore extends JerseyResource implements NodeFileStoreApis
{
- private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private final CoreContainer coreContainer;
- private final SolrQueryRequest req;
- private final SolrQueryResponse rsp;
- private final FileStore fileStore;
-
- @Inject
- public NodeFileStore(
- CoreContainer coreContainer,
- DistribFileStore fileStore,
- SolrQueryRequest req,
- SolrQueryResponse rsp) {
- this.coreContainer = coreContainer;
- this.req = req;
- this.rsp = rsp;
- this.fileStore = fileStore;
- }
-
- // TODO SOLR-17351 - this single "get" operation actually supports several
different chunks of
- // functionality: syncing, directory listing, file-fetching,
metadata-fetching. We should split
- // it up into multiple distinct APIs
- @Override
- @PermissionName(FILESTORE_READ_PERM)
- public SolrJerseyResponse getFile(String path, Boolean sync, String getFrom,
Boolean meta) {
- final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
-
- if (Boolean.TRUE.equals(sync)) {
- try {
- fileStore.syncToAllNodes(path);
- return response;
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error
getting file ", e);
- }
- }
-
- if (path == null) {
- path = "";
- }
- final var pathCopy = path;
- if (getFrom != null) {
- coreContainer
- .getUpdateShardHandler()
- .getUpdateExecutor()
- .submit(
- () -> {
- log.debug("Downloading file {}", pathCopy);
- try {
- fileStore.fetch(pathCopy, getFrom);
- } catch (Exception e) {
- log.error("Failed to download file: {}", pathCopy, e);
- }
- log.info("downloaded file: {}", pathCopy);
- });
- return response;
- }
-
- FileStore.FileType type = fileStore.getType(path, false);
- if (type == FileStore.FileType.NOFILE) {
- final var fileMissingResponse =
- instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);
- fileMissingResponse.files = Collections.singletonMap(path, null);
- return fileMissingResponse;
- }
- if (type == FileStore.FileType.DIRECTORY) {
- final var directoryListingResponse =
- instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);
- final var directoryContents =
- fileStore.list(path, null).stream()
- .map(details -> convertToResponse(details))
- .collect(Collectors.toList());
- directoryListingResponse.files = Collections.singletonMap(path,
directoryContents);
- return directoryListingResponse;
- }
- if (Boolean.TRUE.equals(meta)) {
- if (type == FileStore.FileType.FILE) {
- int idx = path.lastIndexOf('/');
- String fileName = path.substring(idx + 1);
- String parentPath = path.substring(0, path.lastIndexOf('/'));
- List<FileStore.FileDetails> l = fileStore.list(parentPath, s ->
s.equals(fileName));
-
- final var fileMetaResponse =
- instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);
- fileMetaResponse.files =
- Collections.singletonMap(path, l.isEmpty() ? null :
convertToResponse(l.get(0)));
- return fileMetaResponse;
- }
- } else { // User wants to get the "raw" file
- // TODO Should we be trying to json-ify otherwise "raw" files in this
way? It seems like a
- // pretty sketchy idea, esp. for code with very little test coverage.
Consider removing
- if ("json".equals(req.getParams().get(CommonParams.WT))) {
- final var jsonResponse =
instantiateJerseyResponse(FileStoreJsonFileResponse.class);
- try {
- fileStore.get(
- pathCopy,
- it -> {
- try {
- InputStream inputStream = it.getInputStream();
- if (inputStream != null) {
- jsonResponse.response = new
String(inputStream.readAllBytes(), UTF_8);
- }
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error reading
file " + pathCopy);
- }
- },
- false);
- return jsonResponse;
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error getting file from
path " + path);
- }
- } else {
- ModifiableSolrParams solrParams = new ModifiableSolrParams();
- solrParams.add(CommonParams.WT, FILE_STREAM);
- req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));
- rsp.add(
- CONTENT,
- (SolrCore.RawWriter)
- os ->
- fileStore.get(
- pathCopy,
- it -> {
- try {
- InputStream inputStream = it.getInputStream();
- if (inputStream != null) {
- inputStream.transferTo(os);
- }
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR,
- "Error reading file " + pathCopy);
- }
- },
- false));
- }
- }
- return response;
- }
-
- // TODO Modify the filestore implementation itself to return this object, so
conversion isn't
- // needed.
- private FileStoreEntryMetadata convertToResponse(FileStore.FileDetails
details) {
- final var entryMetadata = new FileStoreEntryMetadata();
-
- entryMetadata.name = details.getSimpleName();
- if (details.isDir()) {
- entryMetadata.dir = true;
- return entryMetadata;
- }
-
- entryMetadata.size = details.size();
- entryMetadata.timestamp = details.getTimeStamp();
- if (details.getMetaData() != null) {
- details.getMetaData().toMap(entryMetadata.unknownProperties());
- }
-
- return entryMetadata;
- }
-}
diff --git
a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java
b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java
index 1d69a9c429d..07fb72965eb 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java
@@ -173,7 +173,7 @@ public class PackageManager implements Closeable {
String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version,
"manifest.json"));
for (String filePath : filesToDelete) {
DistribFileStore.deleteZKFileEntry(zkClient, filePath);
- String path = "/api/cluster/files" + filePath;
+ String path = "/api/cluster/filestore/files" + filePath;
printGreen("Deleting " + path);
solrClient.request(new GenericSolrRequest(SolrRequest.METHOD.DELETE,
path));
}
diff --git
a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java
b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java
index 1d1270a2b84..b0bb80cef42 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
+import org.apache.commons.io.IOUtils;
import org.apache.lucene.util.SuppressForbidden;
import org.apache.solr.cli.CLIUtils;
import org.apache.solr.client.solrj.SolrClient;
@@ -40,7 +41,6 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.JsonMapResponseParser;
import org.apache.solr.client.solrj.request.FileStoreApi;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
-import org.apache.solr.client.solrj.request.GenericV2SolrRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -182,26 +182,27 @@ public class PackageUtils {
public static Manifest fetchManifest(
SolrClient solrClient, String manifestFilePath, String expectedSHA512)
throws IOException, SolrServerException {
- GenericSolrRequest request =
- new GenericV2SolrRequest(SolrRequest.METHOD.GET, "/api/node/files" +
manifestFilePath);
- request.setResponseParser(new JsonMapResponseParser());
- NamedList<Object> response = solrClient.request(request);
- String manifestJson = (String) response.get("response");
- String calculatedSHA512 =
-
Utils.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)));
- if (expectedSHA512.equals(calculatedSHA512) == false) {
- throw new SolrException(
- ErrorCode.UNAUTHORIZED,
- "The manifest SHA512 doesn't match expected SHA512. Possible
unauthorized manipulation. "
- + "Expected: "
- + expectedSHA512
- + ", calculated: "
- + calculatedSHA512
- + ", manifest location: "
- + manifestFilePath);
+
+ final var manifestRequest = new FileStoreApi.GetFile(manifestFilePath);
+ final var manifestResponse = manifestRequest.process(solrClient);
+ try (final var manifestStream =
manifestResponse.getResponseStreamIfSuccessful()) {
+ final var manifestJson = IOUtils.toString(manifestStream,
StandardCharsets.UTF_8);
+ String calculatedSHA512 =
+
Utils.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)));
+ if (!expectedSHA512.equals(calculatedSHA512)) {
+ throw new SolrException(
+ ErrorCode.UNAUTHORIZED,
+ "The manifest SHA512 doesn't match expected SHA512. Possible
unauthorized manipulation. "
+ + "Expected: "
+ + expectedSHA512
+ + ", calculated: "
+ + calculatedSHA512
+ + ", manifest location: "
+ + manifestFilePath);
+ }
+ Manifest manifest = getMapper().readValue(manifestJson, Manifest.class);
+ return manifest;
}
- Manifest manifest = getMapper().readValue(manifestJson, Manifest.class);
- return manifest;
}
/**
diff --git
a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java
b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java
index c855f52961e..7c5306ceabc 100644
--- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java
+++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java
@@ -44,6 +44,7 @@ import org.apache.solr.client.api.util.SolrVersion;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.FileStoreApi;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.GenericV2SolrRequest;
import org.apache.solr.client.solrj.request.RequestWriter;
@@ -51,7 +52,6 @@ import
org.apache.solr.client.solrj.request.beans.PackagePayload;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.filestore.ClusterFileStore;
@@ -151,8 +151,14 @@ public class RepositoryManager {
// put the public key into package store's trusted key store and request a
sync.
String path = ClusterFileStore.KEYS_DIR + "/" + destinationKeyFilename;
PackageUtils.uploadKey(key, path, Paths.get(solrHome));
- PackageUtils.getJsonStringFromNonCollectionApi(
- solrClient, "/api/node/files" + path, new
ModifiableSolrParams().add("sync", "true"));
+ final var syncRequest = new FileStoreApi.SyncFile(path);
+ final var syncResponse = syncRequest.process(solrClient).getParsed();
+ final var status = syncResponse.responseHeader.status;
+ if (status != 0) {
+ throw new SolrException(
+ ErrorCode.getErrorCode(status),
+ "Unexpected status " + status + " while syncing filestore upload.");
+ }
}
private String getRepositoriesJson(SolrZkClient zkClient)
diff --git
a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java
b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java
index ff8f68b7897..95732ea3a1c 100644
--- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java
+++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java
@@ -40,6 +40,7 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import
org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteExecutionException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.FileStoreApi;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
import org.apache.solr.client.solrj.response.V2Response;
@@ -115,7 +116,7 @@ public class TestDistribFileStore extends SolrCloudTestCase
{
assertResponseValues(
10,
cluster.getSolrClient(),
- new V2Request.Builder("/node/files/package/mypkg/v1.0")
+ new
V2Request.Builder("/cluster/filestore/metadata/package/mypkg/v1.0")
.withMethod(SolrRequest.METHOD.GET)
.build(),
Map.of(
@@ -128,7 +129,7 @@ public class TestDistribFileStore extends SolrCloudTestCase
{
assertResponseValues(
10,
cluster.getSolrClient(),
- new V2Request.Builder("/node/files/package/mypkg")
+ new V2Request.Builder("/cluster/filestore/metadata/package/mypkg")
.withMethod(SolrRequest.METHOD.GET)
.build(),
Map.of(
@@ -169,14 +170,15 @@ public class TestDistribFileStore extends
SolrCloudTestCase {
});
for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
String baseUrl = jettySolrRunner.getBaseURLV2().toString();
- String url = baseUrl + "/node/files/package/mypkg/v1.0?wt=javabin";
+ String url = baseUrl +
"/cluster/filestore/metadata/package/mypkg/v1.0?wt=javabin";
assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);
}
// Delete Jars
DistribFileStore.deleteZKFileEntry(
cluster.getZkClient(), "/package/mypkg/v1.0/runtimelibs.jar");
JettySolrRunner j = cluster.getRandomJetty(random());
- String path = j.getBaseURLV2() + "/cluster/files" +
"/package/mypkg/v1.0/runtimelibs.jar";
+ String path =
+ j.getBaseURLV2() + "/cluster/filestore/files" +
"/package/mypkg/v1.0/runtimelibs.jar";
HttpDelete del = new HttpDelete(path);
try (HttpSolrClient cl = (HttpSolrClient) j.newClient()) {
Utils.executeHttpMethod(cl.getHttpClient(), path, Utils.JSONCONSUMER,
del);
@@ -196,7 +198,7 @@ public class TestDistribFileStore extends SolrCloudTestCase
{
throws Exception {
for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
String baseUrl = jettySolrRunner.getBaseURLV2().toString();
- String url = baseUrl + "/node/files" + path + "?wt=javabin&meta=true";
+ String url = baseUrl + "/cluster/filestore/metadata" + path +
"?wt=javabin";
assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);
if (verifyContent) {
@@ -204,7 +206,7 @@ public class TestDistribFileStore extends SolrCloudTestCase
{
ByteBuffer buf =
Utils.executeGET(
solrClient.getHttpClient(),
- baseUrl + "/node/files" + path,
+ baseUrl + "/cluster/filestore/files" + path,
Utils.newBytesConsumer(Integer.MAX_VALUE));
assertEquals(
"d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",
@@ -319,9 +321,12 @@ public class TestDistribFileStore extends
SolrCloudTestCase {
JettySolrRunner jetty = cluster.getRandomJetty(random());
try (HttpSolrClient client = (HttpSolrClient) jetty.newClient()) {
PackageUtils.uploadKey(bytes, path,
Paths.get(jetty.getCoreContainer().getSolrHome()));
- String url = jetty.getBaseURLV2() + "/node/files" + path + "?sync=true";
- Object resp = Utils.executeGET(client.getHttpClient(), url, null);
- log.info("sync resp: {} was {}", url, resp);
+
+ final var syncReq = new FileStoreApi.SyncFile(path);
+ final var syncRsp = syncReq.process(client);
+ if (log.isInfoEnabled()) {
+ log.info("sync resp for path {} was {}", path,
syncRsp.getParsed().responseHeader.status);
+ }
}
checkAllNodesForFile(
cluster,
@@ -333,7 +338,7 @@ public class TestDistribFileStore extends SolrCloudTestCase
{
public static NavigableObject postFile(
SolrClient client, ByteBuffer buffer, String name, String sig)
throws SolrServerException, IOException {
- String resource = "/cluster/files" + name;
+ String resource = "/cluster/filestore/files" + name;
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("sig", sig);
V2Response rsp =
diff --git
a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java
b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java
index 26d8261c07f..5fa62780a6b 100644
--- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java
+++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java
@@ -124,7 +124,7 @@ public class PackageStoreSchemaPluginsTest extends
SolrCloudTestCase {
private void uploadPluginJar(String version, Path jarPath) throws Exception {
var pluginRequest =
- new V2Request.Builder("/cluster/files/my-plugin/plugin-" + version +
".jar")
+ new V2Request.Builder("/cluster/filestore/files/my-plugin/plugin-" +
version + ".jar")
.PUT()
.withParams(params("sig", signature(Files.readAllBytes(jarPath))))
.withPayload(Files.newInputStream(jarPath))
diff --git
a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc
b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc
index df8ed1f3907..33b6afcc2a4 100644
---
a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc
+++
b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc
@@ -88,10 +88,9 @@ The metadata file stores the sha512 and signatures of the
jar files too.
The end points are:
-* `PUT /api/cluster/files/{full/path/to/file}` in each node
-* `GET /api/node/files/{full/path/to/file}` to download the file
-* `GET /api/node/files/{full/path/to/file}?meta=true` get the metadata of the
file
-* `GET /api/node/files/{full/path/to/}` get the list of files in
`/full/path/to`
+* `PUT /api/cluster/filestore/files/{full/path/to/file}` to upload a new file
and add it to the file store
+* `GET /api/cluster/filestore/files/{full/path/to/file}` to download a file
already in the filestore
+* `GET /api/cluster/filestore/metadata/{full/path/to/file}` to get the
metadata of a particular file, or a list of files available in a directory
=== Signing Your Artifacts
@@ -115,14 +114,14 @@ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar |
openssl enc -base64 | sed
+
[source, bash]
----
-curl --data-binary @runtimelibs.jar -X PUT
http://localhost:8983/api/cluster/files/mypkg/1.0/myplugins.jar?sig=<signature-of-jar>
+curl --data-binary @runtimelibs.jar -X PUT
http://localhost:8983/api/cluster/filestore/files/mypkg/1.0/myplugins.jar?sig=<signature-of-jar>
----
. Verify your jar upload:
+
[source, bash]
----
-curl http://localhost:8983/api/node/files/mypkg/1.0?omitHeader=true
+curl
http://localhost:8983/api/cluster/filestore/metadata/mypkg/1.0?omitHeader=true
----
+
[source, json]
@@ -328,14 +327,14 @@ curl -o runtimelibs3.jar -LO
https://github.com/apache/solr/blob/releases/solr
openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 |
sed 's/+/%2B/g' | tr -d \\n | sed
-curl --data-binary @runtimelibs3.jar -X PUT
http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=<signature>
+curl --data-binary @runtimelibs3.jar -X PUT
http://localhost:8983/api/cluster/filestore/files/mypkg/2.0/myplugins.jar?sig=<signature>
----
. Verify it:
+
[source, bash]
----
-curl http://localhost:8983/api/node/files/mypkg/2.0?omitHeader=true
+curl
http://localhost:8983/api/cluster/filestore/metadata/mypkg/2.0?omitHeader=true
----
+
[source, json]
diff --git
a/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java
index b86e76f13ac..6770ce81c53 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java
@@ -70,12 +70,11 @@ public class InputStreamResponse extends SimpleSolrResponse
{
/**
* Access the server response as an {@link InputStream}, regardless of the
HTTP status code
*
- * <p>Caller is responsible for consuming and closing the stream, and
releasing it from the
- * tracking done by {@link ObjectReleaseTracker}. No validation is done on
the HTTP status code.
+ * <p>Caller is responsible for consuming and closing the stream. No
validation is done on the
+ * HTTP status code.
*/
public InputStream getResponseStream() {
final NamedList<Object> resp = getResponse();
-
return (InputStream) resp.get(STREAM_KEY);
}
@@ -124,7 +123,6 @@ public class InputStreamResponse extends SimpleSolrResponse
{
inputStream.transferTo(baos);
return baos.toString(Charset.defaultCharset());
} finally {
- ObjectReleaseTracker.release(inputStream);
IOUtils.closeQuietly(baos);
IOUtils.closeQuietly(inputStream);
}
diff --git
a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
index 646ef546c99..7940b8ddce3 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
@@ -97,6 +97,8 @@ public class ClientUtils {
private static String buildReplacementV2Path(String existingPath) {
if (existingPath.contains("/solr")) {
return existingPath.replaceFirst("/solr", "/api");
+ } else if (existingPath.endsWith("/api")) {
+ return existingPath;
} else {
return existingPath + "/api";
}
diff --git a/solr/solrj/src/resources/java-template/api.mustache
b/solr/solrj/src/resources/java-template/api.mustache
index f276f3f0a72..0ed28650056 100644
--- a/solr/solrj/src/resources/java-template/api.mustache
+++ b/solr/solrj/src/resources/java-template/api.mustache
@@ -23,10 +23,12 @@ import java.io.OutputStream;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
@@ -239,6 +241,15 @@ public class {{classname}} {
return params;
}
+ @Override
+ public Set<String> getQueryParams() {
+ final var queryParams = new HashSet<String>();
+ {{#queryParams}}
+ queryParams.add("{{baseName}}");
+ {{/queryParams}}
+ return queryParams;
+ }
+
@Override
protected {{operationIdCamelCase}}Response
createResponse(SolrClient client) {
return new {{operationIdCamelCase}}Response();
