[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331947711
##
File path: solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
##
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.api.Command;
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.cloud.CloudUtil;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.CryptoKeys;
+import org.apache.solr.util.SimplePostTool;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.server.ByteBufferInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
+
+
+public class PackageStoreAPI {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String PACKAGESTORE_DIRECTORY = "filestore";
+
+
+ private final CoreContainer coreContainer;
+ PackageStore packageStore;
+ public final FSRead readAPI = new FSRead();
+ public final FSWrite writeAPI = new FSWrite();
+
+ public PackageStoreAPI(CoreContainer coreContainer) {
+this.coreContainer = coreContainer;
+packageStore = new DistribPackageStore(coreContainer);
+ }
+
+ public PackageStore getPackageStore() {
+return packageStore;
+ }
+
+ @EndPoint(
+ path = "/cluster/files/*",
+ method = SolrRequest.METHOD.POST,
+ permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public class FSWrite {
+
+static final String TMP_ZK_NODE = "/packageStoreWriteInProgress";
+
+@Command
+public void upload(SolrQueryRequest req, SolrQueryResponse rsp) {
+ try {
+coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE,
"true".getBytes(UTF_8),
+CreateMode.EPHEMERAL, true);
+
+Iterable streams = req.getContentStreams();
+if (streams == null) throw new
SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload");
+String path = req.getPathTemplateValues().get("*");
+if (path == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No
path");
+}
+validateName(path);
+ContentStream stream = streams.iterator().next();
+try {
+ ByteBuffer buf =
SimplePostTool.inputStreamToByteArray(stream.getStream());
+ String sha512 = DigestUtils.sha512Hex(new
ByteBufferInputStream(buf));
+ List signatures = readSignatures(req, buf);
+ Map vals = new HashMap<>();
+ vals.put(MetaData.SHA512, sha512);
+ if (signatures != null) {
+vals.put("sig", signatures);
+ }
+ packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals),
path));
+ rsp.add(CommonParams.FILE, path);
+} catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+}
+ } catch (InterruptedException e) {
+log.error("Unexpected error", e);
+ } c
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331950042
##
File path:
solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
##
@@ -111,6 +127,12 @@ public void writeMap(EntryWriter ew) throws IOException {
ew.putIfNotNull("command", payload);
}
+ @Override
+ public ResponseParser getResponseParser() {
+if(parser != null) return parser;
Review comment:
Please add space after if and before (.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331949944
##
File path:
solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
##
@@ -75,6 +81,15 @@ public SolrParams getParams() {
return new RequestWriter.ContentWriter() {
@Override
public void write(OutputStream os) throws IOException {
+if(payload instanceof ByteBuffer){
+ ByteBuffer b = (ByteBuffer) payload;
+ os.write(b.array(), b.arrayOffset(), b.limit());
+ return;
+}
+if(payload instanceof InputStream){
Review comment:
Please add space after if and before (.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331950280
##
File path: solr/core/src/java/org/apache/solr/filestore/PackageStore.java
##
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.Date;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.filestore.PackageStoreAPI.MetaData;
+import org.apache.zookeeper.server.ByteBufferInputStream;
+
+/** The interface to be implemented by any package store provider
+ * * @lucene.experimental
+ */
+public interface PackageStore {
+
+ /**
+ * Store a file into the filestore. This should ensure that it is replicated
+ * across all nodes in the cluster
+ */
+ void put(FileEntry fileEntry) throws IOException;
+
+ /**
+ * read file content from a given path
+ */
+ void get(String path, Consumer filecontent) throws IOException;
+
+ /**
+ * Fetch a resource from another node
+ * internal
+ *
+ */
+ boolean fetch(String path, String from);
+
+ List list(String path, Predicate predicate);
+
+ /**
+ * get the real path on filesystem
+ */
+ Path getRealpath(String path);
+
+ /**
+ * The type of the resource
+ */
+ FileType getType(String path);
+
+ public class FileEntry {
+final ByteBuffer buf;
+final MetaData meta;
+final String path;
+
+FileEntry(ByteBuffer buf, MetaData meta, String path) {
+ this.buf = buf;
+ this.meta = meta;
+ this.path = path;
+}
+
+public String getPath(){
+ return path;
+}
+
+
+
+public InputStream getInputStream() {
+ if(buf != null) return new ByteBufferInputStream(buf);
Review comment:
Please add space after if and before (.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331949585
##
File path: solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
##
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.api.Command;
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.cloud.CloudUtil;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.CryptoKeys;
+import org.apache.solr.util.SimplePostTool;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.server.ByteBufferInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
+
+
+public class PackageStoreAPI {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String PACKAGESTORE_DIRECTORY = "filestore";
+
+
+ private final CoreContainer coreContainer;
+ PackageStore packageStore;
+ public final FSRead readAPI = new FSRead();
+ public final FSWrite writeAPI = new FSWrite();
+
+ public PackageStoreAPI(CoreContainer coreContainer) {
+this.coreContainer = coreContainer;
+packageStore = new DistribPackageStore(coreContainer);
+ }
+
+ public PackageStore getPackageStore() {
+return packageStore;
+ }
+
+ @EndPoint(
+ path = "/cluster/files/*",
+ method = SolrRequest.METHOD.POST,
+ permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public class FSWrite {
+
+static final String TMP_ZK_NODE = "/packageStoreWriteInProgress";
+
+@Command
+public void upload(SolrQueryRequest req, SolrQueryResponse rsp) {
+ try {
+coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE,
"true".getBytes(UTF_8),
+CreateMode.EPHEMERAL, true);
+
+Iterable streams = req.getContentStreams();
+if (streams == null) throw new
SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload");
+String path = req.getPathTemplateValues().get("*");
+if (path == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No
path");
+}
+validateName(path);
+ContentStream stream = streams.iterator().next();
+try {
+ ByteBuffer buf =
SimplePostTool.inputStreamToByteArray(stream.getStream());
+ String sha512 = DigestUtils.sha512Hex(new
ByteBufferInputStream(buf));
+ List signatures = readSignatures(req, buf);
+ Map vals = new HashMap<>();
+ vals.put(MetaData.SHA512, sha512);
+ if (signatures != null) {
+vals.put("sig", signatures);
+ }
+ packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals),
path));
+ rsp.add(CommonParams.FILE, path);
+} catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+}
+ } catch (InterruptedException e) {
+log.error("Unexpected error", e);
+ } c
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331950108
##
File path: solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
##
@@ -40,6 +40,15 @@
}
+ static final String DELIM_CHARS = "/:;.,%#";
+ public static List split(String s, char sep){
+if(DELIM_CHARS.indexOf(s.charAt(0)) >-1){
Review comment:
Please add space after if and before (.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331950188
##
File path: solr/solrj/src/java/org/apache/solr/common/util/Utils.java
##
@@ -713,4 +717,69 @@ public static String getMDCNode() {
return def;
}
+ public interface InputStreamConsumer {
+
+T accept(InputStream is) throws IOException;
+
+ }
+
+ public static final InputStreamConsumer JAVABINCONSUMER = is -> new
JavaBinCodec().unmarshal(is);
+ public static final InputStreamConsumer JSONCONSUMER = is ->
Utils.fromJSON(is);
+ public static InputStreamConsumer newBytesConsumer(int maxSize){
+return is -> {
+ try (BinaryRequestWriter.BAOS bos = new BinaryRequestWriter.BAOS()) {
+long sz = 0;
+int next = is.read();
+while (next > -1) {
+ if (++sz > maxSize) throw new BufferOverflowException();
+ bos.write(next);
+ next = is.read();
+}
+bos.flush();
+return ByteBuffer.wrap( bos.getbuf(), 0, bos.size());
+ } catch (IOException e) {
+throw new RuntimeException(e);
+ }
+};
+
+ }
+
+
+
+
+ public static T executeGET(HttpClient client, String url,
InputStreamConsumer consumer) throws SolrException {
+T result = null;
+HttpGet httpGet = new HttpGet(url);
+HttpResponse rsp = null;
+try {
+ rsp = client.execute(httpGet);
+} catch (IOException e) {
+ log.error("Error in request to url : "+ url, e);
+ throw new SolrException(SolrException.ErrorCode.UNKNOWN, "error sending
request");
+}
+int statusCode = rsp.getStatusLine().getStatusCode();
+if(statusCode != 200) {
Review comment:
Please add space after if and before (.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331948124
##
File path: solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
##
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.api.Command;
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.cloud.CloudUtil;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.CryptoKeys;
+import org.apache.solr.util.SimplePostTool;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.server.ByteBufferInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
+
+
+public class PackageStoreAPI {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String PACKAGESTORE_DIRECTORY = "filestore";
+
+
+ private final CoreContainer coreContainer;
+ PackageStore packageStore;
+ public final FSRead readAPI = new FSRead();
+ public final FSWrite writeAPI = new FSWrite();
+
+ public PackageStoreAPI(CoreContainer coreContainer) {
+this.coreContainer = coreContainer;
+packageStore = new DistribPackageStore(coreContainer);
+ }
+
+ public PackageStore getPackageStore() {
+return packageStore;
+ }
+
+ @EndPoint(
+ path = "/cluster/files/*",
+ method = SolrRequest.METHOD.POST,
+ permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public class FSWrite {
+
+static final String TMP_ZK_NODE = "/packageStoreWriteInProgress";
+
+@Command
+public void upload(SolrQueryRequest req, SolrQueryResponse rsp) {
+ try {
+coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE,
"true".getBytes(UTF_8),
+CreateMode.EPHEMERAL, true);
+
+Iterable streams = req.getContentStreams();
+if (streams == null) throw new
SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload");
+String path = req.getPathTemplateValues().get("*");
+if (path == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No
path");
+}
+validateName(path);
+ContentStream stream = streams.iterator().next();
+try {
+ ByteBuffer buf =
SimplePostTool.inputStreamToByteArray(stream.getStream());
+ String sha512 = DigestUtils.sha512Hex(new
ByteBufferInputStream(buf));
+ List signatures = readSignatures(req, buf);
+ Map vals = new HashMap<>();
+ vals.put(MetaData.SHA512, sha512);
+ if (signatures != null) {
+vals.put("sig", signatures);
+ }
+ packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals),
path));
+ rsp.add(CommonParams.FILE, path);
+} catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+}
+ } catch (InterruptedException e) {
+log.error("Unexpected error", e);
+ } c
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331949996
##
File path:
solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
##
@@ -84,6 +99,7 @@ public void write(OutputStream os) throws IOException {
@Override
public String getContentType() {
+if(mimeType != null) return mimeType;
Review comment:
Please add space after if and before (.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331949884
##
File path:
solr/solrj/src/java/org/apache/solr/client/solrj/request/V2Request.java
##
@@ -75,6 +81,15 @@ public SolrParams getParams() {
return new RequestWriter.ContentWriter() {
@Override
public void write(OutputStream os) throws IOException {
+if(payload instanceof ByteBuffer){
Review comment:
Please add space after if and before (.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331950231
##
File path: solr/solrj/src/java/org/apache/solr/common/util/Utils.java
##
@@ -713,4 +717,69 @@ public static String getMDCNode() {
return def;
}
+ public interface InputStreamConsumer {
+
+T accept(InputStream is) throws IOException;
+
+ }
+
+ public static final InputStreamConsumer JAVABINCONSUMER = is -> new
JavaBinCodec().unmarshal(is);
+ public static final InputStreamConsumer JSONCONSUMER = is ->
Utils.fromJSON(is);
+ public static InputStreamConsumer newBytesConsumer(int maxSize){
+return is -> {
+ try (BinaryRequestWriter.BAOS bos = new BinaryRequestWriter.BAOS()) {
+long sz = 0;
+int next = is.read();
+while (next > -1) {
+ if (++sz > maxSize) throw new BufferOverflowException();
+ bos.write(next);
+ next = is.read();
+}
+bos.flush();
+return ByteBuffer.wrap( bos.getbuf(), 0, bos.size());
+ } catch (IOException e) {
+throw new RuntimeException(e);
+ }
+};
+
+ }
+
+
+
+
+ public static T executeGET(HttpClient client, String url,
InputStreamConsumer consumer) throws SolrException {
+T result = null;
+HttpGet httpGet = new HttpGet(url);
+HttpResponse rsp = null;
+try {
+ rsp = client.execute(httpGet);
+} catch (IOException e) {
+ log.error("Error in request to url : "+ url, e);
+ throw new SolrException(SolrException.ErrorCode.UNKNOWN, "error sending
request");
+}
+int statusCode = rsp.getStatusLine().getStatusCode();
+if(statusCode != 200) {
+ try {
+log.error("Failed a request to : {} , status :{} body {}",url,
rsp.getStatusLine(), EntityUtils.toString(rsp.getEntity(),
StandardCharsets.UTF_8));
+ } catch (IOException e) {
+log.error("could not print error", e);
+ }
+ throw new
SolrException(SolrException.ErrorCode.getErrorCode(statusCode), "Unknown
error");
+}
+HttpEntity entity = rsp.getEntity();
+try{
+ InputStream is = entity.getContent();
+ if(consumer != null) {
Review comment:
Please add space after if and before (.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331949129
##
File path: solr/solrj/src/java/org/apache/solr/common/util/Utils.java
##
@@ -713,4 +717,69 @@ public static String getMDCNode() {
return def;
}
+ public interface InputStreamConsumer {
+
+T accept(InputStream is) throws IOException;
+
+ }
+
+ public static final InputStreamConsumer JAVABINCONSUMER = is -> new
JavaBinCodec().unmarshal(is);
+ public static final InputStreamConsumer JSONCONSUMER = is ->
Utils.fromJSON(is);
+ public static InputStreamConsumer newBytesConsumer(int maxSize){
+return is -> {
+ try (BinaryRequestWriter.BAOS bos = new BinaryRequestWriter.BAOS()) {
+long sz = 0;
+int next = is.read();
+while (next > -1) {
+ if (++sz > maxSize) throw new BufferOverflowException();
+ bos.write(next);
+ next = is.read();
+}
+bos.flush();
+return ByteBuffer.wrap( bos.getbuf(), 0, bos.size());
+ } catch (IOException e) {
+throw new RuntimeException(e);
+ }
+};
+
+ }
+
+
+
+
+ public static T executeGET(HttpClient client, String url,
InputStreamConsumer consumer) throws SolrException {
+T result = null;
+HttpGet httpGet = new HttpGet(url);
+HttpResponse rsp = null;
+try {
+ rsp = client.execute(httpGet);
+} catch (IOException e) {
+ log.error("Error in request to url : "+ url, e);
+ throw new SolrException(SolrException.ErrorCode.UNKNOWN, "error sending
request");
+}
+int statusCode = rsp.getStatusLine().getStatusCode();
+if(statusCode != 200) {
+ try {
+log.error("Failed a request to : {} , status :{} body {}",url,
rsp.getStatusLine(), EntityUtils.toString(rsp.getEntity(),
StandardCharsets.UTF_8));
Review comment:
Please remove space before colon, there should always be a space after colon.
This:
"Failed a request to : {} , status :{} body {}"
Should be:
"Failed a request to: {}, status: {}, body: {}"
Also, please put a space after every comma (before url).
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331949753
##
File path: solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
##
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.api.Command;
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.cloud.CloudUtil;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.CryptoKeys;
+import org.apache.solr.util.SimplePostTool;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.server.ByteBufferInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
+
+
+public class PackageStoreAPI {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String PACKAGESTORE_DIRECTORY = "filestore";
+
+
+ private final CoreContainer coreContainer;
+ PackageStore packageStore;
+ public final FSRead readAPI = new FSRead();
+ public final FSWrite writeAPI = new FSWrite();
+
+ public PackageStoreAPI(CoreContainer coreContainer) {
+this.coreContainer = coreContainer;
+packageStore = new DistribPackageStore(coreContainer);
+ }
+
+ public PackageStore getPackageStore() {
+return packageStore;
+ }
+
+ @EndPoint(
+ path = "/cluster/files/*",
+ method = SolrRequest.METHOD.POST,
+ permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public class FSWrite {
+
+static final String TMP_ZK_NODE = "/packageStoreWriteInProgress";
+
+@Command
+public void upload(SolrQueryRequest req, SolrQueryResponse rsp) {
+ try {
+coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE,
"true".getBytes(UTF_8),
+CreateMode.EPHEMERAL, true);
+
+Iterable streams = req.getContentStreams();
+if (streams == null) throw new
SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload");
+String path = req.getPathTemplateValues().get("*");
+if (path == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No
path");
+}
+validateName(path);
+ContentStream stream = streams.iterator().next();
+try {
+ ByteBuffer buf =
SimplePostTool.inputStreamToByteArray(stream.getStream());
+ String sha512 = DigestUtils.sha512Hex(new
ByteBufferInputStream(buf));
+ List signatures = readSignatures(req, buf);
+ Map vals = new HashMap<>();
+ vals.put(MetaData.SHA512, sha512);
+ if (signatures != null) {
+vals.put("sig", signatures);
+ }
+ packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals),
path));
+ rsp.add(CommonParams.FILE, path);
+} catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+}
+ } catch (InterruptedException e) {
+log.error("Unexpected error", e);
+ } c
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331949489
##
File path: solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java
##
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.solr.api.Command;
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.cloud.CloudUtil;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.CryptoKeys;
+import org.apache.solr.util.SimplePostTool;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.server.ByteBufferInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
+
+
+public class PackageStoreAPI {
+ private static final Logger log =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String PACKAGESTORE_DIRECTORY = "filestore";
+
+
+ private final CoreContainer coreContainer;
+ PackageStore packageStore;
+ public final FSRead readAPI = new FSRead();
+ public final FSWrite writeAPI = new FSWrite();
+
+ public PackageStoreAPI(CoreContainer coreContainer) {
+this.coreContainer = coreContainer;
+packageStore = new DistribPackageStore(coreContainer);
+ }
+
+ public PackageStore getPackageStore() {
+return packageStore;
+ }
+
+ @EndPoint(
+ path = "/cluster/files/*",
+ method = SolrRequest.METHOD.POST,
+ permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public class FSWrite {
+
+static final String TMP_ZK_NODE = "/packageStoreWriteInProgress";
+
+@Command
+public void upload(SolrQueryRequest req, SolrQueryResponse rsp) {
+ try {
+coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE,
"true".getBytes(UTF_8),
+CreateMode.EPHEMERAL, true);
+
+Iterable streams = req.getContentStreams();
+if (streams == null) throw new
SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload");
+String path = req.getPathTemplateValues().get("*");
+if (path == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No
path");
+}
+validateName(path);
+ContentStream stream = streams.iterator().next();
+try {
+ ByteBuffer buf =
SimplePostTool.inputStreamToByteArray(stream.getStream());
+ String sha512 = DigestUtils.sha512Hex(new
ByteBufferInputStream(buf));
+ List signatures = readSignatures(req, buf);
+ Map vals = new HashMap<>();
+ vals.put(MetaData.SHA512, sha512);
+ if (signatures != null) {
+vals.put("sig", signatures);
+ }
+ packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals),
path));
+ rsp.add(CommonParams.FILE, path);
+} catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+}
+ } catch (InterruptedException e) {
+log.error("Unexpected error", e);
+ } c
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331844068
##
File path: solr/core/src/java/org/apache/solr/filestore/FileStore.java
##
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.filestore.FileStoreAPI.MetaData;
+
+
+public interface FileStore {
+
+ /**Store a file into the filestore. This should ensure that it is replicated
+ * across all nodes in the cluster
+ */
+ void put(String path , MetaData metadata, ByteBuffer filecontent) throws
IOException;
Review comment:
In that case, the interface could account for both usecases (in memory and
streamed) if it just used OutputStream instead of ByteBuffer.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331842651
##
File path: solr/core/src/java/org/apache/solr/filestore/FileStore.java
##
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.filestore.FileStoreAPI.MetaData;
+
+
+public interface FileStore {
+
+ /**Store a file into the filestore. This should ensure that it is replicated
+ * across all nodes in the cluster
+ */
+ void put(String path , MetaData metadata, ByteBuffer filecontent) throws
IOException;
+
+ /** read file content from a given path
+ */
+ void get(String path, BiConsumer filecontent) throws
IOException;
+
+ /**Fetch a resource from another node
+ * internal
+ */
+ void fetch(String path, String from);
+
+ List list(String path, Predicate predicate);
Review comment:
We need Javadoc for what these parameters should look like.
The MapWriter seems very generic; but our files have defined structure to
the metadata that itself can be returned.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331841750
##
File path: solr/core/src/java/org/apache/solr/filestore/FileStore.java
##
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.filestore.FileStoreAPI.MetaData;
+
+
+public interface FileStore {
+
+ /**Store a file into the filestore. This should ensure that it is replicated
+ * across all nodes in the cluster
+ */
+ void put(String path , MetaData metadata, ByteBuffer filecontent) throws
IOException;
Review comment:
This contract for putting a file assumes the entire file can fit in memory.
However, for very large files, it might be more preferable to stream the files
in. Even for a package store, this just means larger memory footprint. Can we
use an OutputStream instead?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331842282
##
File path: solr/core/src/java/org/apache/solr/filestore/FileStore.java
##
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.filestore.FileStoreAPI.MetaData;
+
+
+public interface FileStore {
+
+ /**Store a file into the filestore. This should ensure that it is replicated
+ * across all nodes in the cluster
+ */
+ void put(String path , MetaData metadata, ByteBuffer filecontent) throws
IOException;
+
+ /** read file content from a given path
+ */
+ void get(String path, BiConsumer filecontent) throws
IOException;
Review comment:
"filecontent" here refers to both content and metadata. However, for the
put(), it only means the contents.
Instead of dealing with content and metadata separately, can we instead
define an abstraction class that encapsulate both the File and the metadata?
E.g.
class FileStoreFile { ByteBuffer buf, Metadata meta }
and similarly, FileStoreInputStream and FileStoreOutputStream etc.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331840722
##
File path: solr/core/src/java/org/apache/solr/filestore/FileStore.java
##
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.filestore.FileStoreAPI.MetaData;
+
+
+public interface FileStore {
+
+ /**Store a file into the filestore. This should ensure that it is replicated
Review comment:
I think specifying "replicated across all nodes in the cluster" shouldn't be
necessary for all potential implementations (e.g. NFS etc.).
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331840003
##
File path: solr/core/src/java/org/apache/solr/filestore/FileStore.java
##
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.filestore.FileStoreAPI.MetaData;
+
+
+public interface FileStore {
+
+ /**Store a file into the filestore. This should ensure that it is replicated
+ * across all nodes in the cluster
+ */
+ void put(String path , MetaData metadata, ByteBuffer filecontent) throws
IOException;
+
+ /** read file content from a given path
+ */
+ void get(String path, BiConsumer filecontent) throws
IOException;
+
+ /**Fetch a resource from another node
+ * internal
+ */
+ void fetch(String path, String from);
+
+ List list(String path, Predicate predicate);
+
+ /**get the real path on filesystem
+ */
+ Path getRealpath(String path);
Review comment:
Can we call it absolute path?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
[GitHub] [lucene-solr] chatman commented on a change in pull request #929: SOLR-13821: Package Store for storing package artefacts
chatman commented on a change in pull request #929: SOLR-13821: Package Store
for storing package artefacts
URL: https://github.com/apache/lucene-solr/pull/929#discussion_r331840227
##
File path: solr/core/src/java/org/apache/solr/filestore/FileStore.java
##
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.filestore;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.function.BiConsumer;
+import java.util.function.Predicate;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.filestore.FileStoreAPI.MetaData;
+
+
+public interface FileStore {
+
+ /**Store a file into the filestore. This should ensure that it is replicated
+ * across all nodes in the cluster
+ */
+ void put(String path , MetaData metadata, ByteBuffer filecontent) throws
IOException;
Review comment:
path and comma shouldn't have a space in between.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
-
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
