http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java deleted file mode 100644 index 5e65021..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java +++ /dev/null @@ -1,492 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.impl; - -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentProducer; -import org.apache.http.entity.EntityTemplate; -import org.apache.solr.client.solrj.ResponseParser; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.RequestWriter; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.params.UpdateParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SolrjNamedThreadFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.LinkedList; -import java.util.Locale; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * ConcurrentUpdateSolrClient buffers all added documents and writes - * them into open HTTP connections. This class is thread safe. - * - * Params from {@link UpdateRequest} are converted to http request - * parameters. When params change between UpdateRequests a new HTTP - * request is started. - * - * Although any SolrClient request can be made with this implementation, it is - * only recommended to use ConcurrentUpdateSolrClient with /update - * requests. The class {@link HttpSolrClient} is better suited for the - * query interface. - */ -public class ConcurrentUpdateSolrClient extends SolrClient { - private static final long serialVersionUID = 1L; - static final Logger log = LoggerFactory - .getLogger(ConcurrentUpdateSolrClient.class); - private HttpSolrClient client; - final BlockingQueue<UpdateRequest> queue; - final ExecutorService scheduler; - final Queue<Runner> runners; - volatile CountDownLatch lock = null; // used to block everything - final int threadCount; - boolean shutdownExecutor = false; - int pollQueueTime = 250; - private final boolean streamDeletes; - - /** - * Uses an internally managed HttpClient instance. - * - * @param solrServerUrl - * The Solr server URL - * @param queueSize - * The buffer size before the documents are sent to the server - * @param threadCount - * The number of background threads used to empty the queue - */ - public ConcurrentUpdateSolrClient(String solrServerUrl, int queueSize, - int threadCount) { - this(solrServerUrl, null, queueSize, threadCount); - shutdownExecutor = true; - } - - public ConcurrentUpdateSolrClient(String solrServerUrl, - HttpClient client, int queueSize, int threadCount) { - this(solrServerUrl, client, queueSize, threadCount, Executors.newCachedThreadPool( - new SolrjNamedThreadFactory("concurrentUpdateScheduler"))); - shutdownExecutor = true; - } - - /** - * Uses the supplied HttpClient to send documents to the Solr server. - */ - public ConcurrentUpdateSolrClient(String solrServerUrl, - HttpClient client, int queueSize, int threadCount, ExecutorService es) { - this(solrServerUrl, client, queueSize, threadCount, es, false); - } - - /** - * Uses the supplied HttpClient to send documents to the Solr server. - */ - public ConcurrentUpdateSolrClient(String solrServerUrl, - HttpClient client, int queueSize, int threadCount, ExecutorService es, boolean streamDeletes) { - this.client = new HttpSolrClient(solrServerUrl, client); - this.client.setFollowRedirects(false); - queue = new LinkedBlockingQueue<>(queueSize); - this.threadCount = threadCount; - runners = new LinkedList<>(); - scheduler = es; - this.streamDeletes = streamDeletes; - } - - public Set<String> getQueryParams() { - return this.client.getQueryParams(); - } - - /** - * Expert Method. - * @param queryParams set of param keys to only send via the query string - */ - public void setQueryParams(Set<String> queryParams) { - this.client.setQueryParams(queryParams); - } - - /** - * Opens a connection and sends everything... - */ - class Runner implements Runnable { - final Lock runnerLock = new ReentrantLock(); - - @Override - public void run() { - runnerLock.lock(); - - log.debug("starting runner: {}", this); - HttpPost method = null; - HttpResponse response = null; - try { - while (!queue.isEmpty()) { - try { - final UpdateRequest updateRequest = - queue.poll(pollQueueTime, TimeUnit.MILLISECONDS); - if (updateRequest == null) - break; - - String contentType = client.requestWriter.getUpdateContentType(); - final boolean isXml = ClientUtils.TEXT_XML.equals(contentType); - - final ModifiableSolrParams origParams = new ModifiableSolrParams(updateRequest.getParams()); - - EntityTemplate template = new EntityTemplate(new ContentProducer() { - - @Override - public void writeTo(OutputStream out) throws IOException { - try { - if (isXml) { - out.write("<stream>".getBytes(StandardCharsets.UTF_8)); // can be anything - } - UpdateRequest req = updateRequest; - while (req != null) { - SolrParams currentParams = new ModifiableSolrParams(req.getParams()); - if (!origParams.toNamedList().equals(currentParams.toNamedList())) { - queue.add(req); // params are different, push back to queue - break; - } - - client.requestWriter.write(req, out); - if (isXml) { - // check for commit or optimize - SolrParams params = req.getParams(); - if (params != null) { - String fmt = null; - if (params.getBool(UpdateParams.OPTIMIZE, false)) { - fmt = "<optimize waitSearcher=\"%s\" />"; - } else if (params.getBool(UpdateParams.COMMIT, false)) { - fmt = "<commit waitSearcher=\"%s\" />"; - } - if (fmt != null) { - byte[] content = String.format(Locale.ROOT, - fmt, - params.getBool(UpdateParams.WAIT_SEARCHER, false) - + "").getBytes(StandardCharsets.UTF_8); - out.write(content); - } - } - } - out.flush(); - req = queue.poll(pollQueueTime, TimeUnit.MILLISECONDS); - } - - if (isXml) { - out.write("</stream>".getBytes(StandardCharsets.UTF_8)); - } - - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.warn("", e); - } - } - }); - - // The parser 'wt=' and 'version=' params are used instead of the - // original params - ModifiableSolrParams requestParams = new ModifiableSolrParams(origParams); - requestParams.set(CommonParams.WT, client.parser.getWriterType()); - requestParams.set(CommonParams.VERSION, client.parser.getVersion()); - - method = new HttpPost(client.getBaseURL() + "/update" - + ClientUtils.toQueryString(requestParams, false)); - method.setEntity(template); - method.addHeader("User-Agent", HttpSolrClient.AGENT); - method.addHeader("Content-Type", contentType); - - response = client.getHttpClient().execute(method); - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode != HttpStatus.SC_OK) { - StringBuilder msg = new StringBuilder(); - msg.append(response.getStatusLine().getReasonPhrase()); - msg.append("\n\n\n\n"); - msg.append("request: ").append(method.getURI()); - - SolrException solrExc = new SolrException(ErrorCode.getErrorCode(statusCode), msg.toString()); - // parse out the metadata from the SolrException - try { - NamedList<Object> resp = - client.parser.processResponse(response.getEntity().getContent(), - response.getEntity().getContentType().getValue()); - NamedList<Object> error = (NamedList<Object>) resp.get("error"); - if (error != null) - solrExc.setMetadata((NamedList<String>) error.get("metadata")); - } catch (Exception exc) { - // don't want to fail to report error if parsing the response fails - log.warn("Failed to parse error response from "+ client.getBaseURL()+" due to: "+exc); - } - - handleError(solrExc); - } else { - onSuccess(response); - } - } finally { - try { - if (response != null) { - response.getEntity().getContent().close(); - } - } catch (Exception ex) { - log.warn("", ex); - } - } - } - } catch (Throwable e) { - if (e instanceof OutOfMemoryError) { - throw (OutOfMemoryError) e; - } - handleError(e); - } finally { - synchronized (runners) { - if (runners.size() == 1 && !queue.isEmpty()) { - // keep this runner alive - scheduler.execute(this); - } else { - runners.remove(this); - if (runners.isEmpty()) - runners.notifyAll(); - } - } - - log.debug("finished: {}", this); - runnerLock.unlock(); - } - } - } - - @Override - public NamedList<Object> request(final SolrRequest request) - throws SolrServerException, IOException { - if (!(request instanceof UpdateRequest)) { - return client.request(request); - } - UpdateRequest req = (UpdateRequest) request; - - // this happens for commit... - if (streamDeletes) { - if ((req.getDocuments() == null || req.getDocuments().isEmpty()) - && (req.getDeleteById() == null || req.getDeleteById().isEmpty()) - && (req.getDeleteByIdMap() == null || req.getDeleteByIdMap().isEmpty())) { - if (req.getDeleteQuery() == null) { - blockUntilFinished(); - return client.request(request); - } - } - } else { - if ((req.getDocuments() == null || req.getDocuments().isEmpty())) { - blockUntilFinished(); - return client.request(request); - } - } - - - SolrParams params = req.getParams(); - if (params != null) { - // check if it is waiting for the searcher - if (params.getBool(UpdateParams.WAIT_SEARCHER, false)) { - log.info("blocking for commit/optimize"); - blockUntilFinished(); // empty the queue - return client.request(request); - } - } - - try { - CountDownLatch tmpLock = lock; - if (tmpLock != null) { - tmpLock.await(); - } - - boolean success = queue.offer(req); - - for (;;) { - synchronized (runners) { - // see if queue is half full and we can add more runners - // special case: if only using a threadCount of 1 and the queue - // is filling up, allow 1 add'l runner to help process the queue - if (runners.isEmpty() || (queue.remainingCapacity() < queue.size() && runners.size() < threadCount)) - { - // We need more runners, so start a new one. - Runner r = new Runner(); - runners.add(r); - scheduler.execute(r); - } else { - // break out of the retry loop if we added the element to the queue - // successfully, *and* - // while we are still holding the runners lock to prevent race - // conditions. - if (success) - break; - } - } - - // Retry to add to the queue w/o the runners lock held (else we risk - // temporary deadlock) - // This retry could also fail because - // 1) existing runners were not able to take off any new elements in the - // queue - // 2) the queue was filled back up since our last try - // If we succeed, the queue may have been completely emptied, and all - // runners stopped. - // In all cases, we should loop back to the top to see if we need to - // start more runners. - // - if (!success) { - success = queue.offer(req, 100, TimeUnit.MILLISECONDS); - } - } - } catch (InterruptedException e) { - log.error("interrupted", e); - throw new IOException(e.getLocalizedMessage()); - } - - // RETURN A DUMMY result - NamedList<Object> dummy = new NamedList<>(); - dummy.add("NOTE", "the request is processed in a background stream"); - return dummy; - } - - public synchronized void blockUntilFinished() { - lock = new CountDownLatch(1); - try { - synchronized (runners) { - while (!runners.isEmpty()) { - try { - runners.wait(); - } catch (InterruptedException e) { - Thread.interrupted(); - } - - if (scheduler.isTerminated()) - break; - - // if we reach here, then we probably got the notifyAll, but need to check if - // the queue is empty before really considering this is finished (SOLR-4260) - int queueSize = queue.size(); - if (queueSize > 0) { - log.warn("No more runners, but queue still has "+ - queueSize+" adding more runners to process remaining requests on queue"); - Runner r = new Runner(); - runners.add(r); - scheduler.execute(r); - } - } - } - } finally { - lock.countDown(); - lock = null; - } - } - - public void handleError(Throwable ex) { - log.error("error", ex); - } - - /** - * Intended to be used as an extension point for doing post processing after a request completes. - */ - public void onSuccess(HttpResponse resp) { - // no-op by design, override to add functionality - } - - @Override - public void close() { - shutdown(); - } - - @Override - @Deprecated - public void shutdown() { - client.shutdown(); - if (shutdownExecutor) { - scheduler.shutdown(); - try { - if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) { - scheduler.shutdownNow(); - if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) log - .error("ExecutorService did not terminate"); - } - } catch (InterruptedException ie) { - scheduler.shutdownNow(); - Thread.currentThread().interrupt(); - } - } - } - - public void setConnectionTimeout(int timeout) { - HttpClientUtil.setConnectionTimeout(client.getHttpClient(), timeout); - } - - /** - * set soTimeout (read timeout) on the underlying HttpConnectionManager. This is desirable for queries, but probably - * not for indexing. - */ - public void setSoTimeout(int timeout) { - HttpClientUtil.setSoTimeout(client.getHttpClient(), timeout); - } - - public void shutdownNow() { - client.shutdown(); - if (shutdownExecutor) { - scheduler.shutdownNow(); // Cancel currently executing tasks - try { - if (!scheduler.awaitTermination(30, TimeUnit.SECONDS)) - log.error("ExecutorService did not terminate"); - } catch (InterruptedException ie) { - scheduler.shutdownNow(); - Thread.currentThread().interrupt(); - } - } - } - - public void setParser(ResponseParser responseParser) { - client.setParser(responseParser); - } - - - /** - * @param pollQueueTime time for an open connection to wait for updates when - * the queue is empty. - */ - public void setPollQueueTime(int pollQueueTime) { - this.pollQueueTime = pollQueueTime; - } - - public void setRequestWriter(RequestWriter requestWriter) { - client.setRequestWriter(requestWriter); - } -}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java deleted file mode 100644 index 9ace82a..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.impl; - -import org.apache.http.client.HttpClient; - -import java.util.concurrent.ExecutorService; - -/** - * @deprecated Use {@link org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient} - */ -@Deprecated -public class ConcurrentUpdateSolrServer extends ConcurrentUpdateSolrClient { - - public ConcurrentUpdateSolrServer(String solrServerUrl, int queueSize, int threadCount) { - super(solrServerUrl, queueSize, threadCount); - } - - public ConcurrentUpdateSolrServer(String solrServerUrl, HttpClient client, int queueSize, int threadCount) { - super(solrServerUrl, client, queueSize, threadCount); - } - - public ConcurrentUpdateSolrServer(String solrServerUrl, HttpClient client, int queueSize, int threadCount, ExecutorService es) { - super(solrServerUrl, client, queueSize, threadCount, es); - } - - public ConcurrentUpdateSolrServer(String solrServerUrl, HttpClient client, int queueSize, int threadCount, ExecutorService es, boolean streamDeletes) { - super(solrServerUrl, client, queueSize, threadCount, es, streamDeletes); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java deleted file mode 100644 index 78ed726..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.impl; - -import org.apache.http.conn.ssl.SSLSocketFactory; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.solr.common.params.SolrParams; - -/** - * The default http client configurer. If the behaviour needs to be customized a - * new HttpCilentConfigurer can be set by calling - * {@link HttpClientUtil#setConfigurer(HttpClientConfigurer)} - */ -public class HttpClientConfigurer { - - public void configure(DefaultHttpClient httpClient, SolrParams config) { - - if (config.get(HttpClientUtil.PROP_MAX_CONNECTIONS) != null) { - HttpClientUtil.setMaxConnections(httpClient, - config.getInt(HttpClientUtil.PROP_MAX_CONNECTIONS)); - } - - if (config.get(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST) != null) { - HttpClientUtil.setMaxConnectionsPerHost(httpClient, - config.getInt(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST)); - } - - if (config.get(HttpClientUtil.PROP_CONNECTION_TIMEOUT) != null) { - HttpClientUtil.setConnectionTimeout(httpClient, - config.getInt(HttpClientUtil.PROP_CONNECTION_TIMEOUT)); - } - - if (config.get(HttpClientUtil.PROP_SO_TIMEOUT) != null) { - HttpClientUtil.setSoTimeout(httpClient, - config.getInt(HttpClientUtil.PROP_SO_TIMEOUT)); - } - - if (config.get(HttpClientUtil.PROP_FOLLOW_REDIRECTS) != null) { - HttpClientUtil.setFollowRedirects(httpClient, - config.getBool(HttpClientUtil.PROP_FOLLOW_REDIRECTS)); - } - - // always call setUseRetry, whether it is in config or not - HttpClientUtil.setUseRetry(httpClient, - config.getBool(HttpClientUtil.PROP_USE_RETRY, true)); - - final String basicAuthUser = config - .get(HttpClientUtil.PROP_BASIC_AUTH_USER); - final String basicAuthPass = config - .get(HttpClientUtil.PROP_BASIC_AUTH_PASS); - HttpClientUtil.setBasicAuth(httpClient, basicAuthUser, basicAuthPass); - - if (config.get(HttpClientUtil.PROP_ALLOW_COMPRESSION) != null) { - HttpClientUtil.setAllowCompression(httpClient, - config.getBool(HttpClientUtil.PROP_ALLOW_COMPRESSION)); - } - - boolean sslCheckPeerName = toBooleanDefaultIfNull( - toBooleanObject(System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME)), true); - if(sslCheckPeerName == false) { - HttpClientUtil.setHostNameVerifier(httpClient, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); - } - } - - public static boolean toBooleanDefaultIfNull(Boolean bool, boolean valueIfNull) { - if (bool == null) { - return valueIfNull; - } - return bool.booleanValue() ? true : false; - } - - public static Boolean toBooleanObject(String str) { - if ("true".equalsIgnoreCase(str)) { - return Boolean.TRUE; - } else if ("false".equalsIgnoreCase(str)) { - return Boolean.FALSE; - } - // no match - return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java deleted file mode 100644 index 96ed211..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java +++ /dev/null @@ -1,365 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.impl; - -import java.io.IOException; -import java.io.InputStream; -import java.util.zip.GZIPInputStream; -import java.util.zip.InflaterInputStream; - -import org.apache.http.Header; -import org.apache.http.HeaderElement; -import org.apache.http.HttpEntity; -import org.apache.http.HttpException; -import org.apache.http.HttpRequest; -import org.apache.http.HttpRequestInterceptor; -import org.apache.http.HttpResponse; -import org.apache.http.HttpResponseInterceptor; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.HttpClient; -import org.apache.http.client.params.ClientParamBean; -import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.ssl.SSLSocketFactory; -import org.apache.http.conn.ssl.X509HostnameVerifier; -import org.apache.http.entity.HttpEntityWrapper; -//import org.apache.http.impl.client.CloseableHttpClient; //RANGER_UPDATE - to use SystemDefaultHttpClient -import org.apache.http.impl.client.AbstractHttpClient; //RANGER_UPDATE -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; -import org.apache.http.impl.client.SystemDefaultHttpClient; -import org.apache.http.impl.conn.PoolingClientConnectionManager; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; // jdoc -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.protocol.HttpContext; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utility class for creating/configuring httpclient instances. - */ -public class HttpClientUtil { - // socket timeout measured in ms, closes a socket if read - // takes longer than x ms to complete. throws - // java.net.SocketTimeoutException: Read timed out exception - public static final String PROP_SO_TIMEOUT = "socketTimeout"; - // connection timeout measures in ms, closes a socket if connection - // cannot be established within x ms. with a - // java.net.SocketTimeoutException: Connection timed out - public static final String PROP_CONNECTION_TIMEOUT = "connTimeout"; - // Maximum connections allowed per host - public static final String PROP_MAX_CONNECTIONS_PER_HOST = "maxConnectionsPerHost"; - // Maximum total connections allowed - public static final String PROP_MAX_CONNECTIONS = "maxConnections"; - // Retry http requests on error - public static final String PROP_USE_RETRY = "retry"; - // Allow compression (deflate,gzip) if server supports it - public static final String PROP_ALLOW_COMPRESSION = "allowCompression"; - // Follow redirects - public static final String PROP_FOLLOW_REDIRECTS = "followRedirects"; - // Basic auth username - public static final String PROP_BASIC_AUTH_USER = "httpBasicAuthUser"; - // Basic auth password - public static final String PROP_BASIC_AUTH_PASS = "httpBasicAuthPassword"; - - public static final String SYS_PROP_CHECK_PEER_NAME = "solr.ssl.checkPeerName"; - - private static final Logger logger = LoggerFactory - .getLogger(HttpClientUtil.class); - - static final DefaultHttpRequestRetryHandler NO_RETRY = new DefaultHttpRequestRetryHandler( - 0, false); - - private static HttpClientConfigurer configurer = new HttpClientConfigurer(); - - private HttpClientUtil(){} - - /** - * Replace the {@link HttpClientConfigurer} class used in configuring the http - * clients with a custom implementation. - */ - public static void setConfigurer(HttpClientConfigurer newConfigurer) { - configurer = newConfigurer; - } - - public static HttpClientConfigurer getConfigurer() { - return configurer; - } - - /** - * Creates new http client by using the provided configuration. - * - * @param params - * http client configuration, if null a client with default - * configuration (no additional configuration) is created. - */ - //public static CloseableHttpClient createClient(final SolrParams params) { //RANGER_UPDATE - public static AbstractHttpClient createClient(final SolrParams params) { - final ModifiableSolrParams config = new ModifiableSolrParams(params); - if (logger.isDebugEnabled()) { - logger.debug("Creating new http client, config:" + config); - } - final DefaultHttpClient httpClient = new SystemDefaultHttpClient(); - configureClient(httpClient, config); - return httpClient; - } - - /** - * Creates new http client by using the provided configuration. - * - */ - //public static CloseableHttpClient createClient(final SolrParams params, ClientConnectionManager cm) { - public static AbstractHttpClient createClient(final SolrParams params, ClientConnectionManager cm) { - final ModifiableSolrParams config = new ModifiableSolrParams(params); - if (logger.isDebugEnabled()) { - logger.debug("Creating new http client, config:" + config); - } - final DefaultHttpClient httpClient = new DefaultHttpClient(cm); - configureClient(httpClient, config); - return httpClient; - } - - /** - * Configures {@link DefaultHttpClient}, only sets parameters if they are - * present in config. - */ - public static void configureClient(final DefaultHttpClient httpClient, - SolrParams config) { - configurer.configure(httpClient, config); - } - - public static void close(HttpClient httpClient) { -// if (httpClient instanceof CloseableHttpClient) { //RANGER_UPDATE -// org.apache.solr.common.util.IOUtils.closeQuietly((CloseableHttpClient) httpClient); //RANGER_UPDATE -// } else { //RANGER_UPDATE - httpClient.getConnectionManager().shutdown(); -// } //RANGER_UPDATE - } - - /** - * Control HTTP payload compression. - * - * @param allowCompression - * true will enable compression (needs support from server), false - * will disable compression. - */ - public static void setAllowCompression(DefaultHttpClient httpClient, - boolean allowCompression) { - httpClient - .removeRequestInterceptorByClass(UseCompressionRequestInterceptor.class); - httpClient - .removeResponseInterceptorByClass(UseCompressionResponseInterceptor.class); - if (allowCompression) { - httpClient.addRequestInterceptor(new UseCompressionRequestInterceptor()); - httpClient - .addResponseInterceptor(new UseCompressionResponseInterceptor()); - } - } - - /** - * Set http basic auth information. If basicAuthUser or basicAuthPass is null - * the basic auth configuration is cleared. Currently this is not preemtive - * authentication. So it is not currently possible to do a post request while - * using this setting. - */ - public static void setBasicAuth(DefaultHttpClient httpClient, - String basicAuthUser, String basicAuthPass) { - if (basicAuthUser != null && basicAuthPass != null) { - httpClient.getCredentialsProvider().setCredentials(AuthScope.ANY, - new UsernamePasswordCredentials(basicAuthUser, basicAuthPass)); - } else { - httpClient.getCredentialsProvider().clear(); - } - } - - /** - * Set max connections allowed per host. This call will only work when - * {@link ThreadSafeClientConnManager} or - * {@link PoolingClientConnectionManager} is used. - */ - public static void setMaxConnectionsPerHost(HttpClient httpClient, - int max) { - // would have been nice if there was a common interface - if (httpClient.getConnectionManager() instanceof ThreadSafeClientConnManager) { - ThreadSafeClientConnManager mgr = (ThreadSafeClientConnManager)httpClient.getConnectionManager(); - mgr.setDefaultMaxPerRoute(max); - } else if (httpClient.getConnectionManager() instanceof PoolingClientConnectionManager) { - PoolingClientConnectionManager mgr = (PoolingClientConnectionManager)httpClient.getConnectionManager(); - mgr.setDefaultMaxPerRoute(max); - } - } - - /** - * Set max total connections allowed. This call will only work when - * {@link ThreadSafeClientConnManager} or - * {@link PoolingClientConnectionManager} is used. - */ - public static void setMaxConnections(final HttpClient httpClient, - int max) { - // would have been nice if there was a common interface - if (httpClient.getConnectionManager() instanceof ThreadSafeClientConnManager) { - ThreadSafeClientConnManager mgr = (ThreadSafeClientConnManager)httpClient.getConnectionManager(); - mgr.setMaxTotal(max); - } else if (httpClient.getConnectionManager() instanceof PoolingClientConnectionManager) { - PoolingClientConnectionManager mgr = (PoolingClientConnectionManager)httpClient.getConnectionManager(); - mgr.setMaxTotal(max); - } - } - - - /** - * Defines the socket timeout (SO_TIMEOUT) in milliseconds. A timeout value of - * zero is interpreted as an infinite timeout. - * - * @param timeout timeout in milliseconds - */ - public static void setSoTimeout(HttpClient httpClient, int timeout) { - HttpConnectionParams.setSoTimeout(httpClient.getParams(), - timeout); - } - - /** - * Control retry handler - * @param useRetry when false the client will not try to retry failed requests. - */ - public static void setUseRetry(final DefaultHttpClient httpClient, - boolean useRetry) { - if (!useRetry) { - httpClient.setHttpRequestRetryHandler(NO_RETRY); - } else { - // if the request is not fully sent, we retry - // streaming updates are not a problem, because they are not retryable - httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(){ - @Override - protected boolean handleAsIdempotent(final HttpRequest request) { - return false; // we can't tell if a Solr request is idempotent - } - }); - } - } - - /** - * Set connection timeout. A timeout value of zero is interpreted as an - * infinite timeout. - * - * @param timeout - * connection Timeout in milliseconds - */ - public static void setConnectionTimeout(final HttpClient httpClient, - int timeout) { - HttpConnectionParams.setConnectionTimeout(httpClient.getParams(), - timeout); - } - - /** - * Set follow redirects. - * - * @param followRedirects When true the client will follow redirects. - */ - public static void setFollowRedirects(HttpClient httpClient, - boolean followRedirects) { - new ClientParamBean(httpClient.getParams()).setHandleRedirects(followRedirects); - } - - public static void setHostNameVerifier(DefaultHttpClient httpClient, - X509HostnameVerifier hostNameVerifier) { - Scheme httpsScheme = httpClient.getConnectionManager().getSchemeRegistry().get("https"); - if (httpsScheme != null) { - SSLSocketFactory sslSocketFactory = (SSLSocketFactory) httpsScheme.getSchemeSocketFactory(); - sslSocketFactory.setHostnameVerifier(hostNameVerifier); - } - } - - public static void setStaleCheckingEnabled(final HttpClient httpClient, boolean enabled) { - HttpConnectionParams.setStaleCheckingEnabled(httpClient.getParams(), enabled); - } - - public static void setTcpNoDelay(final HttpClient httpClient, boolean tcpNoDelay) { - HttpConnectionParams.setTcpNoDelay(httpClient.getParams(), tcpNoDelay); - } - - private static class UseCompressionRequestInterceptor implements - HttpRequestInterceptor { - - @Override - public void process(HttpRequest request, HttpContext context) - throws HttpException, IOException { - if (!request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip, deflate"); - } - } - } - - private static class UseCompressionResponseInterceptor implements - HttpResponseInterceptor { - - @Override - public void process(final HttpResponse response, final HttpContext context) - throws HttpException, IOException { - - HttpEntity entity = response.getEntity(); - Header ceheader = entity.getContentEncoding(); - if (ceheader != null) { - HeaderElement[] codecs = ceheader.getElements(); - for (int i = 0; i < codecs.length; i++) { - if (codecs[i].getName().equalsIgnoreCase("gzip")) { - response - .setEntity(new GzipDecompressingEntity(response.getEntity())); - return; - } - if (codecs[i].getName().equalsIgnoreCase("deflate")) { - response.setEntity(new DeflateDecompressingEntity(response - .getEntity())); - return; - } - } - } - } - } - - private static class GzipDecompressingEntity extends HttpEntityWrapper { - public GzipDecompressingEntity(final HttpEntity entity) { - super(entity); - } - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - return new GZIPInputStream(wrappedEntity.getContent()); - } - - @Override - public long getContentLength() { - return -1; - } - } - - private static class DeflateDecompressingEntity extends - GzipDecompressingEntity { - public DeflateDecompressingEntity(final HttpEntity entity) { - super(entity); - } - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - return new InflaterInputStream(wrappedEntity.getContent()); - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java deleted file mode 100644 index a073265..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java +++ /dev/null @@ -1,821 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.impl; - -import org.apache.commons.io.IOUtils; -import org.apache.http.Header; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.NoHttpResponseException; -import org.apache.http.client.HttpClient; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.entity.mime.FormBodyPart; -import org.apache.http.entity.mime.HttpMultipartMode; -import org.apache.http.entity.mime.MultipartEntity; -import org.apache.http.entity.mime.content.InputStreamBody; -import org.apache.http.entity.mime.content.StringBody; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.message.BasicHeader; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.apache.solr.client.solrj.ResponseParser; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.RequestWriter; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.ContentStream; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SolrjNamedThreadFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.net.ConnectException; -import java.net.SocketTimeoutException; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -public class HttpSolrClient extends SolrClient { - - private static final String UTF_8 = StandardCharsets.UTF_8.name(); - private static final String DEFAULT_PATH = "/select"; - private static final long serialVersionUID = -946812319974801896L; - - /** - * User-Agent String. - */ - public static final String AGENT = "Solr[" + HttpSolrClient.class.getName() + "] 1.0"; - - private static Logger log = LoggerFactory.getLogger(HttpSolrClient.class); - - /** - * The URL of the Solr server. - */ - protected volatile String baseUrl; - - /** - * Default value: null / empty. - * <p> - * Parameters that are added to every request regardless. This may be a place - * to add something like an authentication token. - */ - protected ModifiableSolrParams invariantParams; - - /** - * Default response parser is BinaryResponseParser - * <p> - * This parser represents the default Response Parser chosen to parse the - * response if the parser were not specified as part of the request. - * - * @see org.apache.solr.client.solrj.impl.BinaryResponseParser - */ - protected volatile ResponseParser parser; - - /** - * The RequestWriter used to write all requests to Solr - * - * @see org.apache.solr.client.solrj.request.RequestWriter - */ - protected volatile RequestWriter requestWriter = new RequestWriter(); - - private final HttpClient httpClient; - - private volatile boolean followRedirects = false; - - private volatile int maxRetries = 0; - - private volatile boolean useMultiPartPost; - private final boolean internalClient; - - private volatile Set<String> queryParams = Collections.emptySet(); - - /** - * @param baseURL - * The URL of the Solr server. For example, " - * <code>http://localhost:8983/solr/</code>" if you are using the - * standard distribution Solr webapp on your local machine. - */ - public HttpSolrClient(String baseURL) { - this(baseURL, null, new BinaryResponseParser()); - } - - public HttpSolrClient(String baseURL, HttpClient client) { - this(baseURL, client, new BinaryResponseParser()); - } - - public HttpSolrClient(String baseURL, HttpClient client, ResponseParser parser) { - this.baseUrl = baseURL; - if (baseUrl.endsWith("/")) { - baseUrl = baseUrl.substring(0, baseUrl.length() - 1); - } - if (baseUrl.indexOf('?') >= 0) { - throw new RuntimeException( - "Invalid base url for solrj. The base URL must not contain parameters: " - + baseUrl); - } - - if (client != null) { - httpClient = client; - internalClient = false; - } else { - internalClient = true; - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 128); - params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 32); - params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, followRedirects); - httpClient = HttpClientUtil.createClient(params); - } - - this.parser = parser; - } - - public Set<String> getQueryParams() { - return queryParams; - } - - /** - * Expert Method - * @param queryParams set of param keys to only send via the query string - * Note that the param will be sent as a query string if the key is part - * of this Set or the SolrRequest's query params. - * @see org.apache.solr.client.solrj.SolrRequest#getQueryParams - */ - public void setQueryParams(Set<String> queryParams) { - this.queryParams = queryParams; - } - - /** - * Process the request. If - * {@link org.apache.solr.client.solrj.SolrRequest#getResponseParser()} is - * null, then use {@link #getParser()} - * - * @param request - * The {@link org.apache.solr.client.solrj.SolrRequest} to process - * @return The {@link org.apache.solr.common.util.NamedList} result - * @throws IOException If there is a low-level I/O error. - * - * @see #request(org.apache.solr.client.solrj.SolrRequest, - * org.apache.solr.client.solrj.ResponseParser) - */ - @Override - public NamedList<Object> request(final SolrRequest request) - throws SolrServerException, IOException { - ResponseParser responseParser = request.getResponseParser(); - if (responseParser == null) { - responseParser = parser; - } - return request(request, responseParser); - } - - public NamedList<Object> request(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException { - return executeMethod(createMethod(request),processor); - } - - /** - * @lucene.experimental - */ - public static class HttpUriRequestResponse { - public HttpUriRequest httpUriRequest; - public Future<NamedList<Object>> future; - } - - /** - * @lucene.experimental - */ - public HttpUriRequestResponse httpUriRequest(final SolrRequest request) - throws SolrServerException, IOException { - ResponseParser responseParser = request.getResponseParser(); - if (responseParser == null) { - responseParser = parser; - } - return httpUriRequest(request, responseParser); - } - - /** - * @lucene.experimental - */ - public HttpUriRequestResponse httpUriRequest(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException { - HttpUriRequestResponse mrr = new HttpUriRequestResponse(); - final HttpRequestBase method = createMethod(request); - ExecutorService pool = Executors.newFixedThreadPool(1, new SolrjNamedThreadFactory("httpUriRequest")); - try { - mrr.future = pool.submit(new Callable<NamedList<Object>>(){ - - @Override - public NamedList<Object> call() throws Exception { - return executeMethod(method, processor); - }}); - - } finally { - pool.shutdown(); - } - assert method != null; - mrr.httpUriRequest = method; - return mrr; - } - - protected ModifiableSolrParams calculateQueryParams(Set<String> queryParamNames, - ModifiableSolrParams wparams) { - ModifiableSolrParams queryModParams = new ModifiableSolrParams(); - if (queryParamNames != null) { - for (String param : queryParamNames) { - String[] value = wparams.getParams(param) ; - if (value != null) { - for (String v : value) { - queryModParams.add(param, v); - } - wparams.remove(param); - } - } - } - return queryModParams; - } - - protected HttpRequestBase createMethod(final SolrRequest request) throws IOException, SolrServerException { - HttpRequestBase method = null; - InputStream is = null; - SolrParams params = request.getParams(); - Collection<ContentStream> streams = requestWriter.getContentStreams(request); - String path = requestWriter.getPath(request); - if (path == null || !path.startsWith("/")) { - path = DEFAULT_PATH; - } - - ResponseParser parser = request.getResponseParser(); - if (parser == null) { - parser = this.parser; - } - - // The parser 'wt=' and 'version=' params are used instead of the original - // params - ModifiableSolrParams wparams = new ModifiableSolrParams(params); - if (parser != null) { - wparams.set(CommonParams.WT, parser.getWriterType()); - wparams.set(CommonParams.VERSION, parser.getVersion()); - } - if (invariantParams != null) { - wparams.add(invariantParams); - } - - int tries = maxRetries + 1; - try { - while( tries-- > 0 ) { - // Note: since we aren't do intermittent time keeping - // ourselves, the potential non-timeout latency could be as - // much as tries-times (plus scheduling effects) the given - // timeAllowed. - try { - if( SolrRequest.METHOD.GET == request.getMethod() ) { - if( streams != null ) { - throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!" ); - } - method = new HttpGet( baseUrl + path + ClientUtils.toQueryString( wparams, false ) ); - } - else if( SolrRequest.METHOD.POST == request.getMethod() || SolrRequest.METHOD.PUT == request.getMethod() ) { - - String url = baseUrl + path; - boolean hasNullStreamName = false; - if (streams != null) { - for (ContentStream cs : streams) { - if (cs.getName() == null) { - hasNullStreamName = true; - break; - } - } - } - boolean isMultipart = ((this.useMultiPartPost && SolrRequest.METHOD.POST == request.getMethod()) - || ( streams != null && streams.size() > 1 )) && !hasNullStreamName; - - LinkedList<NameValuePair> postOrPutParams = new LinkedList<>(); - if (streams == null || isMultipart) { - // send server list and request list as query string params - ModifiableSolrParams queryParams = calculateQueryParams(this.queryParams, wparams); - queryParams.add(calculateQueryParams(request.getQueryParams(), wparams)); - String fullQueryUrl = url + ClientUtils.toQueryString( queryParams, false ); - HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ? - new HttpPost(fullQueryUrl) : new HttpPut(fullQueryUrl); - if (!isMultipart) { - postOrPut.addHeader("Content-Type", - "application/x-www-form-urlencoded; charset=UTF-8"); - } - - List<FormBodyPart> parts = new LinkedList<>(); - Iterator<String> iter = wparams.getParameterNamesIterator(); - while (iter.hasNext()) { - String p = iter.next(); - String[] vals = wparams.getParams(p); - if (vals != null) { - for (String v : vals) { - if (isMultipart) { - parts.add(new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8))); - } else { - postOrPutParams.add(new BasicNameValuePair(p, v)); - } - } - } - } - - if (isMultipart && streams != null) { - for (ContentStream content : streams) { - String contentType = content.getContentType(); - if(contentType==null) { - contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default - } - String name = content.getName(); - if(name==null) { - name = ""; - } - parts.add(new FormBodyPart(name, - new InputStreamBody( - content.getStream(), - contentType, - content.getName()))); - } - } - - if (parts.size() > 0) { - MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT); - for(FormBodyPart p: parts) { - entity.addPart(p); - } - postOrPut.setEntity(entity); - } else { - //not using multipart - postOrPut.setEntity(new UrlEncodedFormEntity(postOrPutParams, StandardCharsets.UTF_8)); - } - - method = postOrPut; - } - // It is has one stream, it is the post body, put the params in the URL - else { - String pstr = ClientUtils.toQueryString(wparams, false); - HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ? - new HttpPost(url + pstr) : new HttpPut(url + pstr); - - // Single stream as body - // Using a loop just to get the first one - final ContentStream[] contentStream = new ContentStream[1]; - for (ContentStream content : streams) { - contentStream[0] = content; - break; - } - if (contentStream[0] instanceof RequestWriter.LazyContentStream) { - postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { - @Override - public Header getContentType() { - return new BasicHeader("Content-Type", contentStream[0].getContentType()); - } - - @Override - public boolean isRepeatable() { - return false; - } - - }); - } else { - postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) { - @Override - public Header getContentType() { - return new BasicHeader("Content-Type", contentStream[0].getContentType()); - } - - @Override - public boolean isRepeatable() { - return false; - } - }); - } - method = postOrPut; - } - } - else { - throw new SolrServerException("Unsupported method: "+request.getMethod() ); - } - } - catch( NoHttpResponseException r ) { - method = null; - if(is != null) { - is.close(); - } - // If out of tries then just rethrow (as normal error). - if (tries < 1) { - throw r; - } - } - } - } catch (IOException ex) { - throw new SolrServerException("error reading streams", ex); - } - - return method; - } - - protected NamedList<Object> executeMethod(HttpRequestBase method, final ResponseParser processor) throws SolrServerException { - method.addHeader("User-Agent", AGENT); - - InputStream respBody = null; - boolean shouldClose = true; - boolean success = false; - try { - // Execute the method. - final HttpResponse response = httpClient.execute(method); - int httpStatus = response.getStatusLine().getStatusCode(); - - // Read the contents - respBody = response.getEntity().getContent(); - Header ctHeader = response.getLastHeader("content-type"); - String contentType; - if (ctHeader != null) { - contentType = ctHeader.getValue(); - } else { - contentType = ""; - } - - // handle some http level checks before trying to parse the response - switch (httpStatus) { - case HttpStatus.SC_OK: - case HttpStatus.SC_BAD_REQUEST: - case HttpStatus.SC_CONFLICT: // 409 - break; - case HttpStatus.SC_MOVED_PERMANENTLY: - case HttpStatus.SC_MOVED_TEMPORARILY: - if (!followRedirects) { - throw new SolrServerException("Server at " + getBaseURL() - + " sent back a redirect (" + httpStatus + ")."); - } - break; - default: - if (processor == null) { - throw new RemoteSolrException(baseUrl, httpStatus, "non ok status: " + httpStatus - + ", message:" + response.getStatusLine().getReasonPhrase(), - null); - } - } - if (processor == null) { - - // no processor specified, return raw stream - NamedList<Object> rsp = new NamedList<>(); - rsp.add("stream", respBody); - // Only case where stream should not be closed - shouldClose = false; - success = true; - return rsp; - } - - String procCt = processor.getContentType(); - if (procCt != null) { - String procMimeType = ContentType.parse(procCt).getMimeType().trim().toLowerCase(Locale.ROOT); - String mimeType = ContentType.parse(contentType).getMimeType().trim().toLowerCase(Locale.ROOT); - if (!procMimeType.equals(mimeType)) { - // unexpected mime type - String msg = "Expected mime type " + procMimeType + " but got " + mimeType + "."; - Header encodingHeader = response.getEntity().getContentEncoding(); - String encoding; - if (encodingHeader != null) { - encoding = encodingHeader.getValue(); - } else { - encoding = "UTF-8"; // try UTF-8 - } - try { - msg = msg + " " + IOUtils.toString(respBody, encoding); - } catch (IOException e) { - throw new RemoteSolrException(baseUrl, httpStatus, "Could not parse response with encoding " + encoding, e); - } - throw new RemoteSolrException(baseUrl, httpStatus, msg, null); - } - } - - NamedList<Object> rsp = null; - String charset = EntityUtils.getContentCharSet(response.getEntity()); - try { - rsp = processor.processResponse(respBody, charset); - } catch (Exception e) { - throw new RemoteSolrException(baseUrl, httpStatus, e.getMessage(), e); - } - if (httpStatus != HttpStatus.SC_OK) { - NamedList<String> metadata = null; - String reason = null; - try { - NamedList err = (NamedList) rsp.get("error"); - if (err != null) { - reason = (String) err.get("msg"); - if(reason == null) { - reason = (String) err.get("trace"); - } - metadata = (NamedList<String>)err.get("metadata"); - } - } catch (Exception ex) {} - if (reason == null) { - StringBuilder msg = new StringBuilder(); - msg.append(response.getStatusLine().getReasonPhrase()); - msg.append("\n\n"); - msg.append("request: " + method.getURI()); - reason = java.net.URLDecoder.decode(msg.toString(), UTF_8); - } - RemoteSolrException rss = new RemoteSolrException(baseUrl, httpStatus, reason, null); - if (metadata != null) rss.setMetadata(metadata); - throw rss; - } - success = true; - return rsp; - } catch (ConnectException e) { - throw new SolrServerException("Server refused connection at: " - + getBaseURL(), e); - } catch (SocketTimeoutException e) { - throw new SolrServerException( - "Timeout occured while waiting response from server at: " - + getBaseURL(), e); - } catch (IOException e) { - throw new SolrServerException( - "IOException occured when talking to server at: " + getBaseURL(), e); - } finally { - if (respBody != null && shouldClose) { - try { - respBody.close(); - } catch (IOException e) { - log.error("", e); - } finally { - if (!success) { - method.abort(); - } - } - } - } - } - - // ------------------------------------------------------------------- - // ------------------------------------------------------------------- - - /** - * Retrieve the default list of parameters are added to every request - * regardless. - * - * @see #invariantParams - */ - public ModifiableSolrParams getInvariantParams() { - return invariantParams; - } - - public String getBaseURL() { - return baseUrl; - } - - public void setBaseURL(String baseURL) { - this.baseUrl = baseURL; - } - - public ResponseParser getParser() { - return parser; - } - - /** - * Note: This setter method is <b>not thread-safe</b>. - * - * @param processor - * Default Response Parser chosen to parse the response if the parser - * were not specified as part of the request. - * @see org.apache.solr.client.solrj.SolrRequest#getResponseParser() - */ - public void setParser(ResponseParser processor) { - parser = processor; - } - - /** - * Return the HttpClient this instance uses. - */ - public HttpClient getHttpClient() { - return httpClient; - } - - /** - * HttpConnectionParams.setConnectionTimeout - * - * @param timeout - * Timeout in milliseconds - **/ - public void setConnectionTimeout(int timeout) { - HttpClientUtil.setConnectionTimeout(httpClient, timeout); - } - - /** - * Set SoTimeout (read timeout). This is desirable - * for queries, but probably not for indexing. - * - * @param timeout - * Timeout in milliseconds - **/ - public void setSoTimeout(int timeout) { - HttpClientUtil.setSoTimeout(httpClient, timeout); - } - - /** - * Configure whether the client should follow redirects or not. - * <p> - * This defaults to false under the assumption that if you are following a - * redirect to get to a Solr installation, something is misconfigured - * somewhere. - * </p> - */ - public void setFollowRedirects(boolean followRedirects) { - this.followRedirects = followRedirects; - HttpClientUtil.setFollowRedirects(httpClient, followRedirects); - } - - /** - * Allow server->client communication to be compressed. Currently gzip and - * deflate are supported. If the server supports compression the response will - * be compressed. This method is only allowed if the http client is of type - * DefatulHttpClient. - */ - public void setAllowCompression(boolean allowCompression) { - if (httpClient instanceof DefaultHttpClient) { - HttpClientUtil.setAllowCompression((DefaultHttpClient) httpClient, allowCompression); - } else { - throw new UnsupportedOperationException( - "HttpClient instance was not of type DefaultHttpClient"); - } - } - - /** - * Set maximum number of retries to attempt in the event of transient errors. - * <p> - * Maximum number of retries to attempt in the event of transient errors. - * Default: 0 (no) retries. No more than 1 recommended. - * </p> - * @param maxRetries - * No more than 1 recommended - */ - public void setMaxRetries(int maxRetries) { - if (maxRetries > 1) { - log.warn("HttpSolrServer: maximum Retries " + maxRetries - + " > 1. Maximum recommended retries is 1."); - } - this.maxRetries = maxRetries; - } - - public void setRequestWriter(RequestWriter requestWriter) { - this.requestWriter = requestWriter; - } - - /** - * Adds the documents supplied by the given iterator. - * - * @param docIterator - * the iterator which returns SolrInputDocument instances - * - * @return the response from the SolrServer - */ - public UpdateResponse add(Iterator<SolrInputDocument> docIterator) - throws SolrServerException, IOException { - UpdateRequest req = new UpdateRequest(); - req.setDocIterator(docIterator); - return req.process(this); - } - - /** - * Adds the beans supplied by the given iterator. - * - * @param beanIterator - * the iterator which returns Beans - * - * @return the response from the SolrServer - */ - public UpdateResponse addBeans(final Iterator<?> beanIterator) - throws SolrServerException, IOException { - UpdateRequest req = new UpdateRequest(); - req.setDocIterator(new Iterator<SolrInputDocument>() { - - @Override - public boolean hasNext() { - return beanIterator.hasNext(); - } - - @Override - public SolrInputDocument next() { - Object o = beanIterator.next(); - if (o == null) return null; - return getBinder().toSolrInputDocument(o); - } - - @Override - public void remove() { - beanIterator.remove(); - } - }); - return req.process(this); - } - - /** - * Close the {@link ClientConnectionManager} from the internal client. - */ - @Override - public void close() throws IOException { - shutdown(); - } - - @Override - @Deprecated - public void shutdown() { - if (httpClient != null && internalClient) { - HttpClientUtil.close(httpClient); - } - } - - /** - * Set the maximum number of connections that can be open to a single host at - * any given time. If http client was created outside the operation is not - * allowed. - */ - public void setDefaultMaxConnectionsPerHost(int max) { - if (internalClient) { - HttpClientUtil.setMaxConnectionsPerHost(httpClient, max); - } else { - throw new UnsupportedOperationException( - "Client was created outside of HttpSolrServer"); - } - } - - /** - * Set the maximum number of connections that can be open at any given time. - * If http client was created outside the operation is not allowed. - */ - public void setMaxTotalConnections(int max) { - if (internalClient) { - HttpClientUtil.setMaxConnections(httpClient, max); - } else { - throw new UnsupportedOperationException( - "Client was created outside of HttpSolrServer"); - } - } - - public boolean isUseMultiPartPost() { - return useMultiPartPost; - } - - /** - * Set the multipart connection properties - */ - public void setUseMultiPartPost(boolean useMultiPartPost) { - this.useMultiPartPost = useMultiPartPost; - } - - /** - * Subclass of SolrException that allows us to capture an arbitrary HTTP - * status code that may have been returned by the remote server or a - * proxy along the way. - */ - public static class RemoteSolrException extends SolrException { - /** - * @param remoteHost the host the error was received from - * @param code Arbitrary HTTP status code - * @param msg Exception Message - * @param th Throwable to wrap with this Exception - */ - public RemoteSolrException(String remoteHost, int code, String msg, Throwable th) { - super(code, "Error from server at " + remoteHost + ": " + msg, th); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d9a661cf/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java ---------------------------------------------------------------------- diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java deleted file mode 100644 index ef5d439..0000000 --- a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.impl; - -import org.apache.http.client.HttpClient; -import org.apache.solr.client.solrj.ResponseParser; - -/** - * @deprecated Use {@link org.apache.solr.client.solrj.impl.HttpSolrClient} - */ -@Deprecated -public class HttpSolrServer extends HttpSolrClient { - - public HttpSolrServer(String baseURL) { - super(baseURL); - } - - public HttpSolrServer(String baseURL, HttpClient client) { - super(baseURL, client); - } - - public HttpSolrServer(String baseURL, HttpClient client, ResponseParser parser) { - super(baseURL, client, parser); - } - -}
