[
https://issues.apache.org/jira/browse/BAHIR-154?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16313229#comment-16313229
]
ASF GitHub Bot commented on BAHIR-154:
--------------------------------------
Github user ricellis commented on a diff in the pull request:
https://github.com/apache/bahir/pull/61#discussion_r159845718
--- Diff:
sql-cloudant/src/main/scala/org/apache/bahir/cloudant/CloudantConfig.scala ---
@@ -16,34 +16,127 @@
*/
package org.apache.bahir.cloudant
-import java.net.URLEncoder
+import java.net.{URL, URLEncoder}
-import play.api.libs.json.{JsArray, JsObject, Json, JsValue}
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.reflect.io.File
+
+import com.cloudant.client.api.{ClientBuilder, CloudantClient, Database}
+import com.cloudant.client.api.model.SearchResult
+import com.cloudant.client.api.views._
+import com.cloudant.http.{Http, HttpConnection}
+import com.cloudant.http.interceptors.Replay429Interceptor
+import com.google.gson.{JsonObject, JsonParser}
import org.apache.bahir.cloudant.common._
+import org.apache.bahir.cloudant.common.JsonUtil.JsonConverter
/*
* Only allow one field pushdown now
* as the filter today does not tell how to link the filters out And v.s. Or
*/
class CloudantConfig(val protocol: String, val host: String,
- val dbName: String, val indexName: String, val
viewName: String)
+ val dbName: String, val indexPath: String, val
viewPath: String)
(implicit val username: String, val password: String,
val partitions: Int, val maxInPartition: Int, val
minInPartition: Int,
val requestTimeout: Long, val bulkSize: Int, val
schemaSampleSize: Int,
val createDBOnSave: Boolean, val endpoint: String,
val useQuery: Boolean = false, val queryLimit: Int)
extends Serializable {
+ @transient private lazy val client: CloudantClient = ClientBuilder
+ .url(getClientUrl)
+ .username(username)
+ .password(password)
+ .interceptors(Replay429Interceptor.WITH_DEFAULTS)
+ .build
+ @transient private lazy val database: Database = client.database(dbName,
false)
lazy val dbUrl: String = {protocol + "://" + host + "/" + dbName}
+ lazy val designDoc: String = {
+ if (viewPath != null && viewPath.nonEmpty) {
+ viewPath.split("/")(1)
+ } else {
+ null
+ }
+ }
+ lazy val searchName: String = {
+ // verify that the index path matches '_design/ddoc/_search/searchname'
+ if (indexPath != null && indexPath.nonEmpty &&
indexPath.matches("\\w+\\/\\w+\\/\\w+\\/\\w+")) {
+ val splitPath = indexPath.split(File.separator)
+ // return 'design-doc/search-name'
+ splitPath(1) + File.separator + splitPath(3)
+ } else {
+ null
+ }
+ }
+ lazy val viewName: String = {
+ if (viewPath != null && viewPath.nonEmpty) {
+ val splitViewPath = viewPath.split(File.separator)
+ if(splitViewPath(3).contains("?")) {
+ splitViewPath(3).substring(0, splitViewPath(3).indexOf("?"))
+ } else {
+ splitViewPath(3)
+ }
+ } else {
+ null
+ }
+ }
val pkField = "_id"
val defaultIndex: String = endpoint
val default_filter: String = "*:*"
- def getDbUrl: String = {
- dbUrl
+ def buildAllDocsRequest(limit: Int, includeDocs: Boolean = true):
AllDocsRequestBuilder = {
+ var allDocsReq =
database.getAllDocsRequestBuilder.includeDocs(includeDocs)
+ if (limit != JsonStoreConfigManager.ALLDOCS_OR_CHANGES_LIMIT) {
+ allDocsReq = allDocsReq.limit(limit)
+ }
+ allDocsReq
+ }
+
+ def buildViewRequest(limit: Int, includeDocs: Boolean = true):
+ UnpaginatedRequestBuilder[String, String] = {
+ val viewReq = database.getViewRequestBuilder(designDoc, viewName)
+ .newRequest(Key.Type.STRING, classOf[String])
+ .includeDocs(includeDocs)
+ if (limit != JsonStoreConfigManager.ALLDOCS_OR_CHANGES_LIMIT) {
+ viewReq.limit(limit)
+ }
+ viewReq
+ }
+
+ def buildSearchRequest(limit: Int): SearchResult[JsonObject] = {
+ val searchReq = database.search(searchName)
+ if (limit != JsonStoreConfigManager.ALLDOCS_OR_CHANGES_LIMIT) {
+ searchReq.limit(limit)
+ }
+ searchReq.querySearchResult(default_filter, classOf[JsonObject])
+ }
+
+ def executeRequest(stringUrl: String, postData: String = null):
HttpConnection = {
+ val url = new URL(stringUrl)
+ if(postData != null) {
+ val conn = Http.POST(url, "application/json")
+ conn.setRequestBody(postData)
+ conn.requestProperties.put("Accept", "application/json")
--- End diff --
Is this necessary? CouchDB will default to sending back JSON anyway won't
it?
> Refactor sql-cloudant to use Cloudant's java-cloudant features
> --------------------------------------------------------------
>
> Key: BAHIR-154
> URL: https://issues.apache.org/jira/browse/BAHIR-154
> Project: Bahir
> Issue Type: Improvement
> Affects Versions: Spark-2.2.0
> Reporter: Esteban Laver
> Assignee: Esteban Laver
>
> Cloudant's java-cloudant library (which is currently used for testing)
> contains several features that sql-cloudant can benefit from:
> - HTTP 429 backoff
> - View builder API to potentially simplify loading for _all_docs/views
> - Improved exception handling when executing HTTP requests
> - Future support for IAM API key
> Would need to replace current scala HTTP library with OkHttp library, and
> also replace play-json with GSON library.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
