[ https://issues.apache.org/jira/browse/BAHIR-154?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16313236#comment-16313236 ]
ASF GitHub Bot commented on BAHIR-154: -------------------------------------- Github user ricellis commented on a diff in the pull request: https://github.com/apache/bahir/pull/61#discussion_r159845855 --- Diff: sql-cloudant/src/main/scala/org/apache/bahir/cloudant/CloudantConfig.scala --- @@ -16,34 +16,127 @@ */ package org.apache.bahir.cloudant -import java.net.URLEncoder +import java.net.{URL, URLEncoder} -import play.api.libs.json.{JsArray, JsObject, Json, JsValue} +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.reflect.io.File + +import com.cloudant.client.api.{ClientBuilder, CloudantClient, Database} +import com.cloudant.client.api.model.SearchResult +import com.cloudant.client.api.views._ +import com.cloudant.http.{Http, HttpConnection} +import com.cloudant.http.interceptors.Replay429Interceptor +import com.google.gson.{JsonObject, JsonParser} import org.apache.bahir.cloudant.common._ +import org.apache.bahir.cloudant.common.JsonUtil.JsonConverter /* * Only allow one field pushdown now * as the filter today does not tell how to link the filters out And v.s. Or */ class CloudantConfig(val protocol: String, val host: String, - val dbName: String, val indexName: String, val viewName: String) + val dbName: String, val indexPath: String, val viewPath: String) (implicit val username: String, val password: String, val partitions: Int, val maxInPartition: Int, val minInPartition: Int, val requestTimeout: Long, val bulkSize: Int, val schemaSampleSize: Int, val createDBOnSave: Boolean, val endpoint: String, val useQuery: Boolean = false, val queryLimit: Int) extends Serializable { + @transient private lazy val client: CloudantClient = ClientBuilder + .url(getClientUrl) + .username(username) + .password(password) + .interceptors(Replay429Interceptor.WITH_DEFAULTS) + .build + @transient private lazy val database: Database = client.database(dbName, false) lazy val dbUrl: String = {protocol + "://" + host + "/" + dbName} + lazy val designDoc: String = { + if (viewPath != null && viewPath.nonEmpty) { + viewPath.split("/")(1) + } else { + null + } + } + lazy val searchName: String = { + // verify that the index path matches '_design/ddoc/_search/searchname' + if (indexPath != null && indexPath.nonEmpty && indexPath.matches("\\w+\\/\\w+\\/\\w+\\/\\w+")) { + val splitPath = indexPath.split(File.separator) + // return 'design-doc/search-name' + splitPath(1) + File.separator + splitPath(3) + } else { + null + } + } + lazy val viewName: String = { + if (viewPath != null && viewPath.nonEmpty) { + val splitViewPath = viewPath.split(File.separator) + if(splitViewPath(3).contains("?")) { + splitViewPath(3).substring(0, splitViewPath(3).indexOf("?")) + } else { + splitViewPath(3) + } + } else { + null + } + } val pkField = "_id" val defaultIndex: String = endpoint val default_filter: String = "*:*" - def getDbUrl: String = { - dbUrl + def buildAllDocsRequest(limit: Int, includeDocs: Boolean = true): AllDocsRequestBuilder = { + var allDocsReq = database.getAllDocsRequestBuilder.includeDocs(includeDocs) + if (limit != JsonStoreConfigManager.ALLDOCS_OR_CHANGES_LIMIT) { + allDocsReq = allDocsReq.limit(limit) + } + allDocsReq + } + + def buildViewRequest(limit: Int, includeDocs: Boolean = true): + UnpaginatedRequestBuilder[String, String] = { + val viewReq = database.getViewRequestBuilder(designDoc, viewName) + .newRequest(Key.Type.STRING, classOf[String]) + .includeDocs(includeDocs) + if (limit != JsonStoreConfigManager.ALLDOCS_OR_CHANGES_LIMIT) { + viewReq.limit(limit) + } + viewReq + } + + def buildSearchRequest(limit: Int): SearchResult[JsonObject] = { + val searchReq = database.search(searchName) + if (limit != JsonStoreConfigManager.ALLDOCS_OR_CHANGES_LIMIT) { + searchReq.limit(limit) + } + searchReq.querySearchResult(default_filter, classOf[JsonObject]) + } + + def executeRequest(stringUrl: String, postData: String = null): HttpConnection = { + val url = new URL(stringUrl) + if(postData != null) { + val conn = Http.POST(url, "application/json") + conn.setRequestBody(postData) + conn.requestProperties.put("Accept", "application/json") + conn.requestProperties.put("User-Agent", "spark-cloudant") + client.executeRequest(conn) + } else { + val conn = Http.GET(url) + conn.requestProperties.put("Accept", "application/json") --- End diff -- As above > Refactor sql-cloudant to use Cloudant's java-cloudant features > -------------------------------------------------------------- > > Key: BAHIR-154 > URL: https://issues.apache.org/jira/browse/BAHIR-154 > Project: Bahir > Issue Type: Improvement > Affects Versions: Spark-2.2.0 > Reporter: Esteban Laver > Assignee: Esteban Laver > > Cloudant's java-cloudant library (which is currently used for testing) > contains several features that sql-cloudant can benefit from: > - HTTP 429 backoff > - View builder API to potentially simplify loading for _all_docs/views > - Improved exception handling when executing HTTP requests > - Future support for IAM API key > Would need to replace current scala HTTP library with OkHttp library, and > also replace play-json with GSON library. -- This message was sent by Atlassian JIRA (v6.4.14#64029)