I had also same problem to use JdbcRDD in java.
For me, I have written a class in scala to get JdbcRDD, and I call this
instance from java.
for instance, JdbcRDDWrapper.scala like this:
...
import java.sql._
import org.apache.spark.SparkContext
import org.apache.spark.rdd.JdbcRDD
import com.gsshop.polaris.domain.event._
class JdbcRDDWrapper(sc: SparkContext, rowCount: Long, from: Long, to: Long)
{
def getItemViewEventJdbcRdd(): JdbcRDD[ItemViewEvent] =
{
val sql = "" +
"SELECT " +
" i.ID as id," +
" i.ITEM_ID as \"itemViewEvent.itemId\"," +
" i.BRAND_ID as \"itemViewEvent.brandId\"," +
" i.ITEM_TYPE as \"itemViewEvent.itemType\"," +
" i.PROMOTION_ID as \"itemViewEvent.promotionId\"," +
" i.PRICE as \"itemViewEvent.price\"," +
" i.ITEM_TITLE as \"itemViewEvent.itemTitle\"," +
" i.ITEM_DESCRIPTION as \"itemViewEvent.itemDescription\"," +
" i.THUMB_NAIL_URL as \"itemViewEvent.thumbnailUrl\"," +
" i.LOAD_DATE as loadDate," +
" b.EVENT_TYPE as \"itemViewEvent.baseProperties.eventType\"," +
" b.TIMESTAMP as \"itemViewEvent.baseProperties.timestamp\"," +
" b.URL as \"itemViewEvent.baseProperties.url\"," +
" b.REFERER as \"itemViewEvent.baseProperties.referer\"," +
" b.UID as \"itemViewEvent.baseProperties.uid\"," +
" b.PCID as \"itemViewEvent.baseProperties.pcid\"," +
" b.SERVICE_ID as \"itemViewEvent.baseProperties.serviceId\"," +
" b.VERSION as \"itemViewEvent.baseProperties.version\"," +
" b.DEVICE_TYPE as \"itemViewEvent.baseProperties.deviceType\"," +
" b.DOMAIN as \"itemViewEvent.baseProperties.domain\"," +
" b.SITE as \"itemViewEvent.baseProperties.site\" " +
"FROM ITEM_VIEW_EVENT AS i " +
"INNER JOIN BASE_PROPERTIES AS b " +
"ON i.ID = b.EVENT_ID " +
"WHERE b.TIMESTAMP != ? AND " + from +
" <= b.TIMESTAMP AND b.TIMESTAMP < " + to + " LIMIT ? "
val rdd = new JdbcRDD(
sc,
() => {
Class.forName("org.apache.phoenix.jdbc.PhoenixDriver")
DriverManager.getConnection("jdbc:phoenix:xxxxxx:/hbase-unsecure")
},
sql,
0, rowCount, 5,
(rs: ResultSet) => {
val baseProperties = new BaseProperties()
baseProperties.setEventType(rs.getString(
"itemViewEvent.baseProperties.eventType"))
baseProperties.setTimestamp(rs.getLong(
"itemViewEvent.baseProperties.timestamp"))
baseProperties.setUrl(rs.getString(
"itemViewEvent.baseProperties.url"))
baseProperties.setReferer(rs.getString(
"itemViewEvent.baseProperties.referer"))
baseProperties.setUid(rs.getString(
"itemViewEvent.baseProperties.uid"))
baseProperties.setPcid(rs.getString(
"itemViewEvent.baseProperties.pcid"))
baseProperties.setServiceId(rs.getString(
"itemViewEvent.baseProperties.serviceId"))
baseProperties.setVersion(rs.getString(
"itemViewEvent.baseProperties.version"))
baseProperties.setDeviceType(rs.getString(
"itemViewEvent.baseProperties.deviceType"))
baseProperties.setDomain(rs.getString(
"itemViewEvent.baseProperties.domain"))
baseProperties.setSite(rs.getString(
"itemViewEvent.baseProperties.site"))
val itemViewEvent = new ItemViewEvent()
itemViewEvent.setItemId(rs.getString("itemViewEvent.itemId"))
itemViewEvent.setBrandId(rs.getString("itemViewEvent.brandId"))
itemViewEvent.setItemType(rs.getString("itemViewEvent.itemType"))
itemViewEvent.setPromotionId(rs.getString(
"itemViewEvent.promotionId"))
itemViewEvent.setPrice(rs.getLong("itemViewEvent.price"))
itemViewEvent.setItemTitle(rs.getString("itemViewEvent.itemTitle"))
itemViewEvent.setItemDescription(rs.getString(
"itemViewEvent.itemDescription"))
itemViewEvent.setThumbnailUrl(rs.getString(
"itemViewEvent.thumbnailUrl"))
itemViewEvent.setBaseProperties(baseProperties)
itemViewEvent
})
rdd
}
}
and from java, JdbcRdd can be received:
import scala.reflect.ClassManifestFactory$;
...
JdbcRDD<ItemViewEvent> jdbcRddItemViewEvent = new
JdbcRDDWrapper(JavaSparkContext.toSparkContext(ctx), rowCountItemViewEvent,
fromTime, toTime).getItemViewEventJdbcRdd();
JavaRDD<ItemViewEvent> javaRddItemViewEvent =
JavaRDD.fromRDD(jdbcRddItemViewEvent,
ClassManifestFactory$.MODULE$.fromClass(ItemViewEvent.class));
- Kidong.
2014-11-19 8:58 GMT+09:00 sparkling [via Apache Spark User List] <
ml-node+s1001560n19233...@n3.nabble.com>:
> Hi,
>
> Are there any examples of using JdbcRDD in java available?
>
> Its not clear what is the last argument in this example (
> https://github.com/apache/spark/blob/master/core/src/test/scala/org/apache/spark/rdd/JdbcRDDSuite.scala
> ):
>
> sc = new SparkContext("local", "test")
> val rdd = new JdbcRDD(
> sc,
> () => { DriverManager.getConnection("jdbc:derby:target/JdbcRDDSuiteDb") },
> "SELECT DATA FROM FOO WHERE ? <= ID AND ID <= ?",
> 1, 100, 3,
> (r: ResultSet) => { r.getInt(1) }
> ).cache()
>
> Thanks
>
>
>
> ------------------------------
> If you reply to this email, your message will be added to the discussion
> below:
> http://apache-spark-user-list.1001560.n3.nabble.com/JdbcRDD-tp19233.html
> To unsubscribe from Apache Spark User List, click here
> <http://apache-spark-user-list.1001560.n3.nabble.com/template/NamlServlet.jtp?macro=unsubscribe_by_code&node=1&code=bXlraWRvbmdAZ21haWwuY29tfDF8LTU1MTIyOTc5NQ==>
> .
> NAML
> <http://apache-spark-user-list.1001560.n3.nabble.com/template/NamlServlet.jtp?macro=macro_viewer&id=instant_html%21nabble%3Aemail.naml&base=nabble.naml.namespaces.BasicNamespace-nabble.view.web.template.NabbleNamespace-nabble.view.web.template.NodeNamespace&breadcrumbs=notify_subscribers%21nabble%3Aemail.naml-instant_emails%21nabble%3Aemail.naml-send_instant_email%21nabble%3Aemail.naml>
>
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/JdbcRDD-tp19233p19235.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
