[ https://issues.apache.org/jira/browse/HUDI-4971?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Raymond Xu updated HUDI-4971: ----------------------------- Description: Putting aws bundle first before spark bundle will cause class loading issue {code} Caused by: java.lang.ClassNotFoundException: org.apache.hudi.org.objenesis.strategy.InstantiatorStrategy at java.net.URLClassLoader.findClass(URLClassLoader.java:387) at java.lang.ClassLoader.loadClass(ClassLoader.java:418) at java.lang.ClassLoader.loadClass(ClassLoader.java:351) {code} Reprod with quickstart example. {code:bash} spark-shell \ --jars /tmp/spark31/hudi-aws-bundle-0.12.1-rc1.jar,/tmp/spark31/hudi-spark3.1-bundle_2.12-0.12.1-rc1.jar \ --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' \ --conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' {code} {code:scala} // spark-shell import org.apache.hudi.QuickstartUtils._ import scala.collection.JavaConversions._ import org.apache.spark.sql.SaveMode._ import org.apache.hudi.DataSourceReadOptions._ import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.config.HoodieWriteConfig._ import org.apache.hudi.common.model.HoodieRecord val tableName = "hudi_trips_cow" val basePath = "file:///tmp/hudi_trips_cow" val dataGen = new DataGenerator val snapshotQuery = "SELECT begin_lat, begin_lon, driver, end_lat, end_lon, fare, partitionpath, rider, ts, uuid FROM hudi_ro_table" val inserts = convertToStringList(dataGen.generateInserts(10)) val df = spark.read.json(spark.sparkContext.parallelize(inserts, 2)) df.write.format("hudi"). options(getQuickstartWriteConfigs). option(PRECOMBINE_FIELD_OPT_KEY, "ts"). option(RECORDKEY_FIELD_OPT_KEY, "uuid"). option(PARTITIONPATH_FIELD_OPT_KEY, "partitionpath"). option(TABLE_NAME, tableName). mode(Overwrite). save(basePath) {code} was: Putting aws bundle first before spark bundle will cause class loading issue {code} Caused by: java.lang.ClassNotFoundException: org.apache.hudi.org.objenesis.strategy.InstantiatorStrategy at java.net.URLClassLoader.findClass(URLClassLoader.java:387) at java.lang.ClassLoader.loadClass(ClassLoader.java:418) at java.lang.ClassLoader.loadClass(ClassLoader.java:351) {code} Reprod with quickstart example. {code:shell} spark-shell \ --jars /tmp/spark31/hudi-aws-bundle-0.12.1-rc1.jar,/tmp/spark31/hudi-spark3.1-bundle_2.12-0.12.1-rc1.jar \ --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' \ --conf 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' {code} {code:scala} // spark-shell import org.apache.hudi.QuickstartUtils._ import scala.collection.JavaConversions._ import org.apache.spark.sql.SaveMode._ import org.apache.hudi.DataSourceReadOptions._ import org.apache.hudi.DataSourceWriteOptions._ import org.apache.hudi.config.HoodieWriteConfig._ import org.apache.hudi.common.model.HoodieRecord val tableName = "hudi_trips_cow" val basePath = "file:///tmp/hudi_trips_cow" val dataGen = new DataGenerator val snapshotQuery = "SELECT begin_lat, begin_lon, driver, end_lat, end_lon, fare, partitionpath, rider, ts, uuid FROM hudi_ro_table" val inserts = convertToStringList(dataGen.generateInserts(10)) val df = spark.read.json(spark.sparkContext.parallelize(inserts, 2)) df.write.format("hudi"). options(getQuickstartWriteConfigs). option(PRECOMBINE_FIELD_OPT_KEY, "ts"). option(RECORDKEY_FIELD_OPT_KEY, "uuid"). option(PARTITIONPATH_FIELD_OPT_KEY, "partitionpath"). option(TABLE_NAME, tableName). mode(Overwrite). save(basePath) {code} > aws bundle causes class loading issue > ------------------------------------- > > Key: HUDI-4971 > URL: https://issues.apache.org/jira/browse/HUDI-4971 > Project: Apache Hudi > Issue Type: Bug > Components: dependencies > Reporter: Raymond Xu > Priority: Critical > Fix For: 0.12.2 > > > Putting aws bundle first before spark bundle will cause class loading issue > {code} > Caused by: java.lang.ClassNotFoundException: > org.apache.hudi.org.objenesis.strategy.InstantiatorStrategy > at java.net.URLClassLoader.findClass(URLClassLoader.java:387) > at java.lang.ClassLoader.loadClass(ClassLoader.java:418) > at java.lang.ClassLoader.loadClass(ClassLoader.java:351) > {code} > Reprod with quickstart example. > {code:bash} > spark-shell \ > --jars > /tmp/spark31/hudi-aws-bundle-0.12.1-rc1.jar,/tmp/spark31/hudi-spark3.1-bundle_2.12-0.12.1-rc1.jar > \ > --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' \ > --conf > 'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' > {code} > {code:scala} > // spark-shell > import org.apache.hudi.QuickstartUtils._ > import scala.collection.JavaConversions._ > import org.apache.spark.sql.SaveMode._ > import org.apache.hudi.DataSourceReadOptions._ > import org.apache.hudi.DataSourceWriteOptions._ > import org.apache.hudi.config.HoodieWriteConfig._ > import org.apache.hudi.common.model.HoodieRecord > val tableName = "hudi_trips_cow" > val basePath = "file:///tmp/hudi_trips_cow" > val dataGen = new DataGenerator > val snapshotQuery = "SELECT begin_lat, begin_lon, driver, end_lat, end_lon, > fare, partitionpath, rider, ts, uuid FROM hudi_ro_table" > val inserts = convertToStringList(dataGen.generateInserts(10)) > val df = spark.read.json(spark.sparkContext.parallelize(inserts, 2)) > df.write.format("hudi"). > options(getQuickstartWriteConfigs). > option(PRECOMBINE_FIELD_OPT_KEY, "ts"). > option(RECORDKEY_FIELD_OPT_KEY, "uuid"). > option(PARTITIONPATH_FIELD_OPT_KEY, "partitionpath"). > option(TABLE_NAME, tableName). > mode(Overwrite). > save(basePath) > {code} -- This message was sent by Atlassian Jira (v8.20.10#820010)