[
https://issues.apache.org/jira/browse/GOBBLIN-1059?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Chhaya Vankhede updated GOBBLIN-1059:
-------------------------------------
Docs Text: (was: I am tring to ingest data from mysql to hdfs but
getting below error. The job file and properties are as below
mysql.pull
```
# Job properties
job.name=GobblinMySql
job.group=MySql
job.description=Data pull from MySql
# Extract properties
extract.namespace=demodb
extract.table.type=snapshot_only
extract.table.name=user
# Property to consider the extract as full dump
extract.is.full=true
# Source properties
source.querybased.schema=user
source.entity=user
source.querybased.extract.type=snapshot
```
mysql.propertes
```
# Source properties - source class to extract data from Mysql Source
source.class=org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
# Source properties
source.max.number.of.partitions=1
source.querybased.partition.interval=1
source.querybased.is.compression=true
source.querybased.watermark.type=timestamp
# Source connection properties
source.conn.driver=com.mysql.jdbc.Driver
source.conn.username=root
source.conn.password=root
source.conn.host=localhost
source.conn.port=3306
source.conn.timeout=1500
# Converter properties - Record from mysql source will be processed by the
below series of converters
converter.classes=org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter
# date columns format
converter.avro.timestamp.format=yyyy-MM-dd HH:mm:ss'.0'
converter.avro.date.format=yyyy-MM-dd
converter.avro.time.format=HH:mm:ss
# Qualitychecker properties
qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy
qualitychecker.task.policy.types=OPTIONAL,OPTIONAL
# Publisher properties
data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher
```
below is error log
```
GMT ERROR [JobScheduler-0]
org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner 637 - Failed
to run job GobblinMySql
org.apache.gobblin.runtime.JobException: Failed to run job GobblinMySql
at
org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:445)
at
org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner.run(JobScheduler.java:635)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Failed to create job launcher:
java.lang.ClassNotFoundException:
org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:158)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:107)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:85)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:68)
at
org.apache.gobblin.scheduler.JobScheduler.buildJobLauncher(JobScheduler.java:450)
at
org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:443)
... 4 more
Caused by: java.lang.ClassNotFoundException:
org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:264)
at
org.apache.gobblin.runtime.JobContext.createSource(JobContext.java:240)
at org.apache.gobblin.runtime.JobContext.<init>(JobContext.java:172)
at
org.apache.gobblin.runtime.AbstractJobLauncher.<init>(AbstractJobLauncher.java:201)
at
org.apache.gobblin.runtime.local.LocalJobLauncher.<init>(LocalJobLauncher.java:86)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:144)
... 9 more
2020-02-25 08:02:45 GMT ERROR [JobScheduler-0]
org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner 637 - Failed
to run job GobblinMySql
org.apache.gobblin.runtime.JobException: Failed to run job GobblinMySql
at
org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:445)
at
org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner.run(JobScheduler.java:635)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Failed to create job launcher:
java.lang.ClassNotFoundException:
org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:158)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:107)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:85)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:68)
at
org.apache.gobblin.scheduler.JobScheduler.buildJobLauncher(JobScheduler.java:450)
at
org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:443)
... 4 more
Caused by: java.lang.ClassNotFoundException:
org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:264)
at
org.apache.gobblin.runtime.JobContext.createSource(JobContext.java:240)
at org.apache.gobblin.runtime.JobContext.<init>(JobContext.java:172)
at
org.apache.gobblin.runtime.AbstractJobLauncher.<init>(AbstractJobLauncher.java:201)
at
org.apache.gobblin.runtime.local.LocalJobLauncher.<init>(LocalJobLauncher.java:86)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:144)
... 9 more```
using command `bin/gobblin.sh service standalone start --jars
/mnt/C/Users/name/incubator-gobblin/build/gobblin-sql/libs/gobblin-sql-0.15.0.jar`
How to solve it?
)
Description:
I am tring to ingest data from mysql to hdfs but getting below error. The job
file and properties are as below
mysql.pull
````
#Job properties
job.name=GobblinMySql
job.group=MySql
job.description=Data pull from MySql
#Extract properties
extract.namespace=demodb
extract.table.type=snapshot_only
extract.table.name=user
#Property to consider the extract as full dump
extract.is.full=true
#Source properties
source.querybased.schema=user
source.entity=user
source.querybased.extract.type=snapshot
```
mysql.propertes
```
#Source properties - source class to extract data from Mysql Source
source.class=org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
#Source properties
source.max.number.of.partitions=1
source.querybased.partition.interval=1
source.querybased.is.compression=true
source.querybased.watermark.type=timestamp
#Source connection properties
source.conn.driver=com.mysql.jdbc.Driver
source.conn.username=root
source.conn.password=root
source.conn.host=localhost
source.conn.port=3306
source.conn.timeout=1500
#Converter properties - Record from mysql source will be processed by the below
series of converters
converter.classes=org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter
#date columns format
converter.avro.timestamp.format=yyyy-MM-dd HH:mm:ss'.0'
converter.avro.date.format=yyyy-MM-dd
converter.avro.time.format=HH:mm:ss
#Qualitychecker properties
qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy
qualitychecker.task.policy.types=OPTIONAL,OPTIONAL
#Publisher properties
data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher
below is error log
```
GMT ERROR [JobScheduler-0]
org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner 637 - Failed to
run job GobblinMySql
org.apache.gobblin.runtime.JobException: Failed to run job GobblinMySql
at org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:445)
at
org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner.run(JobScheduler.java:635)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Failed to create job launcher:
java.lang.ClassNotFoundException:
org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:158)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:107)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:85)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:68)
at
org.apache.gobblin.scheduler.JobScheduler.buildJobLauncher(JobScheduler.java:450)
at org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:443)
... 4 more
Caused by: java.lang.ClassNotFoundException:
org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:264)
at org.apache.gobblin.runtime.JobContext.createSource(JobContext.java:240)
at org.apache.gobblin.runtime.JobContext.<init>(JobContext.java:172)
at
org.apache.gobblin.runtime.AbstractJobLauncher.<init>(AbstractJobLauncher.java:201)
at
org.apache.gobblin.runtime.local.LocalJobLauncher.<init>(LocalJobLauncher.java:86)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:144)
... 9 more
2020-02-25 08:02:45 GMT ERROR [JobScheduler-0]
org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner 637 - Failed to
run job GobblinMySql
org.apache.gobblin.runtime.JobException: Failed to run job GobblinMySql
at org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:445)
at
org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner.run(JobScheduler.java:635)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Failed to create job launcher:
java.lang.ClassNotFoundException:
org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:158)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:107)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:85)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:68)
at
org.apache.gobblin.scheduler.JobScheduler.buildJobLauncher(JobScheduler.java:450)
at org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:443)
... 4 more
Caused by: java.lang.ClassNotFoundException:
org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:264)
at org.apache.gobblin.runtime.JobContext.createSource(JobContext.java:240)
at org.apache.gobblin.runtime.JobContext.<init>(JobContext.java:172)
at
org.apache.gobblin.runtime.AbstractJobLauncher.<init>(AbstractJobLauncher.java:201)
at
org.apache.gobblin.runtime.local.LocalJobLauncher.<init>(LocalJobLauncher.java:86)
at
org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:144)
... 9 more
```
using command `bin/gobblin.sh service standalone start --jars
/mnt/C/Users/name/incubator-gobblin/build/gobblin-sql/libs/gobblin-sql-0.15.0.jar`
How to solve it?
> MySQL to HDFS ingestion
> -----------------------
>
> Key: GOBBLIN-1059
> URL: https://issues.apache.org/jira/browse/GOBBLIN-1059
> Project: Apache Gobblin
> Issue Type: Bug
> Components: gobblin-sql
> Environment: linux subsystem, hadoop 2.3.0,
> Reporter: Chhaya Vankhede
> Assignee: Shirshanka Das
> Priority: Major
> Labels: beginner
>
> I am tring to ingest data from mysql to hdfs but getting below error. The
> job file and properties are as below
> mysql.pull
> ````
> #Job properties
> job.name=GobblinMySql
> job.group=MySql
> job.description=Data pull from MySql
> #Extract properties
> extract.namespace=demodb
> extract.table.type=snapshot_only
> extract.table.name=user
> #Property to consider the extract as full dump
> extract.is.full=true
> #Source properties
> source.querybased.schema=user
> source.entity=user
> source.querybased.extract.type=snapshot
> ```
> mysql.propertes
> ```
> #Source properties - source class to extract data from Mysql Source
> source.class=org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
> #Source properties
> source.max.number.of.partitions=1
> source.querybased.partition.interval=1
> source.querybased.is.compression=true
> source.querybased.watermark.type=timestamp
> #Source connection properties
> source.conn.driver=com.mysql.jdbc.Driver
> source.conn.username=root
> source.conn.password=root
> source.conn.host=localhost
> source.conn.port=3306
> source.conn.timeout=1500
> #Converter properties - Record from mysql source will be processed by the
> below series of converters
>
> converter.classes=org.apache.gobblin.converter.avro.JsonIntermediateToAvroConverter
> #date columns format
> converter.avro.timestamp.format=yyyy-MM-dd HH:mm:ss'.0'
> converter.avro.date.format=yyyy-MM-dd
> converter.avro.time.format=HH:mm:ss
> #Qualitychecker properties
>
> qualitychecker.task.policies=org.apache.gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy
> qualitychecker.task.policy.types=OPTIONAL,OPTIONAL
> #Publisher properties
> data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher
> below is error log
> ```
> GMT ERROR [JobScheduler-0]
> org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner 637 - Failed
> to run job GobblinMySql
> org.apache.gobblin.runtime.JobException: Failed to run job GobblinMySql
> at org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:445)
> at
> org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner.run(JobScheduler.java:635)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.RuntimeException: Failed to create job launcher:
> java.lang.ClassNotFoundException:
> org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:158)
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:107)
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:85)
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:68)
> at
> org.apache.gobblin.scheduler.JobScheduler.buildJobLauncher(JobScheduler.java:450)
> at org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:443)
> ... 4 more
> Caused by: java.lang.ClassNotFoundException:
> org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
> at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
> at java.lang.Class.forName0(Native Method)
> at java.lang.Class.forName(Class.java:264)
> at org.apache.gobblin.runtime.JobContext.createSource(JobContext.java:240)
> at org.apache.gobblin.runtime.JobContext.<init>(JobContext.java:172)
> at
> org.apache.gobblin.runtime.AbstractJobLauncher.<init>(AbstractJobLauncher.java:201)
> at
> org.apache.gobblin.runtime.local.LocalJobLauncher.<init>(LocalJobLauncher.java:86)
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:144)
> ... 9 more
> 2020-02-25 08:02:45 GMT ERROR [JobScheduler-0]
> org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner 637 - Failed
> to run job GobblinMySql
> org.apache.gobblin.runtime.JobException: Failed to run job GobblinMySql
> at org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:445)
> at
> org.apache.gobblin.scheduler.JobScheduler$NonScheduledJobRunner.run(JobScheduler.java:635)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.RuntimeException: Failed to create job launcher:
> java.lang.ClassNotFoundException:
> org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:158)
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:107)
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:85)
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:68)
> at
> org.apache.gobblin.scheduler.JobScheduler.buildJobLauncher(JobScheduler.java:450)
> at org.apache.gobblin.scheduler.JobScheduler.runJob(JobScheduler.java:443)
> ... 4 more
> Caused by: java.lang.ClassNotFoundException:
> org.apache.gobblin.source.extractor.extract.jdbc.MysqlSource
> at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
> at java.lang.Class.forName0(Native Method)
> at java.lang.Class.forName(Class.java:264)
> at org.apache.gobblin.runtime.JobContext.createSource(JobContext.java:240)
> at org.apache.gobblin.runtime.JobContext.<init>(JobContext.java:172)
> at
> org.apache.gobblin.runtime.AbstractJobLauncher.<init>(AbstractJobLauncher.java:201)
> at
> org.apache.gobblin.runtime.local.LocalJobLauncher.<init>(LocalJobLauncher.java:86)
> at
> org.apache.gobblin.runtime.JobLauncherFactory.newJobLauncher(JobLauncherFactory.java:144)
> ... 9 more
> ```
> using command `bin/gobblin.sh service standalone start --jars
> /mnt/C/Users/name/incubator-gobblin/build/gobblin-sql/libs/gobblin-sql-0.15.0.jar`
>
> How to solve it?
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
