> On Nov. 18, 2014, 9:47 a.m., Abraham Elmahrek wrote: > > I have a couple of high level comments and an exception. > > > > It seems like the LinkConfiguration and the ToJobConfiguration are holding > > the wrong pieces? For example, the dataset URI should be part of the > > LinkConfiguration and the data format (Avro, CSV, etc.) should be part of > > the job? Or perhaps they should all be part of the link configuration? > > > > Also, I think there should be a validation step for the dataset for HDFS. > > We should check for "hdfs" type and require a host. It's fine for now for > > this to be a runtime exception, but let's log a separate Jira? > > > > Exception has occurred during processing command > > Exception: org.apache.sqoop.common.SqoopException Message: > > CLIENT_0001:Server has returned exception > > Stack trace: > > at org.apache.sqoop.client.request.ResourceRequest > > (ResourceRequest.java:115) > > at org.apache.sqoop.client.request.ResourceRequest > > (ResourceRequest.java:148) > > at org.apache.sqoop.client.request.JobResourceRequest > > (JobResourceRequest.java:63) > > at org.apache.sqoop.client.request.SqoopResourceRequests > > (SqoopResourceRequests.java:116) > > at org.apache.sqoop.client.SqoopClient (SqoopClient.java:406) > > at org.apache.sqoop.shell.CreateJobFunction > > (CreateJobFunction.java:99) > > at org.apache.sqoop.shell.CreateJobFunction > > (CreateJobFunction.java:64) > > at org.apache.sqoop.shell.SqoopFunction (SqoopFunction.java:51) > > at org.apache.sqoop.shell.SqoopCommand (SqoopCommand.java:127) > > at org.apache.sqoop.shell.SqoopCommand (SqoopCommand.java:103) > > at org.codehaus.groovy.tools.shell.Command$execute (null:-1) > > at org.codehaus.groovy.tools.shell.Shell (Shell.groovy:101) > > at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:-1) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:-2) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:57) > > at sun.reflect.DelegatingMethodAccessorImpl > > (DelegatingMethodAccessorImpl.java:43) > > at java.lang.reflect.Method (Method.java:606) > > at org.codehaus.groovy.reflection.CachedMethod > > (CachedMethod.java:90) > > at groovy.lang.MetaMethod (MetaMethod.java:233) > > at groovy.lang.MetaClassImpl (MetaClassImpl.java:1054) > > at org.codehaus.groovy.runtime.ScriptBytecodeAdapter > > (ScriptBytecodeAdapter.java:128) > > at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:173) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:-2) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:57) > > at sun.reflect.DelegatingMethodAccessorImpl > > (DelegatingMethodAccessorImpl.java:43) > > at java.lang.reflect.Method (Method.java:606) > > at > > org.codehaus.groovy.runtime.callsite.PogoMetaMethodSite$PogoCachedMethodSiteNoUnwrapNoCoerce > > (PogoMetaMethodSite.java:267) > > at org.codehaus.groovy.runtime.callsite.PogoMetaMethodSite > > (PogoMetaMethodSite.java:52) > > at org.codehaus.groovy.runtime.callsite.AbstractCallSite > > (AbstractCallSite.java:141) > > at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:121) > > at org.codehaus.groovy.tools.shell.Shell (Shell.groovy:114) > > at org.codehaus.groovy.tools.shell.Shell$leftShift$0 (null:-1) > > at org.codehaus.groovy.tools.shell.ShellRunner > > (ShellRunner.groovy:88) > > at org.codehaus.groovy.tools.shell.InteractiveShellRunner > > (InteractiveShellRunner.groovy:-1) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:-2) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:57) > > at sun.reflect.DelegatingMethodAccessorImpl > > (DelegatingMethodAccessorImpl.java:43) > > at java.lang.reflect.Method (Method.java:606) > > at org.codehaus.groovy.reflection.CachedMethod > > (CachedMethod.java:90) > > at groovy.lang.MetaMethod (MetaMethod.java:233) > > at groovy.lang.MetaClassImpl (MetaClassImpl.java:1054) > > at org.codehaus.groovy.runtime.ScriptBytecodeAdapter > > (ScriptBytecodeAdapter.java:128) > > at org.codehaus.groovy.runtime.ScriptBytecodeAdapter > > (ScriptBytecodeAdapter.java:148) > > at org.codehaus.groovy.tools.shell.InteractiveShellRunner > > (InteractiveShellRunner.groovy:100) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:-2) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:57) > > at sun.reflect.DelegatingMethodAccessorImpl > > (DelegatingMethodAccessorImpl.java:43) > > at java.lang.reflect.Method (Method.java:606) > > at > > org.codehaus.groovy.runtime.callsite.PogoMetaMethodSite$PogoCachedMethodSiteNoUnwrapNoCoerce > > (PogoMetaMethodSite.java:267) > > at org.codehaus.groovy.runtime.callsite.PogoMetaMethodSite > > (PogoMetaMethodSite.java:52) > > at org.codehaus.groovy.runtime.callsite.AbstractCallSite > > (AbstractCallSite.java:137) > > at org.codehaus.groovy.tools.shell.ShellRunner > > (ShellRunner.groovy:57) > > at org.codehaus.groovy.tools.shell.InteractiveShellRunner > > (InteractiveShellRunner.groovy:-1) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:-2) > > at sun.reflect.NativeMethodAccessorImpl > > (NativeMethodAccessorImpl.java:57) > > at sun.reflect.DelegatingMethodAccessorImpl > > (DelegatingMethodAccessorImpl.java:43) > > at java.lang.reflect.Method (Method.java:606) > > at org.codehaus.groovy.reflection.CachedMethod > > (CachedMethod.java:90) > > at groovy.lang.MetaMethod (MetaMethod.java:233) > > at groovy.lang.MetaClassImpl (MetaClassImpl.java:1054) > > at org.codehaus.groovy.runtime.ScriptBytecodeAdapter > > (ScriptBytecodeAdapter.java:128) > > at org.codehaus.groovy.runtime.ScriptBytecodeAdapter > > (ScriptBytecodeAdapter.java:148) > > at org.codehaus.groovy.tools.shell.InteractiveShellRunner > > (InteractiveShellRunner.groovy:66) > > at java_lang_Runnable$run (null:-1) > > at org.codehaus.groovy.runtime.callsite.CallSiteArray > > (CallSiteArray.java:42) > > at org.codehaus.groovy.runtime.callsite.AbstractCallSite > > (AbstractCallSite.java:108) > > at org.codehaus.groovy.runtime.callsite.AbstractCallSite > > (AbstractCallSite.java:112) > > at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:463) > > at org.codehaus.groovy.tools.shell.Groovysh (Groovysh.groovy:402) > > at org.apache.sqoop.shell.SqoopShell (SqoopShell.java:128) > > Caused by: Exception: java.lang.NullPointerException Message: > > Stack trace: > > at java.util.regex.Matcher (Matcher.java:1234) > > at java.util.regex.Matcher (Matcher.java:308) > > at java.util.regex.Matcher (Matcher.java:228) > > at java.util.regex.Pattern (Pattern.java:1088) > > at org.apache.sqoop.connector.kite.util.InputValidation > > (InputValidation.java:35) > > at > > org.apache.sqoop.connector.kite.configuration.ToJobConfig$ConfigValidator > > (ToJobConfig.java:38) > > at > > org.apache.sqoop.connector.kite.configuration.ToJobConfig$ConfigValidator > > (ToJobConfig.java:33) > > at org.apache.sqoop.validation.ConfigValidationRunner > > (ConfigValidationRunner.java:172) > > at org.apache.sqoop.validation.ConfigValidationRunner > > (ConfigValidationRunner.java:140) > > at org.apache.sqoop.validation.ConfigValidationRunner > > (ConfigValidationRunner.java:121) > > at org.apache.sqoop.validation.ConfigValidationRunner > > (ConfigValidationRunner.java:82) > > at org.apache.sqoop.model.ConfigUtils (ConfigUtils.java:220) > > at org.apache.sqoop.handler.JobRequestHandler > > (JobRequestHandler.java:220) > > at org.apache.sqoop.handler.JobRequestHandler > > (JobRequestHandler.java:102) > > at org.apache.sqoop.server.v1.JobServlet (JobServlet.java:91) > > at org.apache.sqoop.server.SqoopProtocolServlet > > (SqoopProtocolServlet.java:63) > > at javax.servlet.http.HttpServlet (HttpServlet.java:643) > > at javax.servlet.http.HttpServlet (HttpServlet.java:723) > > at org.apache.catalina.core.ApplicationFilterChain > > (ApplicationFilterChain.java:290) > > at org.apache.catalina.core.ApplicationFilterChain > > (ApplicationFilterChain.java:206) > > at > > org.apache.hadoop.security.authentication.server.AuthenticationFilter > > (AuthenticationFilter.java:392) > > at org.apache.catalina.core.ApplicationFilterChain > > (ApplicationFilterChain.java:235) > > at org.apache.catalina.core.ApplicationFilterChain > > (ApplicationFilterChain.java:206) > > at org.apache.catalina.core.StandardWrapperValve > > (StandardWrapperValve.java:233) > > at org.apache.catalina.core.StandardContextValve > > (StandardContextValve.java:191) > > at org.apache.catalina.core.StandardHostValve > > (StandardHostValve.java:127) > > at org.apache.catalina.valves.ErrorReportValve > > (ErrorReportValve.java:103) > > at org.apache.catalina.core.StandardEngineValve > > (StandardEngineValve.java:109) > > at org.apache.catalina.connector.CoyoteAdapter > > (CoyoteAdapter.java:293) > > at org.apache.coyote.http11.Http11Processor > > (Http11Processor.java:861) > > at > > org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler > > (Http11Protocol.java:606) > > at org.apache.tomcat.util.net.JIoEndpoint$Worker > > (JIoEndpoint.java:489) > > at java.lang.Thread (Thread.java:744)
Added https://issues.apache.org/jira/browse/SQOOP-1751 - Qian ----------------------------------------------------------- This is an automatically generated e-mail. To reply, visit: https://reviews.apache.org/r/26963/#review61857 ----------------------------------------------------------- On Nov. 17, 2014, 3:31 p.m., Qian Xu wrote: > > ----------------------------------------------------------- > This is an automatically generated e-mail. To reply, visit: > https://reviews.apache.org/r/26963/ > ----------------------------------------------------------- > > (Updated Nov. 17, 2014, 3:31 p.m.) > > > Review request for Sqoop. > > > Bugs: SQOOP-1588 > https://issues.apache.org/jira/browse/SQOOP-1588 > > > Repository: sqoop-sqoop2 > > > Description > ------- > > Create a basic Kite connector that can write data (i.e. from a jdbc > connection) to HDFS. > > The scope is defined as follows: > * Destination: HDFS > * File Format: Avro Parquet and CSV. > * Compression Codec: Use default > * Partitioner Strategy: Not supported > * Column Mapping: Not supported > > > Diffs > ----- > > connector/connector-kite/pom.xml PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnectorConstants.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnectorError.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToDestroyer.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/LinkConfig.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/LinkConfiguration.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ToJobConfig.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ToJobConfiguration.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/InputValidation.java > PRE-CREATION > > connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java > PRE-CREATION > connector/connector-kite/src/main/resources/connector-configs.properties > PRE-CREATION > > connector/connector-kite/src/main/resources/kite-connector-config.properties > PRE-CREATION > connector/connector-kite/src/main/resources/sqoopconnector.properties > PRE-CREATION > > connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java > PRE-CREATION > > connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java > PRE-CREATION > > connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteToDestroyer.java > PRE-CREATION > > connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteToInitializer.java > PRE-CREATION > connector/connector-kite/src/test/resources/log4j.properties PRE-CREATION > > connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/FileFormat.java > PRE-CREATION > > connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/JarUtil.java > PRE-CREATION > connector/pom.xml e98a0fc > pom.xml 233d3ce > server/pom.xml 4a5eb5e > test/pom.xml 2dbb8c5 > > Diff: https://reviews.apache.org/r/26963/diff/ > > > Testing > ------- > > New unittests included. All passed. > > > Thanks, > > Qian Xu > >
