?????? Pyflink jdbc????

???? Tue, 01 Jun 2021 02:31:10 -0700

????????????2.11??????????????????


------------------&nbsp;????????&nbsp;------------------
??????:                                                                         
                                               "user-zh"                        
                                                            
<dian0511...@gmail.com&gt;;
????????:&nbsp;2021??6??1??(??????) ????5:04
??????:&nbsp;"user-zh"<user-zh@flink.apache.org&gt;;"????"<1129656...@qq.com&gt;;

????:&nbsp;Re: Pyflink jdbc????



Hi??

??????????
1????flink-connector-jdbc_2.11-1.13.1.jar??????????PyFlink??????????JAR????scala
 2.11??


flink run:
1) 
????????sink????????????print??????????table_sink??????????SQL??????????table_sink??


&gt; 2021??6??1?? ????4:33?????? <1129656...@qq.com&gt; ??????
&gt; 
&gt; Hi??
&gt; &amp;nbsp; 
&amp;nbsp;??????????????https://mp.weixin.qq.com/s/GyFTjQl6ch8jc733mpCP7Q 
????????pyflink??????????????????????jdbc??jar??????????????????jdbc??????????flink??????1.13.1
&gt; ????????????????
&gt; 
&gt; 
&gt; from pyflink.datastream import StreamExecutionEnvironment
&gt; from pyflink.table import StreamTableEnvironment, EnvironmentSettings
&gt; env = StreamExecutionEnvironment.get_execution_environment()
&gt; table_env = StreamTableEnvironment.create( 
env,environment_settings=EnvironmentSettings.new_instance().use_blink_planner().build())
&gt; 
table_env.get_config().get_configuration().set_string("pipeline.classpaths", 
"file:///home/flink/lib/flink-connector-jdbc_2.12-1.13.1.jar;file:///home/flink/lib/mysql-connector-java-8.0.25.jar")
&gt; 
&gt; 
&gt; # 2. create source Table
&gt; table_env.execute_sql("""
&gt; 
&gt; 
&gt; CREATE TABLE table_source (
&gt; &amp;nbsp; e string
&gt; ) WITH (
&gt; &amp;nbsp;'connector' = 'jdbc',
&gt; &amp;nbsp; 'url' = 'jdbc:mysql://********:3306/test',
&gt; &amp;nbsp; 'driver' = 'com.mysql.cj.jdbc.Driver',
&gt; &amp;nbsp; 'table-name' = 'enum_test',
&gt; &amp;nbsp; 'username' = 'pms_etl',
&gt; &amp;nbsp; 'password' = 'pms_etl_q'
&gt; )
&gt; 
&gt; 
&gt; """)
&gt; 
&gt; 
&gt; # 3. create sink Table
&gt; table_env.execute_sql("""
&gt; &amp;nbsp; &amp;nbsp; CREATE TABLE print (
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; e string
&gt; &amp;nbsp; &amp;nbsp; ) WITH (
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; 'connector' = 'print'
&gt; &amp;nbsp; &amp;nbsp; )
&gt; """)
&gt; &amp;nbsp; &amp;nbsp;
&gt; 
&gt; 
&gt; table_env.execute_sql("INSERT INTO table_sink SELECT * FROM 
table_source").wait()
&gt; 
&gt; 
&gt; 
&gt; ????????python????????????????????
&gt; 
&gt; 
&gt; Traceback (most recent call last):
&gt; &amp;nbsp; File "demo.py", line 41, in <module&amp;gt;
&gt; &amp;nbsp; &amp;nbsp; table_env.execute_sql("INSERT INTO table_sink SELECT 
* FROM table_source").wait()
&gt; &amp;nbsp; File 
"/usr/local/lib/python3.7/site-packages/pyflink/table/table_environment.py", 
line 804, in execute_sql
&gt; &amp;nbsp; &amp;nbsp; return TableResult(self._j_tenv.executeSql(stmt))
&gt; &amp;nbsp; File 
"/usr/local/lib/python3.7/site-packages/py4j/java_gateway.py", line 1286, in 
__call__
&gt; &amp;nbsp; &amp;nbsp; answer, self.gateway_client, self.target_id, 
self.name)
&gt; &amp;nbsp; File 
"/usr/local/lib/python3.7/site-packages/pyflink/util/exceptions.py", line 146, 
in deco
&gt; &amp;nbsp; &amp;nbsp; return f(*a, **kw)
&gt; &amp;nbsp; File "/usr/local/lib/python3.7/site-packages/py4j/protocol.py", 
line 328, in get_return_value
&gt; &amp;nbsp; &amp;nbsp; format(target_id, ".", name), value)
&gt; py4j.protocol.Py4JJavaError: An error occurred while calling o4.executeSql.
&gt; : org.apache.flink.table.api.ValidationException: Unable to create a 
source for reading table 'default_catalog.default_database.table_source'.
&gt; 
&gt; 
&gt; Table options are:
&gt; 
&gt; 
&gt; 'connector'='jdbc'
&gt; 'driver'='com.mysql.cj.jdbc.Driver'
&gt; 'password'='pms_etl_q'
&gt; 'table-name'='enum_test'
&gt; 'url'='jdbc:mysql://*******:3306/test'
&gt; 'username'='pms_etl'
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.factories.FactoryUtil.createTableSource(FactoryUtil.java:137)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.plan.schema.CatalogSourceTable.createDynamicTableSource(CatalogSourceTable.java:116)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.plan.schema.CatalogSourceTable.toRel(CatalogSourceTable.java:82)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.calcite.sql2rel.SqlToRelConverter.toRel(SqlToRelConverter.java:3585)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.calcite.sql2rel.SqlToRelConverter.convertIdentifier(SqlToRelConverter.java:2507)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.calcite.sql2rel.SqlToRelConverter.convertFrom(SqlToRelConverter.java:2144)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.calcite.sql2rel.SqlToRelConverter.convertFrom(SqlToRelConverter.java:2093)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.calcite.sql2rel.SqlToRelConverter.convertFrom(SqlToRelConverter.java:2050)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.calcite.sql2rel.SqlToRelConverter.convertSelectImpl(SqlToRelConverter.java:663)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.calcite.sql2rel.SqlToRelConverter.convertSelect(SqlToRelConverter.java:644)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.calcite.sql2rel.SqlToRelConverter.convertQueryRecursive(SqlToRelConverter.java:3438)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.calcite.sql2rel.SqlToRelConverter.convertQuery(SqlToRelConverter.java:570)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.calcite.FlinkPlannerImpl.org$apache$flink$table$planner$calcite$FlinkPlannerImpl$$rel(FlinkPlannerImpl.scala:170)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.calcite.FlinkPlannerImpl.rel(FlinkPlannerImpl.scala:162)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.operations.SqlToOperationConverter.toQueryOperation(SqlToOperationConverter.java:967)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.operations.SqlToOperationConverter.convertSqlQuery(SqlToOperationConverter.java:936)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.operations.SqlToOperationConverter.convert(SqlToOperationConverter.java:275)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.operations.SqlToOperationConverter.convertSqlInsert(SqlToOperationConverter.java:595)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.operations.SqlToOperationConverter.convert(SqlToOperationConverter.java:268)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:101)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:724)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
java.lang.reflect.Method.invoke(Method.java:498)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.Gateway.invoke(Gateway.java:282)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.commands.CallCommand.execute(CallCommand.java:79)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.GatewayConnection.run(GatewayConnection.java:238)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
java.lang.Thread.run(Thread.java:748)
&gt; Caused by: org.apache.flink.table.api.ValidationException: Cannot discover 
a connector using option: 'connector'='jdbc'
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.factories.FactoryUtil.enrichNoMatchingConnectorError(FactoryUtil.java:467)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.factories.FactoryUtil.getDynamicTableFactory(FactoryUtil.java:441)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.factories.FactoryUtil.createTableSource(FactoryUtil.java:133)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; ... 31 more
&gt; Caused by: org.apache.flink.table.api.ValidationException: Could not find 
any factory for identifier 'jdbc' that implements 
'org.apache.flink.table.factories.DynamicTableFactory' in the classpath.
&gt; 
&gt; 
&gt; Available factory identifiers are:
&gt; 
&gt; 
&gt; blackhole
&gt; datagen
&gt; filesystem
&gt; print
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.factories.FactoryUtil.discoverFactory(FactoryUtil.java:319)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.factories.FactoryUtil.enrichNoMatchingConnectorError(FactoryUtil.java:463)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; ... 33 more
&gt; 
&gt; 
&gt; 
&gt; ????flink run -py demo.py ??????????????
&gt; 
&gt; 
&gt; &amp;nbsp; File "./demo.py", line 41, in <module&amp;gt;
&gt; &amp;nbsp; &amp;nbsp; table_env.execute_sql("INSERT INTO table_sink SELECT 
* FROM table_source").wait()
&gt; &amp;nbsp; File 
"/home/flink/opt/python/pyflink.zip/pyflink/table/table_environment.py", line 
804, in execute_sql
&gt; &amp;nbsp; File 
"/home/flink/opt/python/py4j-0.10.8.1-src.zip/py4j/java_gateway.py", line 1286, 
in __call__
&gt; &amp;nbsp; File 
"/home/flink/opt/python/pyflink.zip/pyflink/util/exceptions.py", line 158, in 
deco
&gt; pyflink.util.exceptions.TableException: 
org.apache.flink.table.api.TableException: Sink 
`default_catalog`.`default_database`.`table_sink` does not exists
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.delegation.PlannerBase.translateToRel(PlannerBase.scala:233)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.delegation.PlannerBase.$anonfun$translate$1(PlannerBase.scala:162)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:233)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.Iterator.foreach(Iterator.scala:937)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.Iterator.foreach$(Iterator.scala:937)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.AbstractIterator.foreach(Iterator.scala:1425)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.IterableLike.foreach(IterableLike.scala:70)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.IterableLike.foreach$(IterableLike.scala:69)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.AbstractIterable.foreach(Iterable.scala:54)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.TraversableLike.map(TraversableLike.scala:233)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.TraversableLike.map$(TraversableLike.scala:226)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
scala.collection.AbstractTraversable.map(Traversable.scala:104)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.planner.delegation.PlannerBase.translate(PlannerBase.scala:162)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.api.internal.TableEnvironmentImpl.translate(TableEnvironmentImpl.java:1518)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:740)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:856)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:730)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
java.lang.reflect.Method.invoke(Method.java:498)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.Gateway.invoke(Gateway.java:282)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.commands.CallCommand.execute(CallCommand.java:79)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.api.python.shaded.py4j.GatewayConnection.run(GatewayConnection.java:238)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
java.lang.Thread.run(Thread.java:748)
&gt; 
&gt; 
&gt; org.apache.flink.client.program.ProgramAbortException: 
java.lang.RuntimeException: Python process exits with code: 1
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.python.PythonDriver.main(PythonDriver.java:134)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
java.lang.reflect.Method.invoke(Method.java:498)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:355)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:222)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:114)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:812)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:246)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.cli.CliFrontend.parseAndRun(CliFrontend.java:1054)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:1132)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.runtime.security.contexts.NoOpSecurityContext.runSecured(NoOpSecurityContext.java:28)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1132)
&gt; Caused by: java.lang.RuntimeException: Python process exits with code: 1
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; at 
org.apache.flink.client.python.PythonDriver.main(PythonDriver.java:124)
&gt; &amp;nbsp; &amp;nbsp; &amp;nbsp; &amp;nbsp; ... 13 more
&gt; 
&gt; 
&gt; 
&gt; ??????????????????

?????? Pyflink jdbc????

回复