[
https://issues.apache.org/jira/browse/FLINK-22443?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17349752#comment-17349752
]
macdoor615 commented on FLINK-22443:
------------------------------------
I think your source table schema is okay.
Here is mine.
{code:java}
drop table if exists raw_restapi_load.p_hcd;
create table raw_restapi_load.p_hcd
(
gwid string,
id string,
performanceID string,
productName string,
businessPlatform string,
hostName string,
hostIp string,
statisticPeriod string,
beginTime timestamp,
endTime timestamp,
indicatorObject string,
indicatorDimension string,
indicatorLevel string,
indicatorNumber string,
content string,
indicatorValue string,
indicatorUnit string,
indicatorNumeratorValue string,
indicatorNumeratorUnit string,
indicatorDenominatorValue string,
indicatorDenominatorUnit string,
coltime timestamp,
crtime timestamp
)
comment ''
PARTITIONED BY (dt string)
STORED AS orc
TBLPROPERTIES (
'partition.time-extractor.timestamp-pattern'='$dt 00:00:00',
'sink.rolling-policy.rollover-interval'='15 min',
'sink.partition-commit.trigger'='process-time',
'sink.partition-commit.delay'='0 min',
'sink.partition-commit.policy.kind'='metastore,success-file'
);
{code}
> can not be execute an extreme long sql under batch mode
> -------------------------------------------------------
>
> Key: FLINK-22443
> URL: https://issues.apache.org/jira/browse/FLINK-22443
> Project: Flink
> Issue Type: Bug
> Components: Table SQL / Runtime
> Affects Versions: 1.12.2
> Environment: execute command
>
> {code:java}
> bin/sql-client.sh embedded -d conf/sql-client-batch.yaml
> {code}
> content of conf/sql-client-batch.yaml
>
> {code:java}
> catalogs:
> - name: bnpmphive
> type: hive
> hive-conf-dir: /home/gum/hive/conf
> hive-version: 3.1.2
> execution:
> planner: blink
> type: batch
> #type: streaming
> result-mode: table
> parallelism: 4
> max-parallelism: 2000
> current-catalog: bnpmphive
> #current-database: snmpprobe
> #configuration:
> # table.sql-dialect: hivemodules:
> - name: core
> type: core
> - name: myhive
> type: hivedeployment:
> # general cluster communication timeout in ms
> response-timeout: 5000
> # (optional) address from cluster to gateway
> gateway-address: ""
> # (optional) port from cluster to gateway
> gateway-port: 0
> {code}
>
> Reporter: macdoor615
> Priority: Blocker
> Labels: stale-blocker, stale-critical
> Attachments: flink-gum-taskexecutor-8-hb3-prod-hadoop-002.log.4.zip
>
>
> 1. execute an extreme long sql under batch mode
>
> {code:java}
> select
> 'CD' product_name,
> r.code business_platform,
> 5 statisticperiod,
> cast('2021-03-24 00:00:00' as timestamp) coltime,
> cast(r1.indicatorvalue as double) as YWPT_ZHQI_CD_038_GZ_00002,
> cast(r2.indicatorvalue as double) as YWPT_ZHQI_CD_038_YW_00007,
> cast(r3.indicatorvalue as double) as YWPT_ZHQI_CD_038_YW_00005,
> cast(r4.indicatorvalue as double) as YWPT_ZHQI_CD_038_YW_00006,
> cast(r5.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00029,
> cast(r6.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00028,
> cast(r7.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00015,
> cast(r8.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00014,
> cast(r9.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00011,
> cast(r10.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00010,
> cast(r11.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00013,
> cast(r12.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00012,
> cast(r13.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00027,
> cast(r14.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00026,
> cast(r15.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00046,
> cast(r16.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00047,
> cast(r17.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00049,
> cast(r18.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00048,
> cast(r19.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00024,
> cast(r20.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00025,
> cast(r21.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00022,
> cast(r22.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00023,
> cast(r23.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00054,
> cast(r24.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00055,
> cast(r25.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00033,
> cast(r26.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00032,
> cast(r27.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00053,
> cast(r28.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00052,
> cast(r29.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00051,
> cast(r30.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00050,
> cast(r31.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00043,
> cast(r32.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00042,
> cast(r33.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00017,
> cast(r34.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00016,
> cast(r35.indicatorvalue as double) as YWPT_ZHQI_CD_038_GZ_00003,
> cast(r36.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00045,
> cast(r37.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00044,
> cast(r38.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00038,
> cast(r39.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00039,
> cast(r40.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00037,
> cast(r41.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00036,
> cast(r42.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00040,
> cast(r43.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00041,
> cast(r44.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00034,
> cast(r45.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00035,
> cast(r46.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00030,
> cast(r47.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00031,
> cast(r48.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00020,
> cast(r49.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00021,
> cast(r50.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00018,
> cast(r51.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00019,
> cast(r52.indicatorvalue as double) as YWPT_ZHQI_CD_038_YW_00004,
> cast(r53.indicatorvalue as double) as YWPT_ZHQI_CD_038_YW_00008,
> cast(r54.indicatorvalue as double) as YWPT_ZHQI_CD_038_XT_00061,
> cast(r55.indicatorvalue as double) as YWPT_ZHQI_CD_038_YW_00009,
> localtimestamp as crtime,
> '2021-03-24' as dt
> from prod_mysql_bnpmp.r_biz_product r
> left join raw_restapi_load.p_hcd r1 on r1.dt='2021-03-24' and r1.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r1.businessplatform=r.code and
> r1.indicatornumber='YWPT-ZHQI-CD-038-GZ-00002'
> left join raw_restapi_load.p_hcd r2 on r2.dt='2021-03-24' and r2.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r2.businessplatform=r.code and
> r2.indicatornumber='YWPT-ZHQI-CD-038-YW-00007'
> left join raw_restapi_load.p_hcd r3 on r3.dt='2021-03-24' and r3.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r3.businessplatform=r.code and
> r3.indicatornumber='YWPT-ZHQI-CD-038-YW-00005'
> left join raw_restapi_load.p_hcd r4 on r4.dt='2021-03-24' and r4.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r4.businessplatform=r.code and
> r4.indicatornumber='YWPT-ZHQI-CD-038-YW-00006'
> left join raw_restapi_load.p_hcd r5 on r5.dt='2021-03-24' and r5.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r5.businessplatform=r.code and
> r5.indicatornumber='YWPT-ZHQI-CD-038-XT-00029'
> left join raw_restapi_load.p_hcd r6 on r6.dt='2021-03-24' and r6.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r6.businessplatform=r.code and
> r6.indicatornumber='YWPT-ZHQI-CD-038-XT-00028'
> left join raw_restapi_load.p_hcd r7 on r7.dt='2021-03-24' and r7.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r7.businessplatform=r.code and
> r7.indicatornumber='YWPT-ZHQI-CD-038-XT-00015'
> left join raw_restapi_load.p_hcd r8 on r8.dt='2021-03-24' and r8.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r8.businessplatform=r.code and
> r8.indicatornumber='YWPT-ZHQI-CD-038-XT-00014'
> left join raw_restapi_load.p_hcd r9 on r9.dt='2021-03-24' and r9.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r9.businessplatform=r.code and
> r9.indicatornumber='YWPT-ZHQI-CD-038-XT-00011'
> left join raw_restapi_load.p_hcd r10 on r10.dt='2021-03-24' and r10.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r10.businessplatform=r.code and
> r10.indicatornumber='YWPT-ZHQI-CD-038-XT-00010'
> left join raw_restapi_load.p_hcd r11 on r11.dt='2021-03-24' and r11.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r11.businessplatform=r.code and
> r11.indicatornumber='YWPT-ZHQI-CD-038-XT-00013'
> left join raw_restapi_load.p_hcd r12 on r12.dt='2021-03-24' and r12.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r12.businessplatform=r.code and
> r12.indicatornumber='YWPT-ZHQI-CD-038-XT-00012'
> left join raw_restapi_load.p_hcd r13 on r13.dt='2021-03-24' and r13.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r13.businessplatform=r.code and
> r13.indicatornumber='YWPT-ZHQI-CD-038-XT-00027'
> left join raw_restapi_load.p_hcd r14 on r14.dt='2021-03-24' and r14.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r14.businessplatform=r.code and
> r14.indicatornumber='YWPT-ZHQI-CD-038-XT-00026'
> left join raw_restapi_load.p_hcd r15 on r15.dt='2021-03-24' and r15.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r15.businessplatform=r.code and
> r15.indicatornumber='YWPT-ZHQI-CD-038-XT-00046'
> left join raw_restapi_load.p_hcd r16 on r16.dt='2021-03-24' and r16.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r16.businessplatform=r.code and
> r16.indicatornumber='YWPT-ZHQI-CD-038-XT-00047'
> left join raw_restapi_load.p_hcd r17 on r17.dt='2021-03-24' and r17.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r17.businessplatform=r.code and
> r17.indicatornumber='YWPT-ZHQI-CD-038-XT-00049'
> left join raw_restapi_load.p_hcd r18 on r18.dt='2021-03-24' and r18.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r18.businessplatform=r.code and
> r18.indicatornumber='YWPT-ZHQI-CD-038-XT-00048'
> left join raw_restapi_load.p_hcd r19 on r19.dt='2021-03-24' and r19.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r19.businessplatform=r.code and
> r19.indicatornumber='YWPT-ZHQI-CD-038-XT-00024'
> left join raw_restapi_load.p_hcd r20 on r20.dt='2021-03-24' and r20.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r20.businessplatform=r.code and
> r20.indicatornumber='YWPT-ZHQI-CD-038-XT-00025'
> left join raw_restapi_load.p_hcd r21 on r21.dt='2021-03-24' and r21.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r21.businessplatform=r.code and
> r21.indicatornumber='YWPT-ZHQI-CD-038-XT-00022'
> left join raw_restapi_load.p_hcd r22 on r22.dt='2021-03-24' and r22.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r22.businessplatform=r.code and
> r22.indicatornumber='YWPT-ZHQI-CD-038-XT-00023'
> left join raw_restapi_load.p_hcd r23 on r23.dt='2021-03-24' and r23.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r23.businessplatform=r.code and
> r23.indicatornumber='YWPT-ZHQI-CD-038-XT-00054'
> left join raw_restapi_load.p_hcd r24 on r24.dt='2021-03-24' and r24.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r24.businessplatform=r.code and
> r24.indicatornumber='YWPT-ZHQI-CD-038-XT-00055'
> left join raw_restapi_load.p_hcd r25 on r25.dt='2021-03-24' and r25.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r25.businessplatform=r.code and
> r25.indicatornumber='YWPT-ZHQI-CD-038-XT-00033'
> left join raw_restapi_load.p_hcd r26 on r26.dt='2021-03-24' and r26.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r26.businessplatform=r.code and
> r26.indicatornumber='YWPT-ZHQI-CD-038-XT-00032'
> left join raw_restapi_load.p_hcd r27 on r27.dt='2021-03-24' and r27.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r27.businessplatform=r.code and
> r27.indicatornumber='YWPT-ZHQI-CD-038-XT-00053'
> left join raw_restapi_load.p_hcd r28 on r28.dt='2021-03-24' and r28.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r28.businessplatform=r.code and
> r28.indicatornumber='YWPT-ZHQI-CD-038-XT-00052'
> left join raw_restapi_load.p_hcd r29 on r29.dt='2021-03-24' and r29.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r29.businessplatform=r.code and
> r29.indicatornumber='YWPT-ZHQI-CD-038-XT-00051'
> left join raw_restapi_load.p_hcd r30 on r30.dt='2021-03-24' and r30.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r30.businessplatform=r.code and
> r30.indicatornumber='YWPT-ZHQI-CD-038-XT-00050'
> left join raw_restapi_load.p_hcd r31 on r31.dt='2021-03-24' and r31.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r31.businessplatform=r.code and
> r31.indicatornumber='YWPT-ZHQI-CD-038-XT-00043'
> left join raw_restapi_load.p_hcd r32 on r32.dt='2021-03-24' and r32.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r32.businessplatform=r.code and
> r32.indicatornumber='YWPT-ZHQI-CD-038-XT-00042'
> left join raw_restapi_load.p_hcd r33 on r33.dt='2021-03-24' and r33.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r33.businessplatform=r.code and
> r33.indicatornumber='YWPT-ZHQI-CD-038-XT-00017'
> left join raw_restapi_load.p_hcd r34 on r34.dt='2021-03-24' and r34.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r34.businessplatform=r.code and
> r34.indicatornumber='YWPT-ZHQI-CD-038-XT-00016'
> left join raw_restapi_load.p_hcd r35 on r35.dt='2021-03-24' and r35.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r35.businessplatform=r.code and
> r35.indicatornumber='YWPT-ZHQI-CD-038-GZ-00003'
> left join raw_restapi_load.p_hcd r36 on r36.dt='2021-03-24' and r36.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r36.businessplatform=r.code and
> r36.indicatornumber='YWPT-ZHQI-CD-038-XT-00045'
> left join raw_restapi_load.p_hcd r37 on r37.dt='2021-03-24' and r37.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r37.businessplatform=r.code and
> r37.indicatornumber='YWPT-ZHQI-CD-038-XT-00044'
> left join raw_restapi_load.p_hcd r38 on r38.dt='2021-03-24' and r38.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r38.businessplatform=r.code and
> r38.indicatornumber='YWPT-ZHQI-CD-038-XT-00038'
> left join raw_restapi_load.p_hcd r39 on r39.dt='2021-03-24' and r39.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r39.businessplatform=r.code and
> r39.indicatornumber='YWPT-ZHQI-CD-038-XT-00039'
> left join raw_restapi_load.p_hcd r40 on r40.dt='2021-03-24' and r40.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r40.businessplatform=r.code and
> r40.indicatornumber='YWPT-ZHQI-CD-038-XT-00037'
> left join raw_restapi_load.p_hcd r41 on r41.dt='2021-03-24' and r41.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r41.businessplatform=r.code and
> r41.indicatornumber='YWPT-ZHQI-CD-038-XT-00036'
> left join raw_restapi_load.p_hcd r42 on r42.dt='2021-03-24' and r42.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r42.businessplatform=r.code and
> r42.indicatornumber='YWPT-ZHQI-CD-038-XT-00040'
> left join raw_restapi_load.p_hcd r43 on r43.dt='2021-03-24' and r43.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r43.businessplatform=r.code and
> r43.indicatornumber='YWPT-ZHQI-CD-038-XT-00041'
> left join raw_restapi_load.p_hcd r44 on r44.dt='2021-03-24' and r44.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r44.businessplatform=r.code and
> r44.indicatornumber='YWPT-ZHQI-CD-038-XT-00034'
> left join raw_restapi_load.p_hcd r45 on r45.dt='2021-03-24' and r45.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r45.businessplatform=r.code and
> r45.indicatornumber='YWPT-ZHQI-CD-038-XT-00035'
> left join raw_restapi_load.p_hcd r46 on r46.dt='2021-03-24' and r46.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r46.businessplatform=r.code and
> r46.indicatornumber='YWPT-ZHQI-CD-038-XT-00030'
> left join raw_restapi_load.p_hcd r47 on r47.dt='2021-03-24' and r47.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r47.businessplatform=r.code and
> r47.indicatornumber='YWPT-ZHQI-CD-038-XT-00031'
> left join raw_restapi_load.p_hcd r48 on r48.dt='2021-03-24' and r48.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r48.businessplatform=r.code and
> r48.indicatornumber='YWPT-ZHQI-CD-038-XT-00020'
> left join raw_restapi_load.p_hcd r49 on r49.dt='2021-03-24' and r49.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r49.businessplatform=r.code and
> r49.indicatornumber='YWPT-ZHQI-CD-038-XT-00021'
> left join raw_restapi_load.p_hcd r50 on r50.dt='2021-03-24' and r50.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r50.businessplatform=r.code and
> r50.indicatornumber='YWPT-ZHQI-CD-038-XT-00018'
> left join raw_restapi_load.p_hcd r51 on r51.dt='2021-03-24' and r51.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r51.businessplatform=r.code and
> r51.indicatornumber='YWPT-ZHQI-CD-038-XT-00019'
> left join raw_restapi_load.p_hcd r52 on r52.dt='2021-03-24' and r52.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r52.businessplatform=r.code and
> r52.indicatornumber='YWPT-ZHQI-CD-038-YW-00004'
> left join raw_restapi_load.p_hcd r53 on r53.dt='2021-03-24' and r53.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r53.businessplatform=r.code and
> r53.indicatornumber='YWPT-ZHQI-CD-038-YW-00008'
> left join raw_restapi_load.p_hcd r54 on r54.dt='2021-03-24' and r54.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r54.businessplatform=r.code and
> r54.indicatornumber='YWPT-ZHQI-CD-038-XT-00061'
> left join raw_restapi_load.p_hcd r55 on r55.dt='2021-03-24' and r55.coltime
> =cast('2021-03-24 00:00:00' as timestamp) and r55.businessplatform=r.code and
> r55.indicatornumber='YWPT-ZHQI-CD-038-YW-00009'
> where 1=1
> and r.code='YWPT-ZHQI-CD-038'
> and r.type='biz';{code}
>
> 2. get error
>
> {code:java}
> [ERROR] Could not execute SQL statement. Reason:
> java.io.IOException: Can not make a progress: all selected inputs are already
> finished
> {code}
> 3. execute same sql under streaming mode and get expected output
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
