Henry Lu created HIVE-23844:
-------------------------------
Summary: use fastparquet generate parquet format file, import
hive, query error
Key: HIVE-23844
URL: https://issues.apache.org/jira/browse/HIVE-23844
Project: Hive
Issue Type: Bug
Components: Hive
Affects Versions: 2.1.1
Environment: OS: CentOS Linux release 7.6.1810
JDK: 1.8.0_181
hive: hive-common-2.1.1-cdh6.2.0
fastparquet: 0.4.0
Reporter: Henry Lu
Attachments: 1912076_20200330_000334(102_4).parquet
i use fastparquet generate parquet format file(Please check the attachment ！),
message as follows:
message schema {
optional double timestamps;
optional int32 ESC_BrakePressure (UINT_8);
optional int32 ESC_BrakePressureValid (UINT_8);
optional int32 ESC_EBDWork (UINT_8);
optional int32 ESC_ABSWorkLable (UINT_8);
optional int32 ESC_EBDAlarm (UINT_8);
optional int32 ESC_VehSpdValidFlag (UINT_8);
optional int32 ESC_ABSAlarmSignal (UINT_8);
optional float ESC_VehSpd;
optional float ESC_FrontLeftWHeelSpd;
optional int32 ESC_FLWHeelSpdFaultSignal (UINT_8);
optional float ESC_FrontRightWHeelSpd;
optional int32 ESC_FRWHeelSpdFaultSignal (UINT_8);
optional float ESC_RearLeftWheelSpd;
optional int32 ESC_RLWHeelSpdFaultSignal (UINT_8);
optional float ESC_RearRightWheelSpd;
optional int32 ESC_RRWHeelSpdFaultSignal (UINT_8);
optional int32 ESC_Longitudinal_Acceleration_flag (UINT_8);
optional float ESC_Longitudinal_Acceleration;
optional int32 ESC_ESCOFF (UINT_8);
optional int32 ESC_ESCWorkStatus (UINT_8);
optional int32 ESC_ESCAlarmSig (UINT_8);
optional int32 ESC_TCSCFActive (UINT_8);
optional int32 ESC_ReqIncreaseTorqueFlag (UINT_8);
optional int32 ESC_ReqDecreaseTorqueFlag (UINT_8);
optional int32 ESC_ReqIncreaseTorque (UINT_8);
optional int32 ESC_ReqDecreaseTorque (UINT_8);
optional int32 ESC_ESCValidity (UINT_8);
optional int32 ESC_RollingCount_ESC3 (UINT_8);
optional int32 ESC_CICkSum_ESC3 (UINT_8);
}
creator: fastparquet-python version 1.0.0 (build 111)
extra: pandas = \{"column_indexes": [{"field_name": null, "metadata": null,
"name": null, "numpy_type": "object", "pandas_type": "mixed-integer"}],
"columns": [\{"field_name": "timestamps", "metadata": null, "name":
"timestamps", "numpy_type": "float64", "pandas_type": "float64"},
\{"field_name": "ESC_BrakePressure", "metadata": null, "name":
"ESC_BrakePressure", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_BrakePressureValid", "metadata": null, "name":
"ESC_BrakePressureValid", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_EBDWork", "metadata": null, "name": "ESC_EBDWork",
"numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name":
"ESC_ABSWorkLable", "metadata": null, "name": "ESC_ABSWorkLable", "numpy_type":
"uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_EBDAlarm", "metadata":
null, "name": "ESC_EBDAlarm", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_VehSpdValidFlag", "metadata": null, "name":
"ESC_VehSpdValidFlag", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_ABSAlarmSignal", "metadata": null, "name":
"ESC_ABSAlarmSignal", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_VehSpd", "metadata": null, "name": "ESC_VehSpd",
"numpy_type": "float32", "pandas_type": "float32"}, \{"field_name":
"ESC_FrontLeftWHeelSpd", "metadata": null, "name": "ESC_FrontLeftWHeelSpd",
"numpy_type": "float32", "pandas_type": "float32"}, \{"field_name":
"ESC_FLWHeelSpdFaultSignal", "metadata": null, "name":
"ESC_FLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_FrontRightWHeelSpd", "metadata": null, "name":
"ESC_FrontRightWHeelSpd", "numpy_type": "float32", "pandas_type": "float32"},
\{"field_name": "ESC_FRWHeelSpdFaultSignal", "metadata": null, "name":
"ESC_FRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_RearLeftWheelSpd", "metadata": null, "name":
"ESC_RearLeftWheelSpd", "numpy_type": "float32", "pandas_type": "float32"},
\{"field_name": "ESC_RLWHeelSpdFaultSignal", "metadata": null, "name":
"ESC_RLWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_RearRightWheelSpd", "metadata": null, "name":
"ESC_RearRightWheelSpd", "numpy_type": "float32", "pandas_type": "float32"},
\{"field_name": "ESC_RRWHeelSpdFaultSignal", "metadata": null, "name":
"ESC_RRWHeelSpdFaultSignal", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_Longitudinal_Acceleration_flag", "metadata": null, "name":
"ESC_Longitudinal_Acceleration_flag", "numpy_type": "uint8", "pandas_type":
"uint8"}, \{"field_name": "ESC_Longitudinal_Acceleration", "metadata": null,
"name": "ESC_Longitudinal_Acceleration", "numpy_type": "float32",
"pandas_type": "float32"}, \{"field_name": "ESC_ESCOFF", "metadata": null,
"name": "ESC_ESCOFF", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_ESCWorkStatus", "metadata": null, "name":
"ESC_ESCWorkStatus", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_ESCAlarmSig", "metadata": null, "name": "ESC_ESCAlarmSig",
"numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name":
"ESC_TCSCFActive", "metadata": null, "name": "ESC_TCSCFActive", "numpy_type":
"uint8", "pandas_type": "uint8"}, \{"field_name": "ESC_ReqIncreaseTorqueFlag",
"metadata": null, "name": "ESC_ReqIncreaseTorqueFlag", "numpy_type": "uint8",
"pandas_type": "uint8"}, \{"field_name": "ESC_ReqDecreaseTorqueFlag",
"metadata": null, "name": "ESC_ReqDecreaseTorqueFlag", "numpy_type": "uint8",
"pandas_type": "uint8"}, \{"field_name": "ESC_ReqIncreaseTorque", "metadata":
null, "name": "ESC_ReqIncreaseTorque", "numpy_type": "uint8", "pandas_type":
"uint8"}, \{"field_name": "ESC_ReqDecreaseTorque", "metadata": null, "name":
"ESC_ReqDecreaseTorque", "numpy_type": "uint8", "pandas_type": "uint8"},
\{"field_name": "ESC_ESCValidity", "metadata": null, "name": "ESC_ESCValidity",
"numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name":
"ESC_RollingCount_ESC3", "metadata": null, "name": "ESC_RollingCount_ESC3",
"numpy_type": "uint8", "pandas_type": "uint8"}, \{"field_name":
"ESC_CICkSum_ESC3", "metadata": null, "name": "ESC_CICkSum_ESC3", "numpy_type":
"uint8", "pandas_type": "uint8"}], "creator": \{"library": "fastparquet",
"version": "0.4.0"}, "index_columns": ["timestamps"], "pandas_version": "1.0.1"}
file schema: schema
--------------------------------------------------------------------------------
timestamps: OPTIONAL DOUBLE R:0 D:1
ESC_BrakePressure: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_BrakePressureValid: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_EBDWork: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ABSWorkLable: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_EBDAlarm: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_VehSpdValidFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ABSAlarmSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_VehSpd: OPTIONAL FLOAT R:0 D:1
ESC_FrontLeftWHeelSpd: OPTIONAL FLOAT R:0 D:1
ESC_FLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_FrontRightWHeelSpd: OPTIONAL FLOAT R:0 D:1
ESC_FRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_RearLeftWheelSpd: OPTIONAL FLOAT R:0 D:1
ESC_RLWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_RearRightWheelSpd: OPTIONAL FLOAT R:0 D:1
ESC_RRWHeelSpdFaultSignal: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_Longitudinal_Acceleration_flag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_Longitudinal_Acceleration: OPTIONAL FLOAT R:0 D:1
ESC_ESCOFF: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ESCWorkStatus: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ESCAlarmSig: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_TCSCFActive: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqIncreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqDecreaseTorqueFlag: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqIncreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ReqDecreaseTorque: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_ESCValidity: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_RollingCount_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
ESC_CICkSum_ESC3: OPTIONAL INT32 O:UINT_8 R:0 D:1
row group 1: RC:67096 TS:8321051 OFFSET:4
--------------------------------------------------------------------------------
timestamps: DOUBLE GZIP DO:0 FPO:4 SZ:168467/536807/3.19 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_BrakePressure: INT32 GZIP DO:0 FPO:168549 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_BrakePressureValid: INT32 GZIP DO:0 FPO:168954 SZ:327/268422/820.86
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_EBDWork: INT32 GZIP DO:0 FPO:169364 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_ABSWorkLable: INT32 GZIP DO:0 FPO:169763 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_EBDAlarm: INT32 GZIP DO:0 FPO:170167 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_VehSpdValidFlag: INT32 GZIP DO:0 FPO:170567 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_ABSAlarmSignal: INT32 GZIP DO:0 FPO:170974 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_VehSpd: FLOAT GZIP DO:0 FPO:171380 SZ:118126/268423/2.27 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_FrontLeftWHeelSpd: FLOAT GZIP DO:0 FPO:289578 SZ:111048/268423/2.42
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:400709 SZ:327/268422/820.86
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FrontRightWHeelSpd: FLOAT GZIP DO:0 FPO:401122 SZ:110788/268423/2.42
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_FRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:511994 SZ:327/268422/820.86
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RearLeftWheelSpd: FLOAT GZIP DO:0 FPO:512407 SZ:114567/268423/2.34 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_RLWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:627056 SZ:327/268422/820.86
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RearRightWheelSpd: FLOAT GZIP DO:0 FPO:627469 SZ:115113/268423/2.33
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_RRWHeelSpdFaultSignal: INT32 GZIP DO:0 FPO:742665 SZ:327/268422/820.86
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_Longitudinal_Acceleration_flag: INT32 GZIP DO:0 FPO:743078
SZ:327/268422/820.86 VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_Longitudinal_Acceleration: FLOAT GZIP DO:0 FPO:743500 SZ:46863/268423/5.73
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCOFF: INT32 GZIP DO:0 FPO:790454 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCWorkStatus: INT32 GZIP DO:0 FPO:790852 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCAlarmSig: INT32 GZIP DO:0 FPO:791257 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_TCSCFActive: INT32 GZIP DO:0 FPO:791660 SZ:332/268422/808.50 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqIncreaseTorqueFlag: INT32 GZIP DO:0 FPO:792068 SZ:327/268422/820.86
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqDecreaseTorqueFlag: INT32 GZIP DO:0 FPO:792481 SZ:327/268422/820.86
VC:67096 ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqIncreaseTorque: INT32 GZIP DO:0 FPO:792894 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_ReqDecreaseTorque: INT32 GZIP DO:0 FPO:793303 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_ESCValidity: INT32 GZIP DO:0 FPO:793712 SZ:327/268422/820.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_RollingCount_ESC3: INT32 GZIP DO:0 FPO:794115 SZ:4339/268422/61.86 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
ESC_CICkSum_ESC3: INT32 GZIP DO:0 FPO:798536 SZ:4341/268422/61.83 VC:67096
ENC:PLAIN,BIT_PACKED,RLE
hive create table sql:
create external table if not exists test(`timestamps` double,
`ESC_BrakePressure` int, `ESC_EBDWork` int)
stored as parquet
location '/dfmc/testcar'
TBLPROPERTIES ('PARQUET.COMPRESS'='GZIP');
load data inpath '/dfmc/testcar' into table test;
exe sql : select * from test;
error:
Failed with exception java.io.IOException:java.lang.IllegalStateException:
Group type [message schema {
optional double timestamps;
optional int32 ESC_BrakePressure (INTEGER(8,false));
optional int32 ESC_EBDWork (INTEGER(8,false));
}
] does not contain requested field: optional double timestamps
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
