Repository: hive Updated Branches: refs/heads/master 33fa62f18 -> 48f92c31d
http://git-wip-us.apache.org/repos/asf/hive/blob/48f92c31/ql/src/test/results/clientpositive/druid/kafka_storage_handler.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/druid/kafka_storage_handler.q.out b/ql/src/test/results/clientpositive/druid/kafka_storage_handler.q.out new file mode 100644 index 0000000..3dec33d --- /dev/null +++ b/ql/src/test/results/clientpositive/druid/kafka_storage_handler.q.out @@ -0,0 +1,845 @@ +PREHOOK: query: CREATE EXTERNAL TABLE kafka_table +(`__time` timestamp , `page` string, `user` string, `language` string, +`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, +`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) +STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler' +WITH SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") +TBLPROPERTIES +("kafka.topic" = "test-topic", +"kafka.bootstrap.servers"="localhost:9092", +"kafka.serde.class"="org.apache.hadoop.hive.serde2.JsonSerDe") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@kafka_table +POSTHOOK: query: CREATE EXTERNAL TABLE kafka_table +(`__time` timestamp , `page` string, `user` string, `language` string, +`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, +`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) +STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler' +WITH SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") +TBLPROPERTIES +("kafka.topic" = "test-topic", +"kafka.bootstrap.servers"="localhost:9092", +"kafka.serde.class"="org.apache.hadoop.hive.serde2.JsonSerDe") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@kafka_table +PREHOOK: query: DESCRIBE EXTENDED kafka_table +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@kafka_table +POSTHOOK: query: DESCRIBE EXTENDED kafka_table +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@kafka_table +__time timestamp from deserializer +page string from deserializer +user string from deserializer +language string from deserializer +country string from deserializer +continent string from deserializer +namespace string from deserializer +newpage boolean from deserializer +unpatrolled boolean from deserializer +anonymous boolean from deserializer +robot boolean from deserializer +added int from deserializer +deleted int from deserializer +delta bigint from deserializer +__partition int from deserializer +__offset bigint from deserializer +__timestamp bigint from deserializer +__start_offset bigint from deserializer +__end_offset bigint from deserializer + +#### A masked pattern was here #### +PREHOOK: query: Select `__partition` ,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta FROM kafka_table +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select `__partition` ,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta FROM kafka_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 10 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 0 10 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 0 10 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 0 10 3 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 0 10 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +0 0 10 5 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 0 10 6 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 0 10 7 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 0 10 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 0 10 9 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +PREHOOK: query: Select count(*) FROM kafka_table +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select count(*) FROM kafka_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +10 +PREHOOK: query: Select `__partition`, `__offset`,`__start_offset`,`__end_offset`, `__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +from kafka_table where `__timestamp` > 1533960760123 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select `__partition`, `__offset`,`__start_offset`,`__end_offset`, `__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +from kafka_table where `__timestamp` > 1533960760123 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 0 10 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 1 0 10 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 2 0 10 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 3 0 10 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 4 0 10 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +0 5 0 10 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 6 0 10 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 7 0 10 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 8 0 10 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 9 0 10 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +PREHOOK: query: Select `__partition`, `__offset` ,`__start_offset`,`__end_offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +from kafka_table where `__timestamp` > 533960760123 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select `__partition`, `__offset` ,`__start_offset`,`__end_offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +from kafka_table where `__timestamp` > 533960760123 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 0 10 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 1 0 10 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 2 0 10 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 3 0 10 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 4 0 10 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +0 5 0 10 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 6 0 10 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 7 0 10 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 8 0 10 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 9 0 10 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +PREHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +from kafka_table where (`__offset` > 7 and `__partition` = 0 and `__offset` <9 ) OR +`__offset` = 4 and `__partition` = 0 OR (`__offset` <= 1 and `__partition` = 0 and `__offset` > 0) +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +from kafka_table where (`__offset` > 7 and `__partition` = 0 and `__offset` <9 ) OR +`__offset` = 4 and `__partition` = 0 OR (`__offset` <= 1 and `__partition` = 0 and `__offset` > 0) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 1 9 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 1 9 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +0 1 9 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +PREHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user` from kafka_table where `__offset` = 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user` from kafka_table where `__offset` = 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 5 6 5 NULL Gypsy Danger nuclear +PREHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user` from kafka_table where `__offset` < 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user` from kafka_table where `__offset` < 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 5 0 NULL Gypsy Danger nuclear +0 0 5 1 NULL Striker Eureka speed +0 0 5 2 NULL Cherno Alpha masterYi +0 0 5 3 NULL Crimson Typhoon triplets +0 0 5 4 NULL Coyote Tango stringer +PREHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user` from kafka_table where `__offset` > 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`,`__time`, `page`, `user` from kafka_table where `__offset` > 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 6 10 6 NULL Striker Eureka speed +0 6 10 7 NULL Cherno Alpha masterYi +0 6 10 8 NULL Crimson Typhoon triplets +0 6 10 9 NULL Coyote Tango stringer +PREHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`, `user` from kafka_table where +`__timestamp` > 1000 * to_unix_timestamp(CURRENT_TIMESTAMP - interval '1' HOURS) +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select `__partition`,`__start_offset`,`__end_offset`, `__offset`, `user` from kafka_table where +`__timestamp` > 1000 * to_unix_timestamp(CURRENT_TIMESTAMP - interval '1' HOURS) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 10 0 nuclear +0 0 10 1 speed +0 0 10 2 masterYi +0 0 10 3 triplets +0 0 10 4 stringer +0 0 10 5 nuclear +0 0 10 6 speed +0 0 10 7 masterYi +0 0 10 8 triplets +0 0 10 9 stringer +PREHOOK: query: Select count(*) from kafka_table where `__partition` = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select count(*) from kafka_table where `__partition` = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 +PREHOOK: query: Select count(*) from kafka_table where `__offset` = 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select count(*) from kafka_table where `__offset` = 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 +PREHOOK: query: Select count(*) from kafka_table where `__offset` <= 100 and `__partition` <= 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select count(*) from kafka_table where `__offset` <= 100 and `__partition` <= 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +10 +PREHOOK: query: Drop table kafka_table_offsets +PREHOOK: type: DROPTABLE +POSTHOOK: query: Drop table kafka_table_offsets +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table kafka_table_offsets(partition_id int, max_offset bigint, insert_time timestamp) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@kafka_table_offsets +POSTHOOK: query: create table kafka_table_offsets(partition_id int, max_offset bigint, insert_time timestamp) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@kafka_table_offsets +PREHOOK: query: insert overwrite table kafka_table_offsets select `__partition`, min(`__offset`) - 1, CURRENT_TIMESTAMP from kafka_table group by `__partition`, CURRENT_TIMESTAMP +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Output: default@kafka_table_offsets +POSTHOOK: query: insert overwrite table kafka_table_offsets select `__partition`, min(`__offset`) - 1, CURRENT_TIMESTAMP from kafka_table group by `__partition`, CURRENT_TIMESTAMP +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Output: default@kafka_table_offsets +POSTHOOK: Lineage: kafka_table_offsets.insert_time SIMPLE [] +POSTHOOK: Lineage: kafka_table_offsets.max_offset EXPRESSION [(kafka_table)kafka_table.FieldSchema(name:__offset, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: kafka_table_offsets.partition_id SIMPLE [(kafka_table)kafka_table.FieldSchema(name:__partition, type:int, comment:from deserializer), ] +PREHOOK: query: select partition_id, max_offset from kafka_table_offsets +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table_offsets +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select partition_id, max_offset from kafka_table_offsets +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table_offsets +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 -1 +PREHOOK: query: Drop table orc_kafka_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: Drop table orc_kafka_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: Create table orc_kafka_table (partition_id int, row_offset bigint, kafka_ts bigint, + `__time` timestamp , `page` string, `user` string, `language` string, +`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, +`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint +) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_kafka_table +POSTHOOK: query: Create table orc_kafka_table (partition_id int, row_offset bigint, kafka_ts bigint, + `__time` timestamp , `page` string, `user` string, `language` string, +`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, +`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint +) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_kafka_table +PREHOOK: query: From kafka_table ktable JOIN kafka_table_offsets offset_table +on (ktable.`__partition` = offset_table.partition_id and ktable.`__offset` > offset_table.max_offset and ktable.`__offset` < 3 ) +insert into table orc_kafka_table select `__partition`, `__offset`, `__timestamp`, +`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +Insert overwrite table kafka_table_offsets select +`__partition`, max(`__offset`), CURRENT_TIMESTAMP group by `__partition`, CURRENT_TIMESTAMP +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Input: default@kafka_table_offsets +PREHOOK: Output: default@kafka_table_offsets +PREHOOK: Output: default@orc_kafka_table +POSTHOOK: query: From kafka_table ktable JOIN kafka_table_offsets offset_table +on (ktable.`__partition` = offset_table.partition_id and ktable.`__offset` > offset_table.max_offset and ktable.`__offset` < 3 ) +insert into table orc_kafka_table select `__partition`, `__offset`, `__timestamp`, +`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +Insert overwrite table kafka_table_offsets select +`__partition`, max(`__offset`), CURRENT_TIMESTAMP group by `__partition`, CURRENT_TIMESTAMP +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Input: default@kafka_table_offsets +POSTHOOK: Output: default@kafka_table_offsets +POSTHOOK: Output: default@orc_kafka_table +POSTHOOK: Lineage: kafka_table_offsets.insert_time EXPRESSION [] +POSTHOOK: Lineage: kafka_table_offsets.max_offset EXPRESSION [(kafka_table)ktable.FieldSchema(name:__offset, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: kafka_table_offsets.partition_id SIMPLE [(kafka_table)ktable.FieldSchema(name:__partition, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.__time SIMPLE [(kafka_table)ktable.FieldSchema(name:__time, type:timestamp, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.added SIMPLE [(kafka_table)ktable.FieldSchema(name:added, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.anonymous SIMPLE [(kafka_table)ktable.FieldSchema(name:anonymous, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.continent SIMPLE [(kafka_table)ktable.FieldSchema(name:continent, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.country SIMPLE [(kafka_table)ktable.FieldSchema(name:country, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.deleted SIMPLE [(kafka_table)ktable.FieldSchema(name:deleted, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.delta SIMPLE [(kafka_table)ktable.FieldSchema(name:delta, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.kafka_ts SIMPLE [(kafka_table)ktable.FieldSchema(name:__timestamp, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.language SIMPLE [(kafka_table)ktable.FieldSchema(name:language, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.namespace SIMPLE [(kafka_table)ktable.FieldSchema(name:namespace, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.newpage SIMPLE [(kafka_table)ktable.FieldSchema(name:newpage, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.page SIMPLE [(kafka_table)ktable.FieldSchema(name:page, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.partition_id SIMPLE [(kafka_table)ktable.FieldSchema(name:__partition, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.robot SIMPLE [(kafka_table)ktable.FieldSchema(name:robot, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.row_offset SIMPLE [(kafka_table)ktable.FieldSchema(name:__offset, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.unpatrolled SIMPLE [(kafka_table)ktable.FieldSchema(name:unpatrolled, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.user SIMPLE [(kafka_table)ktable.FieldSchema(name:user, type:string, comment:from deserializer), ] +PREHOOK: query: select count(*) from orc_kafka_table +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from orc_kafka_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +3 +PREHOOK: query: select partition_id, max_offset from kafka_table_offsets +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table_offsets +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select partition_id, max_offset from kafka_table_offsets +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table_offsets +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 2 +PREHOOK: query: select `partition_id`, `row_offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta from orc_kafka_table +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select `partition_id`, `row_offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta from orc_kafka_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +PREHOOK: query: From kafka_table ktable JOIN kafka_table_offsets offset_table +on (ktable.`__partition` = offset_table.partition_id and ktable.`__offset` > offset_table.max_offset) +insert into table orc_kafka_table select `__partition`, `__offset`, `__timestamp`, +`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +Insert overwrite table kafka_table_offsets select +`__partition`, max(`__offset`), CURRENT_TIMESTAMP group by `__partition`, CURRENT_TIMESTAMP +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Input: default@kafka_table_offsets +PREHOOK: Output: default@kafka_table_offsets +PREHOOK: Output: default@orc_kafka_table +POSTHOOK: query: From kafka_table ktable JOIN kafka_table_offsets offset_table +on (ktable.`__partition` = offset_table.partition_id and ktable.`__offset` > offset_table.max_offset) +insert into table orc_kafka_table select `__partition`, `__offset`, `__timestamp`, +`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +Insert overwrite table kafka_table_offsets select +`__partition`, max(`__offset`), CURRENT_TIMESTAMP group by `__partition`, CURRENT_TIMESTAMP +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Input: default@kafka_table_offsets +POSTHOOK: Output: default@kafka_table_offsets +POSTHOOK: Output: default@orc_kafka_table +POSTHOOK: Lineage: kafka_table_offsets.insert_time EXPRESSION [] +POSTHOOK: Lineage: kafka_table_offsets.max_offset EXPRESSION [(kafka_table)ktable.FieldSchema(name:__offset, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: kafka_table_offsets.partition_id SIMPLE [(kafka_table)ktable.FieldSchema(name:__partition, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.__time SIMPLE [(kafka_table)ktable.FieldSchema(name:__time, type:timestamp, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.added SIMPLE [(kafka_table)ktable.FieldSchema(name:added, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.anonymous SIMPLE [(kafka_table)ktable.FieldSchema(name:anonymous, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.continent SIMPLE [(kafka_table)ktable.FieldSchema(name:continent, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.country SIMPLE [(kafka_table)ktable.FieldSchema(name:country, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.deleted SIMPLE [(kafka_table)ktable.FieldSchema(name:deleted, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.delta SIMPLE [(kafka_table)ktable.FieldSchema(name:delta, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.kafka_ts SIMPLE [(kafka_table)ktable.FieldSchema(name:__timestamp, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.language SIMPLE [(kafka_table)ktable.FieldSchema(name:language, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.namespace SIMPLE [(kafka_table)ktable.FieldSchema(name:namespace, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.newpage SIMPLE [(kafka_table)ktable.FieldSchema(name:newpage, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.page SIMPLE [(kafka_table)ktable.FieldSchema(name:page, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.partition_id SIMPLE [(kafka_table)ktable.FieldSchema(name:__partition, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.robot SIMPLE [(kafka_table)ktable.FieldSchema(name:robot, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.row_offset SIMPLE [(kafka_table)ktable.FieldSchema(name:__offset, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.unpatrolled SIMPLE [(kafka_table)ktable.FieldSchema(name:unpatrolled, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.user SIMPLE [(kafka_table)ktable.FieldSchema(name:user, type:string, comment:from deserializer), ] +PREHOOK: query: select partition_id, max_offset from kafka_table_offsets +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table_offsets +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select partition_id, max_offset from kafka_table_offsets +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table_offsets +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 9 +PREHOOK: query: select count(*) from orc_kafka_table +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from orc_kafka_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +10 +PREHOOK: query: select `partition_id`, `row_offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta from orc_kafka_table +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select `partition_id`, `row_offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta from orc_kafka_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 3 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +0 5 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 6 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 7 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 9 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +PREHOOK: query: Drop table kafka_table_offsets +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@kafka_table_offsets +PREHOOK: Output: default@kafka_table_offsets +POSTHOOK: query: Drop table kafka_table_offsets +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@kafka_table_offsets +POSTHOOK: Output: default@kafka_table_offsets +PREHOOK: query: create table kafka_table_offsets(partition_id int, max_offset bigint, insert_time timestamp) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@kafka_table_offsets +POSTHOOK: query: create table kafka_table_offsets(partition_id int, max_offset bigint, insert_time timestamp) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@kafka_table_offsets +PREHOOK: query: Drop table orc_kafka_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@orc_kafka_table +PREHOOK: Output: default@orc_kafka_table +POSTHOOK: query: Drop table orc_kafka_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@orc_kafka_table +POSTHOOK: Output: default@orc_kafka_table +PREHOOK: query: Create table orc_kafka_table (partition_id int, row_offset bigint, kafka_ts bigint, + `__time` timestamp , `page` string, `user` string, `language` string, +`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, +`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint +) stored as ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_kafka_table +POSTHOOK: query: Create table orc_kafka_table (partition_id int, row_offset bigint, kafka_ts bigint, + `__time` timestamp , `page` string, `user` string, `language` string, +`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, +`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint +) stored as ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_kafka_table +PREHOOK: query: From kafka_table ktable LEFT OUTER JOIN kafka_table_offsets offset_table +on (ktable.`__partition` = offset_table.partition_id and ktable.`__offset` > offset_table.max_offset ) +insert into table orc_kafka_table select `__partition`, `__offset`, `__timestamp`, +`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +Insert overwrite table kafka_table_offsets select +`__partition`, max(`__offset`), CURRENT_TIMESTAMP group by `__partition`, CURRENT_TIMESTAMP +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table +PREHOOK: Input: default@kafka_table_offsets +PREHOOK: Output: default@kafka_table_offsets +PREHOOK: Output: default@orc_kafka_table +POSTHOOK: query: From kafka_table ktable LEFT OUTER JOIN kafka_table_offsets offset_table +on (ktable.`__partition` = offset_table.partition_id and ktable.`__offset` > offset_table.max_offset ) +insert into table orc_kafka_table select `__partition`, `__offset`, `__timestamp`, +`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +Insert overwrite table kafka_table_offsets select +`__partition`, max(`__offset`), CURRENT_TIMESTAMP group by `__partition`, CURRENT_TIMESTAMP +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table +POSTHOOK: Input: default@kafka_table_offsets +POSTHOOK: Output: default@kafka_table_offsets +POSTHOOK: Output: default@orc_kafka_table +POSTHOOK: Lineage: kafka_table_offsets.insert_time EXPRESSION [] +POSTHOOK: Lineage: kafka_table_offsets.max_offset EXPRESSION [(kafka_table)ktable.FieldSchema(name:__offset, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: kafka_table_offsets.partition_id SIMPLE [(kafka_table)ktable.FieldSchema(name:__partition, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.__time SIMPLE [(kafka_table)ktable.FieldSchema(name:__time, type:timestamp, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.added SIMPLE [(kafka_table)ktable.FieldSchema(name:added, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.anonymous SIMPLE [(kafka_table)ktable.FieldSchema(name:anonymous, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.continent SIMPLE [(kafka_table)ktable.FieldSchema(name:continent, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.country SIMPLE [(kafka_table)ktable.FieldSchema(name:country, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.deleted SIMPLE [(kafka_table)ktable.FieldSchema(name:deleted, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.delta SIMPLE [(kafka_table)ktable.FieldSchema(name:delta, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.kafka_ts SIMPLE [(kafka_table)ktable.FieldSchema(name:__timestamp, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.language SIMPLE [(kafka_table)ktable.FieldSchema(name:language, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.namespace SIMPLE [(kafka_table)ktable.FieldSchema(name:namespace, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.newpage SIMPLE [(kafka_table)ktable.FieldSchema(name:newpage, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.page SIMPLE [(kafka_table)ktable.FieldSchema(name:page, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.partition_id SIMPLE [(kafka_table)ktable.FieldSchema(name:__partition, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.robot SIMPLE [(kafka_table)ktable.FieldSchema(name:robot, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.row_offset SIMPLE [(kafka_table)ktable.FieldSchema(name:__offset, type:bigint, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.unpatrolled SIMPLE [(kafka_table)ktable.FieldSchema(name:unpatrolled, type:boolean, comment:from deserializer), ] +POSTHOOK: Lineage: orc_kafka_table.user SIMPLE [(kafka_table)ktable.FieldSchema(name:user, type:string, comment:from deserializer), ] +PREHOOK: query: select count(*) from orc_kafka_table +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from orc_kafka_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +10 +PREHOOK: query: select partition_id, max_offset from kafka_table_offsets +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table_offsets +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select partition_id, max_offset from kafka_table_offsets +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table_offsets +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 9 +PREHOOK: query: select `partition_id`, `row_offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta from orc_kafka_table +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_kafka_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select `partition_id`, `row_offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta from orc_kafka_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_kafka_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 3 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +0 5 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 6 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 7 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 9 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +PREHOOK: query: CREATE EXTERNAL TABLE kafka_table_2 +(`__time` timestamp with local time zone , `page` string, `user` string, `language` string, +`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, +`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) +STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler' +TBLPROPERTIES +("kafka.topic" = "test-topic", +"kafka.bootstrap.servers"="localhost:9092") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@kafka_table_2 +POSTHOOK: query: CREATE EXTERNAL TABLE kafka_table_2 +(`__time` timestamp with local time zone , `page` string, `user` string, `language` string, +`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean, +`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint) +STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler' +TBLPROPERTIES +("kafka.topic" = "test-topic", +"kafka.bootstrap.servers"="localhost:9092") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@kafka_table_2 +PREHOOK: query: Select `__partition`, `__offset`, `__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +FROM kafka_table_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table_2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select `__partition`, `__offset`, `__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` , +`unpatrolled` , `anonymous` , `robot` , added , deleted , delta +FROM kafka_table_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table_2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 0 2013-08-30 18:02:33.0 US/Pacific Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 1 2013-08-30 20:32:45.0 US/Pacific Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 2 2013-08-31 00:11:21.0 US/Pacific Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 3 2013-08-31 04:58:39.0 US/Pacific Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 4 2013-08-31 05:41:27.0 US/Pacific Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +0 5 2013-08-31 18:02:33.0 US/Pacific Gypsy Danger nuclear en United States North America article true true false false 57 200 -143 +0 6 2013-08-31 20:32:45.0 US/Pacific Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330 +0 7 2013-09-01 00:11:21.0 US/Pacific Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111 +0 8 2013-09-01 04:58:39.0 US/Pacific Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900 +0 9 2013-09-01 05:41:27.0 US/Pacific Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9 +PREHOOK: query: Select count(*) FROM kafka_table_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@kafka_table_2 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: Select count(*) FROM kafka_table_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@kafka_table_2 +POSTHOOK: Output: hdfs://### HDFS PATH ### +10 +PREHOOK: query: CREATE EXTERNAL TABLE wiki_kafka_avro_table +STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler' +TBLPROPERTIES +("kafka.topic" = "wiki_kafka_avro_table", +"kafka.bootstrap.servers"="localhost:9092", +"kafka.serde.class"="org.apache.hadoop.hive.serde2.avro.AvroSerDe", +'avro.schema.literal'='{ + "type" : "record", + "name" : "Wikipedia", + "namespace" : "org.apache.hive.kafka", + "version": "1", + "fields" : [ { + "name" : "isrobot", + "type" : "boolean" + }, { + "name" : "channel", + "type" : "string" + }, { + "name" : "timestamp", + "type" : "string" + }, { + "name" : "flags", + "type" : "string" + }, { + "name" : "isunpatrolled", + "type" : "boolean" + }, { + "name" : "page", + "type" : "string" + }, { + "name" : "diffurl", + "type" : "string" + }, { + "name" : "added", + "type" : "long" + }, { + "name" : "comment", + "type" : "string" + }, { + "name" : "commentlength", + "type" : "long" + }, { + "name" : "isnew", + "type" : "boolean" + }, { + "name" : "isminor", + "type" : "boolean" + }, { + "name" : "delta", + "type" : "long" + }, { + "name" : "isanonymous", + "type" : "boolean" + }, { + "name" : "user", + "type" : "string" + }, { + "name" : "deltabucket", + "type" : "double" + }, { + "name" : "deleted", + "type" : "long" + }, { + "name" : "namespace", + "type" : "string" + } ] +}' +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@wiki_kafka_avro_table +POSTHOOK: query: CREATE EXTERNAL TABLE wiki_kafka_avro_table +STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler' +TBLPROPERTIES +("kafka.topic" = "wiki_kafka_avro_table", +"kafka.bootstrap.servers"="localhost:9092", +"kafka.serde.class"="org.apache.hadoop.hive.serde2.avro.AvroSerDe", +'avro.schema.literal'='{ + "type" : "record", + "name" : "Wikipedia", + "namespace" : "org.apache.hive.kafka", + "version": "1", + "fields" : [ { + "name" : "isrobot", + "type" : "boolean" + }, { + "name" : "channel", + "type" : "string" + }, { + "name" : "timestamp", + "type" : "string" + }, { + "name" : "flags", + "type" : "string" + }, { + "name" : "isunpatrolled", + "type" : "boolean" + }, { + "name" : "page", + "type" : "string" + }, { + "name" : "diffurl", + "type" : "string" + }, { + "name" : "added", + "type" : "long" + }, { + "name" : "comment", + "type" : "string" + }, { + "name" : "commentlength", + "type" : "long" + }, { + "name" : "isnew", + "type" : "boolean" + }, { + "name" : "isminor", + "type" : "boolean" + }, { + "name" : "delta", + "type" : "long" + }, { + "name" : "isanonymous", + "type" : "boolean" + }, { + "name" : "user", + "type" : "string" + }, { + "name" : "deltabucket", + "type" : "double" + }, { + "name" : "deleted", + "type" : "long" + }, { + "name" : "namespace", + "type" : "string" + } ] +}' +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@wiki_kafka_avro_table +PREHOOK: query: describe extended wiki_kafka_avro_table +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@wiki_kafka_avro_table +POSTHOOK: query: describe extended wiki_kafka_avro_table +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@wiki_kafka_avro_table +isrobot boolean from deserializer +channel string from deserializer +timestamp string from deserializer +flags string from deserializer +isunpatrolled boolean from deserializer +page string from deserializer +diffurl string from deserializer +added bigint from deserializer +comment string from deserializer +commentlength bigint from deserializer +isnew boolean from deserializer +isminor boolean from deserializer +delta bigint from deserializer +isanonymous boolean from deserializer +user string from deserializer +deltabucket double from deserializer +deleted bigint from deserializer +namespace string from deserializer +__partition int from deserializer +__offset bigint from deserializer +__timestamp bigint from deserializer +__start_offset bigint from deserializer +__end_offset bigint from deserializer + +#### A masked pattern was here #### +PREHOOK: query: select cast ((`__timestamp`/1000) as timestamp) as kafka_record_ts, `__partition`, `__offset`, `timestamp`, `user`, `page`, `deleted`, `deltabucket`, `isanonymous`, `commentlength` from wiki_kafka_avro_table +PREHOOK: type: QUERY +PREHOOK: Input: default@wiki_kafka_avro_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select cast ((`__timestamp`/1000) as timestamp) as kafka_record_ts, `__partition`, `__offset`, `timestamp`, `user`, `page`, `deleted`, `deltabucket`, `isanonymous`, `commentlength` from wiki_kafka_avro_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@wiki_kafka_avro_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +2018-08-20 03:37:05.09 0 0 08/19/2018 20:37:05 test-user-0 page is 0 0 0.0 false 0 +2018-08-20 04:37:05.09 0 1 08/19/2018 21:37:05 test-user-1 page is 100 -1 100.4 true 1 +2018-08-20 05:37:05.09 0 2 08/19/2018 22:37:05 test-user-2 page is 200 -2 200.8 true 2 +2018-08-20 06:37:05.09 0 3 08/19/2018 23:37:05 test-user-3 page is 300 -3 301.20000000000005 false 3 +2018-08-20 07:37:05.09 0 4 08/20/2018 00:37:05 test-user-4 page is 400 -4 401.6 true 4 +2018-08-20 08:37:05.09 0 5 08/20/2018 01:37:05 test-user-5 page is 500 -5 502.0 true 5 +2018-08-20 09:37:05.09 0 6 08/20/2018 02:37:05 test-user-6 page is 600 -6 602.4000000000001 false 6 +2018-08-20 10:37:05.09 0 7 08/20/2018 03:37:05 test-user-7 page is 700 -7 702.8000000000001 true 7 +2018-08-20 11:37:05.09 0 8 08/20/2018 04:37:05 test-user-8 page is 800 -8 803.2 true 8 +2018-08-20 12:37:05.09 0 9 08/20/2018 05:37:05 test-user-9 page is 900 -9 903.6 false 9 +2018-08-20 13:37:05.09 0 10 08/20/2018 06:37:05 test-user-10 page is 1000 -10 1004.0 true 10 +PREHOOK: query: select count(*) from wiki_kafka_avro_table +PREHOOK: type: QUERY +PREHOOK: Input: default@wiki_kafka_avro_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from wiki_kafka_avro_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@wiki_kafka_avro_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: select count(distinct `user`) from wiki_kafka_avro_table +PREHOOK: type: QUERY +PREHOOK: Input: default@wiki_kafka_avro_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(distinct `user`) from wiki_kafka_avro_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@wiki_kafka_avro_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 +PREHOOK: query: select sum(deltabucket), min(commentlength) from wiki_kafka_avro_table +PREHOOK: type: QUERY +PREHOOK: Input: default@wiki_kafka_avro_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select sum(deltabucket), min(commentlength) from wiki_kafka_avro_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@wiki_kafka_avro_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +5522.000000000001 0 +PREHOOK: query: select cast ((`__timestamp`/1000) as timestamp) as kafka_record_ts, `__timestamp` as kafka_record_ts_long, +`__partition`, `__start_offset`,`__end_offset`,`__offset`, `timestamp`, `user`, `page`, `deleted`, `deltabucket`, +`isanonymous`, `commentlength` from wiki_kafka_avro_table where `__timestamp` > 1534750625090 +PREHOOK: type: QUERY +PREHOOK: Input: default@wiki_kafka_avro_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select cast ((`__timestamp`/1000) as timestamp) as kafka_record_ts, `__timestamp` as kafka_record_ts_long, +`__partition`, `__start_offset`,`__end_offset`,`__offset`, `timestamp`, `user`, `page`, `deleted`, `deltabucket`, +`isanonymous`, `commentlength` from wiki_kafka_avro_table where `__timestamp` > 1534750625090 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@wiki_kafka_avro_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +2018-08-20 08:37:05.09 1534754225090 0 5 11 5 08/20/2018 01:37:05 test-user-5 page is 500 -5 502.0 true 5 +2018-08-20 09:37:05.09 1534757825090 0 5 11 6 08/20/2018 02:37:05 test-user-6 page is 600 -6 602.4000000000001 false 6 +2018-08-20 10:37:05.09 1534761425090 0 5 11 7 08/20/2018 03:37:05 test-user-7 page is 700 -7 702.8000000000001 true 7 +2018-08-20 11:37:05.09 1534765025090 0 5 11 8 08/20/2018 04:37:05 test-user-8 page is 800 -8 803.2 true 8 +2018-08-20 12:37:05.09 1534768625090 0 5 11 9 08/20/2018 05:37:05 test-user-9 page is 900 -9 903.6 false 9 +2018-08-20 13:37:05.09 1534772225090 0 5 11 10 08/20/2018 06:37:05 test-user-10 page is 1000 -10 1004.0 true 10 http://git-wip-us.apache.org/repos/asf/hive/blob/48f92c31/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java index e1e4d8b..cd37bcc 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java @@ -386,7 +386,7 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector { * Convert the given object to a lazy object using the given {@link ObjectInspector} * * @param obj Object to be converted to a {@link LazyObject} - * @param oi ObjectInspector used for the conversion + * @param objectInspector ObjectInspector used for the conversion * @return the created {@link LazyObject lazy object} * */ private Object toLazyListObject(Object obj, ObjectInspector objectInspector) { @@ -414,7 +414,7 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector { * Convert the given object to a lazy object using the given {@link ObjectInspector} * * @param obj Object to be converted to a {@link LazyObject} - * @param oi ObjectInspector used for the conversion + * @param objectInspector ObjectInspector used for the conversion * @return the created {@link LazyObject lazy object} * */ @SuppressWarnings({ "rawtypes", "unchecked" }) @@ -451,7 +451,7 @@ public class AvroLazyObjectInspector extends LazySimpleStructObjectInspector { * Convert the given object to a lazy object using the given {@link ObjectInspector} * * @param obj Object to be converted to a {@link LazyObject} - * @param oi ObjectInspector used for the conversion + * @param objectInspector ObjectInspector used for the conversion * @return the created {@link LazyObject lazy object} * */ private Object toLazyUnionObject(Object obj, ObjectInspector objectInspector) {
