hive git commit: HIVE-13616: Investigate renaming a table without invalidating the column stats (Reviewed by Chaoyu Tang, Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master df722342a -> e5ba2690f
HIVE-13616: Investigate renaming a table without invalidating the column stats
(Reviewed by Chaoyu Tang, Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e5ba2690
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e5ba2690
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e5ba2690
Branch: refs/heads/master
Commit: e5ba2690f1886952ee366de0310ea2aec031f347
Parents: df72234
Author: Aihua Xu
Authored: Wed May 11 15:11:20 2016 -0400
Committer: Aihua Xu
Committed: Fri May 20 09:26:25 2016 -0400
--
.../hadoop/hive/metastore/HiveAlterHandler.java | 85 --
.../rename_table_update_column_stats.q | 55
.../rename_table_update_column_stats.q.out | 280 +++
3 files changed, 390 insertions(+), 30 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/e5ba2690/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 0652b9d..68c6e44 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
@@ -237,9 +238,8 @@ public class HiveAlterHandler implements AlterHandler {
// alterPartition()
MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true,
environmentContext);
}
- updateTableColumnStatsForAlterTable(msdb, oldt, newt);
- // now finally call alter table
- msdb.alterTable(dbname, name, newt);
+
+ alterTableUpdateTableColumnStats(msdb, oldt, newt);
// commit the changes
success = msdb.commitTransaction();
} catch (InvalidObjectException e) {
@@ -644,48 +644,73 @@ public class HiveAlterHandler implements AlterHandler {
}
}
- private void updateTableColumnStatsForAlterTable(RawStore msdb, Table
oldTable, Table newTable)
+ private void alterTableUpdateTableColumnStats(RawStore msdb,
+ Table oldTable, Table newTable)
throws MetaException, InvalidObjectException {
-String dbName = oldTable.getDbName();
-String tableName = oldTable.getTableName();
-String newDbName =
HiveStringUtils.normalizeIdentifier(newTable.getDbName());
+String dbName = oldTable.getDbName().toLowerCase();
+String tableName =
HiveStringUtils.normalizeIdentifier(oldTable.getTableName());
+String newDbName = newTable.getDbName().toLowerCase();
String newTableName =
HiveStringUtils.normalizeIdentifier(newTable.getTableName());
try {
- if (!dbName.equals(newDbName) || !tableName.equals(newTableName)) {
-msdb.deleteTableColumnStatistics(dbName, tableName, null);
- } else {
-List oldCols = oldTable.getSd().getCols();
-List newCols = newTable.getSd().getCols();
-if (!MetaStoreUtils.areSameColumns(oldCols, newCols)) {
+ List oldCols = oldTable.getSd().getCols();
+ List newCols = newTable.getSd().getCols();
+ List newStatsObjs = new
ArrayList();
+ ColumnStatistics colStats = null;
+ boolean updateColumnStats = true;
+
+ // Nothing to update if everything is the same
+if (newDbName.equals(dbName) &&
+newTableName.equals(tableName) &&
+MetaStoreUtils.areSameColumns(oldCols, newCols)) {
+ updateColumnStats = false;
+}
+
+if (updateColumnStats) {
List oldColNames = new ArrayList(oldCols.size());
for (FieldSchema oldCol : oldCols) {
oldColNames.add(oldCol.getName());
}
- ColumnStatistics cs = msdb.getTableColumnStatistics(dbName,
tableName, oldColNames);
- if (cs == null) {
-return;
- }
+ // Collect column stats which need to be rewritten and remove old
stats
+ colStats = msdb.getTableColumnStatistics(dbName, tableName,
oldColNames);
+ if (colStats == null) {
+updateColumnStats = false;
+ } else {
+List statsObjs = colStats.getStatsObj();
+
hive git commit: HIVE-13796 : fix some tests on branch-1 (Sergey Shelukhin via Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/branch-1 c0b532fce -> 5fe252b93 HIVE-13796 : fix some tests on branch-1 (Sergey Shelukhin via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5fe252b9 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5fe252b9 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5fe252b9 Branch: refs/heads/branch-1 Commit: 5fe252b93d96cfa02fc43bf0aaa61241d51e7a8e Parents: c0b532f Author: Sergey Shelukhin Authored: Fri May 20 07:51:01 2016 -0700 Committer: Ashutosh Chauhan Committed: Fri May 20 07:51:01 2016 -0700 -- .../test/queries/clientpositive/orc_analyze.q | 14 -- .../vectorization_short_regress.q | 15 +- .../results/clientpositive/orc_analyze.q.out| 188 --- .../clientpositive/tez/orc_analyze.q.out| 188 --- .../tez/vectorization_short_regress.q.out | 36 +++- .../vectorization_short_regress.q.out | 36 +++- 6 files changed, 65 insertions(+), 412 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/5fe252b9/ql/src/test/queries/clientpositive/orc_analyze.q -- diff --git a/ql/src/test/queries/clientpositive/orc_analyze.q b/ql/src/test/queries/clientpositive/orc_analyze.q index bd22e6f..295dfd8 100644 --- a/ql/src/test/queries/clientpositive/orc_analyze.q +++ b/ql/src/test/queries/clientpositive/orc_analyze.q @@ -181,28 +181,14 @@ STORED AS orc; INSERT OVERWRITE TABLE orc_create_people PARTITION (state) SELECT * FROM orc_create_people_staging ORDER BY id; --- set the table to text format -ALTER TABLE orc_create_people SET SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'; -ALTER TABLE orc_create_people SET FILEFORMAT TEXTFILE; - --- load the text data into a new partition -LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' OVERWRITE INTO TABLE orc_create_people PARTITION(state="OH"); - --- set the table back to orc -ALTER TABLE orc_create_people SET SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde'; -ALTER TABLE orc_create_people SET FILEFORMAT ORC; - set hive.stats.autogather = true; analyze table orc_create_people partition(state) compute statistics; desc formatted orc_create_people partition(state="Ca"); -desc formatted orc_create_people partition(state="OH"); analyze table orc_create_people partition(state) compute statistics partialscan; desc formatted orc_create_people partition(state="Ca"); -desc formatted orc_create_people partition(state="OH"); analyze table orc_create_people partition(state) compute statistics noscan; desc formatted orc_create_people partition(state="Ca"); -desc formatted orc_create_people partition(state="OH"); drop table orc_create_people; http://git-wip-us.apache.org/repos/asf/hive/blob/5fe252b9/ql/src/test/queries/clientpositive/vectorization_short_regress.q -- diff --git a/ql/src/test/queries/clientpositive/vectorization_short_regress.q b/ql/src/test/queries/clientpositive/vectorization_short_regress.q index bf70507..b2e7aa6 100644 --- a/ql/src/test/queries/clientpositive/vectorization_short_regress.q +++ b/ql/src/test/queries/clientpositive/vectorization_short_regress.q @@ -890,8 +890,19 @@ select count(i) from test_count; select count(i) from test_count; -create table alltypesnull like alltypesorc; -alter table alltypesnull set fileformat textfile; +CREATE TABLE alltypesnull( +ctinyint TINYINT, +csmallint SMALLINT, +cint INT, +cbigint BIGINT, +cfloat FLOAT, +cdouble DOUBLE, +cstring1 STRING, +cstring2 STRING, +ctimestamp1 TIMESTAMP, +ctimestamp2 TIMESTAMP, +cboolean1 BOOLEAN, +cboolean2 BOOLEAN); insert into table alltypesnull select null, null, null, null, null, null, null, null, null, null, null, null from alltypesorc; http://git-wip-us.apache.org/repos/asf/hive/blob/5fe252b9/ql/src/test/results/clientpositive/orc_analyze.q.out -- diff --git a/ql/src/test/results/clientpositive/orc_analyze.q.out b/ql/src/test/results/clientpositive/orc_analyze.q.out index a61a2e6..06baa4e 100644 --- a/ql/src/test/results/clientpositive/orc_analyze.q.out +++ b/ql/src/test/results/clientpositive/orc_analyze.q.out @@ -1405,66 +1405,17 @@ POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).id SIMPLE [(orc_create_ POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ] POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).salary SIMPLE [(orc_create_people_stagin
[09/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.java1.8.out
--
diff --git
a/ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.java1.8.out
b/ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.java1.8.out
deleted file mode 100644
index 12f41eb..000
--- a/ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.java1.8.out
+++ /dev/null
@@ -1,280 +0,0 @@
-PREHOOK: query: -- run this test case in minimr to ensure it works in cluster
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns.
--- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- 5263 00_0
--- 5263 01_0
--- ds=2008-04-08/hr=11/key=103/value=val_103:
--- 99 00_0
--- 99 01_0
--- ds=2008-04-08/hr=11/key=484/value=val_484:
--- 87 00_0
--- 87 01_0
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key) on ('484','51','103')
-stored as DIRECTORIES
-STORED AS RCFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@list_bucketing_static_part
-POSTHOOK: query: -- run this test case in minimr to ensure it works in cluster
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns.
--- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- 5263 00_0
--- 5263 01_0
--- ds=2008-04-08/hr=11/key=103/value=val_103:
--- 99 00_0
--- 99 01_0
--- ds=2008-04-08/hr=11/key=484/value=val_484:
--- 87 00_0
--- 87 01_0
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key) on ('484','51','103')
-stored as DIRECTORIES
-STORED AS RCFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@list_bucketing_static_part
-PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from src
-PREHOOK: type: QUERY
-POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from src
-POSTHOOK: type: QUERY
-ABSTRACT SYNTAX TREE:
-
-TOK_QUERY
- TOK_FROM
- TOK_TABREF
- TOK_TABNAME
-src
- TOK_INSERT
- TOK_DESTINATION
- TOK_TAB
-TOK_TABNAME
- list_bucketing_static_part
-TOK_PARTSPEC
- TOK_PARTVAL
- ds
- '2008-04-08'
- TOK_PARTVAL
- hr
- '11'
- TOK_SELECT
- TOK_SELEXPR
-TOK_TABLE_OR_COL
- key
- TOK_SELEXPR
-TOK_TABLE_OR_COL
- value
-
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
-
-STAGE PLANS:
- Stage: Stage-1
-Spark
- A masked pattern was here
- Vertices:
-Map 1
-Map Operator Tree:
-TableScan
- alias: src
- Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
- GatherStats: false
- Select Operator
-expressions: key (type: string), value (type: string)
-outputColumnNames: _col0, _col1
-Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
-File Output Operator
- compressed: false
- GlobalTableId: 1
- A masked pattern was here
- NumFilesPerFileSink: 1
- Static Partition Specification: ds=2008-04-08/hr=11/
- Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
- A masked pattern was here
- table:
- input format:
org.apache.hadoop.hive.ql.io.RCFileInputFormat
- output format:
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
- properties:
-bucket_count -1
-columns key,value
-columns.comments
-columns.types string:string
- A masked pattern was here
-name default.list_bucketing_static_part
-partition_columns ds/hr
-
[19/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.7.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.7.out
b/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.7.out
deleted file mode 100644
index c15c6a2..000
--- a/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.7.out
+++ /dev/null
@@ -1,813 +0,0 @@
-PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns. merge.
--- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- 5263 00_0
--- 5263 01_0
--- ds=2008-04-08/hr=11/key=103/value=val_103:
--- 99 00_0
--- 99 01_0
--- after merge
--- 142 00_0
--- ds=2008-04-08/hr=11/key=484/value=val_484:
--- 87 00_0
--- 87 01_0
--- after merge
--- 118 01_0
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
-stored as DIRECTORIES
-STORED AS RCFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@list_bucketing_static_part
-POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns. merge.
--- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- 5263 00_0
--- 5263 01_0
--- ds=2008-04-08/hr=11/key=103/value=val_103:
--- 99 00_0
--- 99 01_0
--- after merge
--- 142 00_0
--- ds=2008-04-08/hr=11/key=484/value=val_484:
--- 87 00_0
--- 87 01_0
--- after merge
--- 118 01_0
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
-stored as DIRECTORIES
-STORED AS RCFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@list_bucketing_static_part
-PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from srcpart where ds = '2008-04-08'
-PREHOOK: type: QUERY
-POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from srcpart where ds = '2008-04-08'
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
-
-STAGE PLANS:
- Stage: Stage-1
-Map Reduce
- Map Operator Tree:
- TableScan
-alias: srcpart
-Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE
Column stats: NONE
-GatherStats: false
-Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
- File Output Operator
-compressed: false
-GlobalTableId: 1
- A masked pattern was here
-NumFilesPerFileSink: 1
-Static Partition Specification: ds=2008-04-08/hr=11/
-Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
- A masked pattern was here
-table:
-input format:
org.apache.hadoop.hive.ql.io.RCFileInputFormat
-output format:
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
-properties:
- bucket_count -1
- columns key,value
- columns.comments
- columns.types string:string
- A masked pattern was here
- name default.list_bucketing_static_part
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct list_bucketing_static_part {
string key, string value}
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- A masked pattern was here
-serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-name: default.list_bucketing_static_part
-TotalFiles: 1
-GatherStat
[24/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/char_udf1.q.java1.8.out
--
diff --git a/ql/src/test/results/clientpositive/char_udf1.q.java1.8.out
b/ql/src/test/results/clientpositive/char_udf1.q.java1.8.out
deleted file mode 100644
index 5691a06..000
--- a/ql/src/test/results/clientpositive/char_udf1.q.java1.8.out
+++ /dev/null
@@ -1,457 +0,0 @@
-PREHOOK: query: drop table char_udf_1
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table char_udf_1
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table char_udf_1 (c1 string, c2 string, c3 char(10), c4
char(20))
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@char_udf_1
-POSTHOOK: query: create table char_udf_1 (c1 string, c2 string, c3 char(10),
c4 char(20))
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@char_udf_1
-PREHOOK: query: insert overwrite table char_udf_1
- select key, value, key, value from src where key = '238' limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Output: default@char_udf_1
-POSTHOOK: query: insert overwrite table char_udf_1
- select key, value, key, value from src where key = '238' limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Output: default@char_udf_1
-POSTHOOK: Lineage: char_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
-POSTHOOK: Lineage: char_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value,
type:string, comment:default), ]
-POSTHOOK: Lineage: char_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
-POSTHOOK: Lineage: char_udf_1.c4 EXPRESSION [(src)src.FieldSchema(name:value,
type:string, comment:default), ]
-PREHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- UDFs with char support
-select
- concat(c1, c2),
- concat(c3, c4),
- concat(c1, c2) = concat(c3, c4)
-from char_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@char_udf_1
- A masked pattern was here
-POSTHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- UDFs with char support
-select
- concat(c1, c2),
- concat(c3, c4),
- concat(c1, c2) = concat(c3, c4)
-from char_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_udf_1
- A masked pattern was here
-238val_238 238val_238 true
-PREHOOK: query: select
- upper(c2),
- upper(c4),
- upper(c2) = upper(c4)
-from char_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@char_udf_1
- A masked pattern was here
-POSTHOOK: query: select
- upper(c2),
- upper(c4),
- upper(c2) = upper(c4)
-from char_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_udf_1
- A masked pattern was here
-VAL_238VAL_238 true
-PREHOOK: query: select
- lower(c2),
- lower(c4),
- lower(c2) = lower(c4)
-from char_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@char_udf_1
- A masked pattern was here
-POSTHOOK: query: select
- lower(c2),
- lower(c4),
- lower(c2) = lower(c4)
-from char_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_udf_1
- A masked pattern was here
-val_238val_238 true
-PREHOOK: query: -- Scalar UDFs
-select
- ascii(c2),
- ascii(c4),
- ascii(c2) = ascii(c4)
-from char_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@char_udf_1
- A masked pattern was here
-POSTHOOK: query: -- Scalar UDFs
-select
- ascii(c2),
- ascii(c4),
- ascii(c2) = ascii(c4)
-from char_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_udf_1
- A masked pattern was here
-118118 true
-PREHOOK: query: select
- concat_ws('|', c1, c2),
- concat_ws('|', c3, c4),
- concat_ws('|', c1, c2) = concat_ws('|', c3, c4)
-from char_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@char_udf_1
- A masked pattern was here
-POSTHOOK: query: select
- concat_ws('|', c1, c2),
- concat_ws('|', c3, c4),
- concat_ws('|', c1, c2) = concat_ws('|', c3, c4)
-from char_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_udf_1
- A masked pattern was here
-238|val_238238|val_238 true
-PREHOOK: query: select
- decode(encode(c2, 'US-ASCII'), 'US-ASCII'),
- decode(encode(c4, 'US-ASCII'), 'US-ASCII'),
- decode(encode(c2, 'US-ASCII'), 'US-ASCII') = decode(encode(c4, 'US-ASCII'),
'US-ASCII')
-from char_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@char_udf_1
- A masked pattern was here
-POSTHOOK: query: select
- decode(encode(c2, 'US-ASCII'), 'US-ASCII'),
- decode(encode(c4, 'US-ASCII'), 'US-ASCII'),
- decode(encode(c2, 'US-ASCII'), 'US-ASCII') = decode(encode(c4, 'US-ASCII'),
'US-ASCII')
-from char_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@char_udf_1
- A masked pattern was here
-va
[13/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.7.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.7.out
b/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.7.out
deleted file mode 100644
index 752ea4e..000
--- a/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.7.out
+++ /dev/null
@@ -1,813 +0,0 @@
-PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns. merge.
--- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- 5263 00_0
--- 5263 01_0
--- ds=2008-04-08/hr=11/key=103:
--- 99 00_0
--- 99 01_0
--- after merge
--- 142 00_0
--- ds=2008-04-08/hr=11/key=484:
--- 87 00_0
--- 87 01_0
--- after merge
--- 118 01_0
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key) on ('484','103')
-stored as DIRECTORIES
-STORED AS RCFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@list_bucketing_static_part
-POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns. merge.
--- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- 5263 00_0
--- 5263 01_0
--- ds=2008-04-08/hr=11/key=103:
--- 99 00_0
--- 99 01_0
--- after merge
--- 142 00_0
--- ds=2008-04-08/hr=11/key=484:
--- 87 00_0
--- 87 01_0
--- after merge
--- 118 01_0
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key) on ('484','103')
-stored as DIRECTORIES
-STORED AS RCFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@list_bucketing_static_part
-PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from srcpart where ds = '2008-04-08'
-PREHOOK: type: QUERY
-POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from srcpart where ds = '2008-04-08'
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
-
-STAGE PLANS:
- Stage: Stage-1
-Map Reduce
- Map Operator Tree:
- TableScan
-alias: srcpart
-Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE
Column stats: NONE
-GatherStats: false
-Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
- File Output Operator
-compressed: false
-GlobalTableId: 1
- A masked pattern was here
-NumFilesPerFileSink: 1
-Static Partition Specification: ds=2008-04-08/hr=11/
-Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
- A masked pattern was here
-table:
-input format:
org.apache.hadoop.hive.ql.io.RCFileInputFormat
-output format:
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
-properties:
- bucket_count -1
- columns key,value
- columns.comments
- columns.types string:string
- A masked pattern was here
- name default.list_bucketing_static_part
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct list_bucketing_static_part {
string key, string value}
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- A masked pattern was here
-serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-name: default.list_bucketing_static_part
-TotalFiles: 1
-GatherStats: true
-MultiFileSpray: false
- Path -> Alias:
- A masked pattern was here
- Path -> Partition:
- A masked patte
[14/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.8.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.8.out
b/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.8.out
deleted file mode 100644
index 9947c1a..000
--- a/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.8.out
+++ /dev/null
@@ -1,712 +0,0 @@
-PREHOOK: query: -- list bucketing alter table ... concatenate:
--- Use list bucketing DML to generate mutilple files in partitions by turning
off merge
--- dynamic partition. multiple skewed columns. merge.
--- The following explains merge example used in this test case
--- DML will generated 2 partitions
--- ds=2008-04-08/hr=a1
--- ds=2008-04-08/hr=b1
--- without merge, each partition has more files
--- ds=2008-04-08/hr=a1 has 2 files
--- ds=2008-04-08/hr=b1 has 6 files
--- with merge each partition has more files
--- ds=2008-04-08/hr=a1 has 1 files
--- ds=2008-04-08/hr=b1 has 4 files
--- The following shows file size and name in each directory
---
hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- without merge
--- 155 00_0
--- 155 01_0
--- with merge
--- 254 00_0
--- hr=b1/key=103/value=val_103:
--- without merge
--- 99 00_0
--- 99 01_0
--- with merge
--- 142 01_0
---
hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- without merge
--- 5181 00_0
--- 5181 01_0
--- with merge
--- 5181 00_0
--- 5181 01_0
--- hr=b1/key=484/value=val_484
--- without merge
--- 87 00_0
--- 87 01_0
--- with merge
--- 118 02_0
-
--- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- create a skewed table
-create table list_bucketing_dynamic_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
-stored as DIRECTORIES
-STORED AS RCFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@list_bucketing_dynamic_part
-POSTHOOK: query: -- list bucketing alter table ... concatenate:
--- Use list bucketing DML to generate mutilple files in partitions by turning
off merge
--- dynamic partition. multiple skewed columns. merge.
--- The following explains merge example used in this test case
--- DML will generated 2 partitions
--- ds=2008-04-08/hr=a1
--- ds=2008-04-08/hr=b1
--- without merge, each partition has more files
--- ds=2008-04-08/hr=a1 has 2 files
--- ds=2008-04-08/hr=b1 has 6 files
--- with merge each partition has more files
--- ds=2008-04-08/hr=a1 has 1 files
--- ds=2008-04-08/hr=b1 has 4 files
--- The following shows file size and name in each directory
---
hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- without merge
--- 155 00_0
--- 155 01_0
--- with merge
--- 254 00_0
--- hr=b1/key=103/value=val_103:
--- without merge
--- 99 00_0
--- 99 01_0
--- with merge
--- 142 01_0
---
hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- without merge
--- 5181 00_0
--- 5181 01_0
--- with merge
--- 5181 00_0
--- 5181 01_0
--- hr=b1/key=484/value=val_484
--- without merge
--- 87 00_0
--- 87 01_0
--- with merge
--- 118 02_0
-
--- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- create a skewed table
-create table list_bucketing_dynamic_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
-stored as DIRECTORIES
-STORED AS RCFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@list_bucketing_dynamic_part
-PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
-explain extended
-insert overwrite table list_bucketing_dynamic_part partition (ds =
'2008-04-08', hr)
-select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds =
'2008-04-08'
-PREHOOK: type: QUERY
-POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
-explain extended
-insert overwrite table list_bucketing_dynamic_part partition (ds =
'2008-04-08', hr)
-select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds =
'2008-04-08'
-POSTHOOK: type: QUERY
-ABSTRACT SYNTAX TREE:
-
-TOK_QUERY
- TOK_FROM
- TOK_TABREF
- TOK_TABNAME
-srcpart
- TOK_INSERT
- TOK_DESTINATION
- TOK_TAB
-TOK_TABNAME
- list_bucketing_dynamic_part
-TOK_PARTSPEC
- TOK_PARTVAL
- ds
- '2008-04-08'
- TOK_PARTVAL
- hr
- TOK_SELECT
- TOK_S
[01/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
Repository: hive
Updated Branches:
refs/heads/java8 2695a6356 -> 060aa5799
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/varchar_udf1.q.java1.7.out
--
diff --git a/ql/src/test/results/clientpositive/varchar_udf1.q.java1.7.out
b/ql/src/test/results/clientpositive/varchar_udf1.q.java1.7.out
deleted file mode 100644
index 459d93b..000
--- a/ql/src/test/results/clientpositive/varchar_udf1.q.java1.7.out
+++ /dev/null
@@ -1,457 +0,0 @@
-PREHOOK: query: drop table varchar_udf_1
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table varchar_udf_1
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table varchar_udf_1 (c1 string, c2 string, c3
varchar(10), c4 varchar(20))
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@varchar_udf_1
-POSTHOOK: query: create table varchar_udf_1 (c1 string, c2 string, c3
varchar(10), c4 varchar(20))
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@varchar_udf_1
-PREHOOK: query: insert overwrite table varchar_udf_1
- select key, value, key, value from src where key = '238' limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Output: default@varchar_udf_1
-POSTHOOK: query: insert overwrite table varchar_udf_1
- select key, value, key, value from src where key = '238' limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Output: default@varchar_udf_1
-POSTHOOK: Lineage: varchar_udf_1.c1 SIMPLE [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
-POSTHOOK: Lineage: varchar_udf_1.c2 SIMPLE [(src)src.FieldSchema(name:value,
type:string, comment:default), ]
-POSTHOOK: Lineage: varchar_udf_1.c3 EXPRESSION [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
-POSTHOOK: Lineage: varchar_udf_1.c4 EXPRESSION
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- UDFs with varchar support
-select
- concat(c1, c2),
- concat(c3, c4),
- concat(c1, c2) = concat(c3, c4)
-from varchar_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-POSTHOOK: query: -- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- UDFs with varchar support
-select
- concat(c1, c2),
- concat(c3, c4),
- concat(c1, c2) = concat(c3, c4)
-from varchar_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-238val_238 238val_238 true
-PREHOOK: query: select
- upper(c2),
- upper(c4),
- upper(c2) = upper(c4)
-from varchar_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-POSTHOOK: query: select
- upper(c2),
- upper(c4),
- upper(c2) = upper(c4)
-from varchar_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-VAL_238VAL_238 true
-PREHOOK: query: select
- lower(c2),
- lower(c4),
- lower(c2) = lower(c4)
-from varchar_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-POSTHOOK: query: select
- lower(c2),
- lower(c4),
- lower(c2) = lower(c4)
-from varchar_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-val_238val_238 true
-PREHOOK: query: -- Scalar UDFs
-select
- ascii(c2),
- ascii(c4),
- ascii(c2) = ascii(c4)
-from varchar_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-POSTHOOK: query: -- Scalar UDFs
-select
- ascii(c2),
- ascii(c4),
- ascii(c2) = ascii(c4)
-from varchar_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-118118 true
-PREHOOK: query: select
- concat_ws('|', c1, c2),
- concat_ws('|', c3, c4),
- concat_ws('|', c1, c2) = concat_ws('|', c3, c4)
-from varchar_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-POSTHOOK: query: select
- concat_ws('|', c1, c2),
- concat_ws('|', c3, c4),
- concat_ws('|', c1, c2) = concat_ws('|', c3, c4)
-from varchar_udf_1 limit 1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-238|val_238238|val_238 true
-PREHOOK: query: select
- decode(encode(c2, 'US-ASCII'), 'US-ASCII'),
- decode(encode(c4, 'US-ASCII'), 'US-ASCII'),
- decode(encode(c2, 'US-ASCII'), 'US-ASCII') = decode(encode(c4, 'US-ASCII'),
'US-ASCII')
-from varchar_udf_1 limit 1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@varchar_udf_1
- A masked pattern was here
-POSTHOOK: query: select
- decode(encode(c2, 'US-ASCII'), 'US-ASCII'),
- decode(encode(c4, 'US-ASCII'), 'US-ASCII'),
- decode(encode(c2, 'US-ASCII'), 'US-ASCII') =
[17/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out
b/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out
new file mode 100644
index 000..09cb847
--- /dev/null
+++ b/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out
@@ -0,0 +1,504 @@
+PREHOOK: query: -- list bucketing DML: multiple skewed columns. 2 stages
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- create a skewed table
+create table list_bucketing_dynamic_part (key String, value String)
+partitioned by (ds String, hr String)
+skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
+stored as DIRECTORIES
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@list_bucketing_dynamic_part
+POSTHOOK: query: -- list bucketing DML: multiple skewed columns. 2 stages
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- create a skewed table
+create table list_bucketing_dynamic_part (key String, value String)
+partitioned by (ds String, hr String)
+skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
+stored as DIRECTORIES
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@list_bucketing_dynamic_part
+PREHOOK: query: -- list bucketing DML
+explain extended
+insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08',
hr) select key, value, hr from srcpart where ds='2008-04-08'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- list bucketing DML
+explain extended
+insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08',
hr) select key, value, hr from srcpart where ds='2008-04-08'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+Map Reduce
+ Map Operator Tree:
+ TableScan
+alias: srcpart
+Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE
Column stats: NONE
+GatherStats: false
+Select Operator
+ expressions: key (type: string), value (type: string), hr (type:
string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+compressed: false
+GlobalTableId: 1
+ A masked pattern was here
+NumFilesPerFileSink: 1
+Static Partition Specification: ds=2008-04-08/
+Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
+ A masked pattern was here
+table:
+input format: org.apache.hadoop.mapred.TextInputFormat
+output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+ A masked pattern was here
+ name default.list_bucketing_dynamic_part
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_dynamic_part {
string key, string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ A masked pattern was here
+serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+name: default.list_bucketing_dynamic_part
+TotalFiles: 1
+GatherStats: true
+MultiFileSpray: false
+ Path -> Alias:
+ A masked pattern was here
+ Path -> Partition:
+ A masked pattern was here
+ Partition
+base file name: hr=11
+input format: org.apache.hadoop.mapred.TextInputFormat
+output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+partition values:
+ ds 2008-04-08
+ hr 11
+properties:
+ COLUMN_STATS_ACCURATE
{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+ A masked pattern was here
+ name default.srcpart
+ numFiles 1
+ numRows 500
+ partition_columns ds/hr
+ partition_columns.types st
[11/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out b/ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out new file mode 100644 index 000..fc8eb1c --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_cast_constant.q.out @@ -0,0 +1,216 @@ +PREHOOK: query: DROP TABLE over1k +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE over1k +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE over1korc +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE over1korc +POSTHOOK: type: DROPTABLE +PREHOOK: query: -- data setup +CREATE TABLE over1k(t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + dec decimal(4,2), + bin binary) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1k +POSTHOOK: query: -- data setup +CREATE TABLE over1k(t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + dec decimal(4,2), + bin binary) +ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1k +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/over1k' OVERWRITE INTO TABLE over1k +PREHOOK: type: LOAD + A masked pattern was here +PREHOOK: Output: default@over1k +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/over1k' OVERWRITE INTO TABLE over1k +POSTHOOK: type: LOAD + A masked pattern was here +POSTHOOK: Output: default@over1k +PREHOOK: query: CREATE TABLE over1korc(t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + dec decimal(4,2), + bin binary) +STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@over1korc +POSTHOOK: query: CREATE TABLE over1korc(t tinyint, + si smallint, + i int, + b bigint, + f float, + d double, + bo boolean, + s string, + ts timestamp, + dec decimal(4,2), + bin binary) +STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@over1korc +PREHOOK: query: INSERT INTO TABLE over1korc SELECT * FROM over1k +PREHOOK: type: QUERY +PREHOOK: Input: default@over1k +PREHOOK: Output: default@over1korc +POSTHOOK: query: INSERT INTO TABLE over1korc SELECT * FROM over1k +POSTHOOK: type: QUERY +POSTHOOK: Input: default@over1k +POSTHOOK: Output: default@over1korc +POSTHOOK: Lineage: over1korc.b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ] +POSTHOOK: Lineage: over1korc.bin SIMPLE [(over1k)over1k.FieldSchema(name:bin, type:binary, comment:null), ] +POSTHOOK: Lineage: over1korc.bo SIMPLE [(over1k)over1k.FieldSchema(name:bo, type:boolean, comment:null), ] +POSTHOOK: Lineage: over1korc.d SIMPLE [(over1k)over1k.FieldSchema(name:d, type:double, comment:null), ] +POSTHOOK: Lineage: over1korc.dec SIMPLE [(over1k)over1k.FieldSchema(name:dec, type:decimal(4,2), comment:null), ] +POSTHOOK: Lineage: over1korc.f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ] +POSTHOOK: Lineage: over1korc.i SIMPLE [(over1k)over1k.FieldSchema(name:i, type:int, comment:null), ] +POSTHOOK: Lineage: over1korc.s SIMPLE [(over1k)over1k.FieldSchema(name:s, type:string, comment:null), ] +POSTHOOK: Lineage: over1korc.si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ] +POSTHOOK: Lineage: over1korc.t SIMPLE [(over1k)over1k.FieldSchema(name:t, type:tinyint, comment:null), ] +POSTHOOK: Lineage: over1korc.ts SIMPLE [(over1k)over1k.FieldSchema(name:ts, type:timestamp, comment:null), ] +PREHOOK: query: EXPLAIN SELECT + i, + AVG(CAST(50 AS INT)) AS `avg_int_ok`, + AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`, + AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok` + FROM over1korc GROUP BY i ORDER BY i LIMIT 10 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT + i, + AVG(CAST(50 AS INT)) AS `avg_int_ok`, + AVG(CAST(50 AS DOUBLE)) AS `avg_double_ok`, + AVG(CAST(50 AS DECIMAL)) AS `avg_decimal_ok` + FROM over1korc GROUP BY i ORDER BY i LIMIT 10 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Tez + A masked pa
[15/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_6.q.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_6.q.out
b/ql/src/test/results/clientpositive/list_bucket_dml_6.q.out
new file mode 100644
index 000..e53fee7
--- /dev/null
+++ b/ql/src/test/results/clientpositive/list_bucket_dml_6.q.out
@@ -0,0 +1,1005 @@
+PREHOOK: query: -- list bucketing DML: dynamic partition. multiple skewed
columns. merge.
+-- The following explains merge example used in this test case
+-- DML will generated 2 partitions
+-- ds=2008-04-08/hr=a1
+-- ds=2008-04-08/hr=b1
+-- without merge, each partition has more files
+-- ds=2008-04-08/hr=a1 has 2 files
+-- ds=2008-04-08/hr=b1 has 6 files
+-- with merge each partition has more files
+-- ds=2008-04-08/hr=a1 has 1 files
+-- ds=2008-04-08/hr=b1 has 4 files
+-- The following shows file size and name in each directory
+--
hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
+-- without merge
+-- 155 00_0
+-- 155 01_0
+-- with merge
+-- 254 00_0
+-- hr=b1/key=103/value=val_103:
+-- without merge
+-- 99 00_0
+-- 99 01_0
+-- with merge
+-- 142 01_0
+--
hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
+-- without merge
+-- 5181 00_0
+-- 5181 01_0
+-- with merge
+-- 5181 00_0
+-- 5181 01_0
+-- hr=b1/key=484/value=val_484
+-- without merge
+-- 87 00_0
+-- 87 01_0
+-- with merge
+-- 118 02_0
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- create a skewed table
+create table list_bucketing_dynamic_part (key String, value String)
+partitioned by (ds String, hr String)
+skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
+stored as DIRECTORIES
+STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@list_bucketing_dynamic_part
+POSTHOOK: query: -- list bucketing DML: dynamic partition. multiple skewed
columns. merge.
+-- The following explains merge example used in this test case
+-- DML will generated 2 partitions
+-- ds=2008-04-08/hr=a1
+-- ds=2008-04-08/hr=b1
+-- without merge, each partition has more files
+-- ds=2008-04-08/hr=a1 has 2 files
+-- ds=2008-04-08/hr=b1 has 6 files
+-- with merge each partition has more files
+-- ds=2008-04-08/hr=a1 has 1 files
+-- ds=2008-04-08/hr=b1 has 4 files
+-- The following shows file size and name in each directory
+--
hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
+-- without merge
+-- 155 00_0
+-- 155 01_0
+-- with merge
+-- 254 00_0
+-- hr=b1/key=103/value=val_103:
+-- without merge
+-- 99 00_0
+-- 99 01_0
+-- with merge
+-- 142 01_0
+--
hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
+-- without merge
+-- 5181 00_0
+-- 5181 01_0
+-- with merge
+-- 5181 00_0
+-- 5181 01_0
+-- hr=b1/key=484/value=val_484
+-- without merge
+-- 87 00_0
+-- 87 01_0
+-- with merge
+-- 118 02_0
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- create a skewed table
+create table list_bucketing_dynamic_part (key String, value String)
+partitioned by (ds String, hr String)
+skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
+stored as DIRECTORIES
+STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@list_bucketing_dynamic_part
+PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
+explain extended
+insert overwrite table list_bucketing_dynamic_part partition (ds =
'2008-04-08', hr)
+select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds =
'2008-04-08'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
+explain extended
+insert overwrite table list_bucketing_dynamic_part partition (ds =
'2008-04-08', hr)
+select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds =
'2008-04-08'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+Map Reduce
+ Map Operator Tree:
+ TableScan
+alias: srcpart
+Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE
Column stats: NONE
+GatherStats: false
+Select Operator
+ expressions: key (type: string), value (type: string),
if(((UDFToDouble(key) % 100.0) = 0.0), 'a1', 'b1') (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
+ Fil
[18/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out
b/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out
new file mode 100644
index 000..5f0406a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out
@@ -0,0 +1,811 @@
+PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- list bucketing DML: static partition. multiple skewed columns. merge.
+-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
+-- 5263 00_0
+-- 5263 01_0
+-- ds=2008-04-08/hr=11/key=103/value=val_103:
+-- 99 00_0
+-- 99 01_0
+-- after merge
+-- 142 00_0
+-- ds=2008-04-08/hr=11/key=484/value=val_484:
+-- 87 00_0
+-- 87 01_0
+-- after merge
+-- 118 01_0
+
+-- create a skewed table
+create table list_bucketing_static_part (key String, value String)
+partitioned by (ds String, hr String)
+skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
+stored as DIRECTORIES
+STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@list_bucketing_static_part
+POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- list bucketing DML: static partition. multiple skewed columns. merge.
+-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
+-- 5263 00_0
+-- 5263 01_0
+-- ds=2008-04-08/hr=11/key=103/value=val_103:
+-- 99 00_0
+-- 99 01_0
+-- after merge
+-- 142 00_0
+-- ds=2008-04-08/hr=11/key=484/value=val_484:
+-- 87 00_0
+-- 87 01_0
+-- after merge
+-- 118 01_0
+
+-- create a skewed table
+create table list_bucketing_static_part (key String, value String)
+partitioned by (ds String, hr String)
+skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
+stored as DIRECTORIES
+STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@list_bucketing_static_part
+PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
+explain extended
+insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
+select key, value from srcpart where ds = '2008-04-08'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
+explain extended
+insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
+select key, value from srcpart where ds = '2008-04-08'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+Map Reduce
+ Map Operator Tree:
+ TableScan
+alias: srcpart
+Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE
Column stats: NONE
+GatherStats: false
+Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+compressed: false
+GlobalTableId: 1
+ A masked pattern was here
+NumFilesPerFileSink: 1
+Static Partition Specification: ds=2008-04-08/hr=11/
+Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
+ A masked pattern was here
+table:
+input format:
org.apache.hadoop.hive.ql.io.RCFileInputFormat
+output format:
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+ A masked pattern was here
+ name default.list_bucketing_static_part
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_static_part {
string key, string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ A masked pattern was here
+serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+name: default.list_bucketing_static_part
+TotalFiles: 1
+GatherStats: true
+MultiFileSpray: false
+ Path -> Alias:
+ A masked pattern was here #
[25/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/cbo_rp_outer_join_ppr.q.java1.7.out
--
diff --git
a/ql/src/test/results/clientpositive/cbo_rp_outer_join_ppr.q.java1.7.out
b/ql/src/test/results/clientpositive/cbo_rp_outer_join_ppr.q.java1.7.out
deleted file mode 100644
index 5c40dc4..000
--- a/ql/src/test/results/clientpositive/cbo_rp_outer_join_ppr.q.java1.7.out
+++ /dev/null
@@ -1,693 +0,0 @@
-PREHOOK: query: -- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
-EXPLAIN EXTENDED
- FROM
- src a
- FULL OUTER JOIN
- srcpart b
- ON (a.key = b.key AND b.ds = '2008-04-08')
- SELECT a.key, a.value, b.key, b.value
- WHERE a.key > 10 AND a.key < 20 AND b.key > 15 AND b.key < 25
-PREHOOK: type: QUERY
-POSTHOOK: query: -- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
-EXPLAIN EXTENDED
- FROM
- src a
- FULL OUTER JOIN
- srcpart b
- ON (a.key = b.key AND b.ds = '2008-04-08')
- SELECT a.key, a.value, b.key, b.value
- WHERE a.key > 10 AND a.key < 20 AND b.key > 15 AND b.key < 25
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
-Map Reduce
- Map Operator Tree:
- TableScan
-alias: a
-Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE
Column stats: NONE
-GatherStats: false
-Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE
Column stats: NONE
- Reduce Output Operator
-key expressions: key (type: string)
-null sort order: a
-sort order: +
-Map-reduce partition columns: key (type: string)
-Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
-tag: 0
-value expressions: value (type: string)
-auto parallelism: false
- TableScan
-alias: b
-Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE
Column stats: NONE
-GatherStats: false
-Select Operator
- expressions: key (type: string), value (type: string), ds (type:
string)
- outputColumnNames: key, value, ds
- Statistics: Num rows: 2000 Data size: 21248 Basic stats:
COMPLETE Column stats: NONE
- Reduce Output Operator
-key expressions: key (type: string)
-null sort order: a
-sort order: +
-Map-reduce partition columns: key (type: string)
-Statistics: Num rows: 2000 Data size: 21248 Basic stats:
COMPLETE Column stats: NONE
-tag: 1
-value expressions: value (type: string), ds (type: string)
-auto parallelism: false
- Path -> Alias:
- A masked pattern was here
- Path -> Partition:
- A masked pattern was here
- Partition
-base file name: src
-input format: org.apache.hadoop.mapred.TextInputFormat
-output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-properties:
- COLUMN_STATS_ACCURATE
{"COLUMN_STATS":{"key":"true","value":"true"},"BASIC_STATS":"true"}
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
- A masked pattern was here
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
- A masked pattern was here
-serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
-COLUMN_STATS_ACCURATE
{"COLUMN_STATS":{"key":"true","value":"true"},"BASIC_STATS":"true"}
-bucket_count -1
-columns key,value
-columns.comments 'default','default'
-columns.types string:string
- A masked pattern was here
-name default.src
-numFiles 1
-numRows 500
-rawDataSize 5312
-serialization.ddl struct src { string key, string value}
-serialization.format 1
-serialization.lib
org.apache.hadoop.hive.serd
[02/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/subquery_notin_having.q.out -- diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.out new file mode 100644 index 000..6aeac65 --- /dev/null +++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.out @@ -0,0 +1,764 @@ +Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product +PREHOOK: query: -- non agg, non corr + +explain +select key, count(*) +from src +group by key +having key not in + ( select key from src s1 +where s1.key > '12' + ) +PREHOOK: type: QUERY +POSTHOOK: query: -- non agg, non corr + +explain +select key, count(*) +from src +group by key +having key not in + ( select key from src s1 +where s1.key > '12' + ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-4 + Stage-3 depends on stages: Stage-2 + Stage-4 is a root stage + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: src +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator +aggregations: count() +keys: key (type: string) +mode: hash +outputColumnNames: _col0, _col1 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Reduce Operator Tree: +Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 +Map Reduce + Map Operator Tree: + TableScan +Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: bigint) + TableScan +Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: +Join Operator + condition map: + Inner Join 0 to 1 + keys: +0 +1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 +Map Reduce + Map Operator Tree: + TableScan +Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + TableScan +alias: src +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (key > '12') (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: string) +outputColumnNames: _col0 +Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLE
[16/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_6.q.java1.8.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_6.q.java1.8.out
b/ql/src/test/results/clientpositive/list_bucket_dml_6.q.java1.8.out
deleted file mode 100644
index 1960d41..000
--- a/ql/src/test/results/clientpositive/list_bucket_dml_6.q.java1.8.out
+++ /dev/null
@@ -1,1119 +0,0 @@
-PREHOOK: query: -- list bucketing DML: dynamic partition. multiple skewed
columns. merge.
--- The following explains merge example used in this test case
--- DML will generated 2 partitions
--- ds=2008-04-08/hr=a1
--- ds=2008-04-08/hr=b1
--- without merge, each partition has more files
--- ds=2008-04-08/hr=a1 has 2 files
--- ds=2008-04-08/hr=b1 has 6 files
--- with merge each partition has more files
--- ds=2008-04-08/hr=a1 has 1 files
--- ds=2008-04-08/hr=b1 has 4 files
--- The following shows file size and name in each directory
---
hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- without merge
--- 155 00_0
--- 155 01_0
--- with merge
--- 254 00_0
--- hr=b1/key=103/value=val_103:
--- without merge
--- 99 00_0
--- 99 01_0
--- with merge
--- 142 01_0
---
hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- without merge
--- 5181 00_0
--- 5181 01_0
--- with merge
--- 5181 00_0
--- 5181 01_0
--- hr=b1/key=484/value=val_484
--- without merge
--- 87 00_0
--- 87 01_0
--- with merge
--- 118 02_0
-
--- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- create a skewed table
-create table list_bucketing_dynamic_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
-stored as DIRECTORIES
-STORED AS RCFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@list_bucketing_dynamic_part
-POSTHOOK: query: -- list bucketing DML: dynamic partition. multiple skewed
columns. merge.
--- The following explains merge example used in this test case
--- DML will generated 2 partitions
--- ds=2008-04-08/hr=a1
--- ds=2008-04-08/hr=b1
--- without merge, each partition has more files
--- ds=2008-04-08/hr=a1 has 2 files
--- ds=2008-04-08/hr=b1 has 6 files
--- with merge each partition has more files
--- ds=2008-04-08/hr=a1 has 1 files
--- ds=2008-04-08/hr=b1 has 4 files
--- The following shows file size and name in each directory
---
hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- without merge
--- 155 00_0
--- 155 01_0
--- with merge
--- 254 00_0
--- hr=b1/key=103/value=val_103:
--- without merge
--- 99 00_0
--- 99 01_0
--- with merge
--- 142 01_0
---
hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- without merge
--- 5181 00_0
--- 5181 01_0
--- with merge
--- 5181 00_0
--- 5181 01_0
--- hr=b1/key=484/value=val_484
--- without merge
--- 87 00_0
--- 87 01_0
--- with merge
--- 118 02_0
-
--- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- create a skewed table
-create table list_bucketing_dynamic_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
-stored as DIRECTORIES
-STORED AS RCFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@list_bucketing_dynamic_part
-PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
-explain extended
-insert overwrite table list_bucketing_dynamic_part partition (ds =
'2008-04-08', hr)
-select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds =
'2008-04-08'
-PREHOOK: type: QUERY
-POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
-explain extended
-insert overwrite table list_bucketing_dynamic_part partition (ds =
'2008-04-08', hr)
-select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds =
'2008-04-08'
-POSTHOOK: type: QUERY
-ABSTRACT SYNTAX TREE:
-
-TOK_QUERY
- TOK_FROM
- TOK_TABREF
- TOK_TABNAME
-srcpart
- TOK_INSERT
- TOK_DESTINATION
- TOK_TAB
-TOK_TABNAME
- list_bucketing_dynamic_part
-TOK_PARTSPEC
- TOK_PARTVAL
- ds
- '2008-04-08'
- TOK_PARTVAL
- hr
- TOK_SELECT
- TOK_SELEXPR
-TOK_TABLE_OR_COL
- key
- TOK_SELEXPR
-TOK_TABLE_OR_COL
- value
- TOK_SELEXPR
-TOK_FUNCTION
-
[12/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out
b/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out
new file mode 100644
index 000..81f3af3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out
@@ -0,0 +1,811 @@
+PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- list bucketing DML: static partition. multiple skewed columns. merge.
+-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
+-- 5263 00_0
+-- 5263 01_0
+-- ds=2008-04-08/hr=11/key=103:
+-- 99 00_0
+-- 99 01_0
+-- after merge
+-- 142 00_0
+-- ds=2008-04-08/hr=11/key=484:
+-- 87 00_0
+-- 87 01_0
+-- after merge
+-- 118 01_0
+
+-- create a skewed table
+create table list_bucketing_static_part (key String, value String)
+partitioned by (ds String, hr String)
+skewed by (key) on ('484','103')
+stored as DIRECTORIES
+STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@list_bucketing_static_part
+POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- list bucketing DML: static partition. multiple skewed columns. merge.
+-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
+-- 5263 00_0
+-- 5263 01_0
+-- ds=2008-04-08/hr=11/key=103:
+-- 99 00_0
+-- 99 01_0
+-- after merge
+-- 142 00_0
+-- ds=2008-04-08/hr=11/key=484:
+-- 87 00_0
+-- 87 01_0
+-- after merge
+-- 118 01_0
+
+-- create a skewed table
+create table list_bucketing_static_part (key String, value String)
+partitioned by (ds String, hr String)
+skewed by (key) on ('484','103')
+stored as DIRECTORIES
+STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@list_bucketing_static_part
+PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
+explain extended
+insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
+select key, value from srcpart where ds = '2008-04-08'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
+explain extended
+insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
+select key, value from srcpart where ds = '2008-04-08'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+Map Reduce
+ Map Operator Tree:
+ TableScan
+alias: srcpart
+Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE
Column stats: NONE
+GatherStats: false
+Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+compressed: false
+GlobalTableId: 1
+ A masked pattern was here
+NumFilesPerFileSink: 1
+Static Partition Specification: ds=2008-04-08/hr=11/
+Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
+ A masked pattern was here
+table:
+input format:
org.apache.hadoop.hive.ql.io.RCFileInputFormat
+output format:
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+ A masked pattern was here
+ name default.list_bucketing_static_part
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_static_part {
string key, string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ A masked pattern was here
+serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+name: default.list_bucketing_static_part
+TotalFiles: 1
+GatherStats: true
+MultiFileSpray: false
+ Path -> Alias:
+ A masked pattern was here
+ Path -> Partition:
+ A masked pattern was here
+ Partition
+base file name: hr=11
+input format: or
hive git commit: HIVE-13657: Spark driver stderr logs should appear in hive client logs (Mohit Sabharwal, reviewed by Sergio Pena)
Repository: hive
Updated Branches:
refs/heads/master e5ba2690f -> 360dfa0ff
HIVE-13657: Spark driver stderr logs should appear in hive client logs (Mohit
Sabharwal, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/360dfa0f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/360dfa0f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/360dfa0f
Branch: refs/heads/master
Commit: 360dfa0ffd0d0500db016861ef24299f1596274d
Parents: e5ba269
Author: Mohit Sabharwal
Authored: Fri May 20 11:16:43 2016 -0500
Committer: Sergio Pena
Committed: Fri May 20 11:16:43 2016 -0500
--
.../hive/spark/client/SparkClientImpl.java | 35
1 file changed, 28 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/360dfa0f/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
--
diff --git
a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
index ae78bc3..dfe263f 100644
---
a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
+++
b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
@@ -42,6 +42,7 @@ import java.io.Serializable;
import java.io.Writer;
import java.net.URI;
import java.net.URL;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -68,6 +69,7 @@ class SparkClientImpl implements SparkClient {
private static final Logger LOG =
LoggerFactory.getLogger(SparkClientImpl.class);
private static final long DEFAULT_SHUTDOWN_TIMEOUT = 1; // In
milliseconds
+ private static final long MAX_ERR_LOG_LINES_FOR_RPC = 1000;
private static final String OSX_TEST_OPTS = "SPARK_OSX_TEST_OPTS";
private static final String SPARK_HOME_ENV = "SPARK_HOME";
@@ -391,7 +393,6 @@ class SparkClientImpl implements SparkClient {
argv.add(numOfExecutors);
}
}
-
if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) {
try {
String currentUser = Utils.getUGI().getShortUserName();
@@ -445,8 +446,9 @@ class SparkClientImpl implements SparkClient {
final Process child = pb.start();
int childId = childIdGenerator.incrementAndGet();
- redirect("stdout-redir-" + childId, child.getInputStream());
- redirect("stderr-redir-" + childId, child.getErrorStream());
+ final List childErrorLog = new ArrayList();
+ redirect("stdout-redir-" + childId, new
Redirector(child.getInputStream()));
+ redirect("stderr-redir-" + childId, new
Redirector(child.getErrorStream(), childErrorLog));
runnable = new Runnable() {
@Override
@@ -454,8 +456,15 @@ class SparkClientImpl implements SparkClient {
try {
int exitCode = child.waitFor();
if (exitCode != 0) {
- rpcServer.cancelClient(clientId, "Child process exited before
connecting back");
- LOG.warn("Child process exited with code {}.", exitCode);
+ StringBuilder errStr = new StringBuilder();
+ for (String s : childErrorLog) {
+errStr.append(s);
+errStr.append('\n');
+ }
+
+ rpcServer.cancelClient(clientId,
+ "Child process exited before connecting back with error log
" + errStr.toString());
+ LOG.warn("Child process exited with code {}", exitCode);
}
} catch (InterruptedException ie) {
LOG.warn("Waiting thread interrupted, killing child process.");
@@ -475,8 +484,8 @@ class SparkClientImpl implements SparkClient {
return thread;
}
- private void redirect(String name, InputStream in) {
-Thread thread = new Thread(new Redirector(in));
+ private void redirect(String name, Redirector redirector) {
+Thread thread = new Thread(redirector);
thread.setName(name);
thread.setDaemon(true);
thread.start();
@@ -587,17 +596,29 @@ class SparkClientImpl implements SparkClient {
private class Redirector implements Runnable {
private final BufferedReader in;
+private List errLogs;
+private int numErrLogLines = 0;
Redirector(InputStream in) {
this.in = new BufferedReader(new InputStreamReader(in));
}
+Redirector(InputStream in, List errLogs) {
+ this.in = new BufferedReader(new InputStreamReader(in));
+ this.errLogs = errLogs;
+}
+
@Override
public void run() {
try {
String line = null;
while ((line = in.readLine()) != null) {
LOG.info(line);
+ if (e
[07/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out
--
diff --git a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out
b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out
index 217fe76..dfa6ea5 100644
--- a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out
+++ b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out
@@ -20,90 +20,6 @@ EXPLAIN EXTENDED
SELECT a.key, a.value, b.key, b.value
WHERE a.key > 10 AND a.key < 20 AND b.key > 15 AND b.key < 25
POSTHOOK: type: QUERY
-ABSTRACT SYNTAX TREE:
-
-TOK_QUERY
- TOK_FROM
- TOK_FULLOUTERJOIN
- TOK_TABREF
-TOK_TABNAME
- src
-a
- TOK_TABREF
-TOK_TABNAME
- srcpart
-b
- AND
-=
- .
- TOK_TABLE_OR_COL
- a
- key
- .
- TOK_TABLE_OR_COL
- b
- key
-=
- .
- TOK_TABLE_OR_COL
- b
- ds
- '2008-04-08'
- TOK_INSERT
- TOK_DESTINATION
- TOK_DIR
-TOK_TMP_FILE
- TOK_SELECT
- TOK_SELEXPR
-.
- TOK_TABLE_OR_COL
- a
- key
- TOK_SELEXPR
-.
- TOK_TABLE_OR_COL
- a
- value
- TOK_SELEXPR
-.
- TOK_TABLE_OR_COL
- b
- key
- TOK_SELEXPR
-.
- TOK_TABLE_OR_COL
- b
- value
- TOK_WHERE
- AND
-AND
- AND
- >
- .
-TOK_TABLE_OR_COL
- a
-key
- 10
- <
- .
-TOK_TABLE_OR_COL
- a
-key
- 20
- >
- .
- TOK_TABLE_OR_COL
-b
- key
- 15
-<
- .
- TOK_TABLE_OR_COL
- b
- key
- 25
-
-
STAGE DEPENDENCIES:
Stage-1 is a root stage
Stage-0 depends on stages: Stage-1
@@ -112,7 +28,7 @@ STAGE PLANS:
Stage: Stage-1
Spark
Edges:
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL
SORT, 1)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 4), Map 3 (PARTITION-LEVEL
SORT, 4)
A masked pattern was here
Vertices:
Map 1
@@ -121,14 +37,19 @@ STAGE PLANS:
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
GatherStats: false
- Reduce Output Operator
-key expressions: key (type: string)
-sort order: +
-Map-reduce partition columns: key (type: string)
+ Select Operator
+expressions: key (type: string), value (type: string)
+outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
-tag: 0
-value expressions: value (type: string)
-auto parallelism: false
+Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ tag: 0
+ value expressions: _col1 (type: string)
+ auto parallelism: false
Path -> Alias:
A masked pattern was here
Path -> Partition:
@@ -138,7 +59,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
-COLUMN_STATS_ACCURATE true
+COLUMN_STATS_ACCURATE
{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
columns key,value
columns.comments 'default','default'
@@ -158,7 +79,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
[05/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/stats_list_bucket.q.out
--
diff --git a/ql/src/test/results/clientpositive/stats_list_bucket.q.out
b/ql/src/test/results/clientpositive/stats_list_bucket.q.out
new file mode 100644
index 000..c34c414
--- /dev/null
+++ b/ql/src/test/results/clientpositive/stats_list_bucket.q.out
@@ -0,0 +1,189 @@
+PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+
+drop table stats_list_bucket
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+
+drop table stats_list_bucket
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table stats_list_bucket_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table stats_list_bucket_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table stats_list_bucket (
+ c1 string,
+ c2 string
+) partitioned by (ds string, hr string)
+skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
+stored as directories
+stored as rcfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@stats_list_bucket
+POSTHOOK: query: create table stats_list_bucket (
+ c1 string,
+ c2 string
+) partitioned by (ds string, hr string)
+skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
+stored as directories
+stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@stats_list_bucket
+PREHOOK: query: -- Try partitioned table with list bucketing.
+-- The stats should show 500 rows loaded, as many rows as the src table has.
+
+insert overwrite table stats_list_bucket partition (ds = '2008-04-08', hr =
'11')
+ select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@stats_list_bucket@ds=2008-04-08/hr=11
+POSTHOOK: query: -- Try partitioned table with list bucketing.
+-- The stats should show 500 rows loaded, as many rows as the src table has.
+
+insert overwrite table stats_list_bucket partition (ds = '2008-04-08', hr =
'11')
+ select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@stats_list_bucket@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: stats_list_bucket PARTITION(ds=2008-04-08,hr=11).c1 SIMPLE
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: stats_list_bucket PARTITION(ds=2008-04-08,hr=11).c2 SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: desc formatted stats_list_bucket partition (ds = '2008-04-08',
hr = '11')
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@stats_list_bucket
+POSTHOOK: query: desc formatted stats_list_bucket partition (ds =
'2008-04-08', hr = '11')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@stats_list_bucket
+# col_name data_type comment
+
+c1 string
+c2 string
+
+# Partition Information
+# col_name data_type comment
+
+ds string
+hr string
+
+# Detailed Partition Information
+Partition Value: [2008-04-08, 11]
+Database: default
+Table: stats_list_bucket
+ A masked pattern was here
+Partition Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ numFiles4
+ numRows 500
+ rawDataSize 4812
+ totalSize 5522
+ A masked pattern was here
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed:No
+Num Buckets: -1
+Bucket Columns:[]
+Sort Columns: []
+Stored As SubDirectories: Yes
+Skewed Columns:[c1, c2]
+Skewed Values: [[466, val_466], [287, val_287], [82, val_82]]
+ A masked pattern was here
+Skewed Value to Truncated Path:{[466,
val_466]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=466/c2=val_466, [82,
val_82]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=82/c2=val_82, [287,
val_287]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=287/c2=val_287}
+Storage Desc Params:
[26/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/avro_nullable_fields.q.java1.7.out
--
diff --git
a/ql/src/test/results/clientpositive/avro_nullable_fields.q.java1.7.out
b/ql/src/test/results/clientpositive/avro_nullable_fields.q.java1.7.out
deleted file mode 100644
index 52b09d4..000
--- a/ql/src/test/results/clientpositive/avro_nullable_fields.q.java1.7.out
+++ /dev/null
@@ -1,179 +0,0 @@
-PREHOOK: query: -- Verify that nullable fields properly work
-
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
-CREATE TABLE test_serializer(string1 STRING,
- int1 INT,
- tinyint1 TINYINT,
- smallint1 SMALLINT,
- bigint1 BIGINT,
- boolean1 BOOLEAN,
- float1 FLOAT,
- double1 DOUBLE,
- list1 ARRAY,
- map1 MAP,
- struct1
STRUCT,
- enum1 STRING,
- nullableint INT,
- bytes1 BINARY,
- fixed1 BINARY)
- ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' COLLECTION ITEMS TERMINATED BY
':' MAP KEYS TERMINATED BY '#' LINES TERMINATED BY '\n'
- STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@test_serializer
-POSTHOOK: query: -- Verify that nullable fields properly work
-
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
-CREATE TABLE test_serializer(string1 STRING,
- int1 INT,
- tinyint1 TINYINT,
- smallint1 SMALLINT,
- bigint1 BIGINT,
- boolean1 BOOLEAN,
- float1 FLOAT,
- double1 DOUBLE,
- list1 ARRAY,
- map1 MAP,
- struct1
STRUCT,
- enum1 STRING,
- nullableint INT,
- bytes1 BINARY,
- fixed1 BINARY)
- ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' COLLECTION ITEMS TERMINATED BY
':' MAP KEYS TERMINATED BY '#' LINES TERMINATED BY '\n'
- STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@test_serializer
-PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/csv.txt' INTO TABLE
test_serializer
-PREHOOK: type: LOAD
- A masked pattern was here
-PREHOOK: Output: default@test_serializer
-POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/csv.txt' INTO TABLE
test_serializer
-POSTHOOK: type: LOAD
- A masked pattern was here
-POSTHOOK: Output: default@test_serializer
-PREHOOK: query: CREATE TABLE as_avro
- ROW FORMAT
- SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
- STORED AS
- INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
- OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
- TBLPROPERTIES (
-'avro.schema.literal'='{
- "namespace": "com.howdy",
- "name": "some_schema",
- "type": "record",
- "fields": [
-{ "name": "string1", "type": ["null", "string"] },
-{ "name": "int1", "type": ["null", "int"] },
-{ "name": "tinyint1", "type": ["null", "int"] },
-{ "name": "smallint1", "type": ["null", "int"] },
-{ "name": "bigint1", "type": ["null", "long"] },
-{ "name": "boolean1", "type": ["null", "boolean"] },
-{ "name": "float1", "type": ["null", "float"] },
-{ "name": "double1", "type": ["null", "double"] },
-{ "name": "list1", "type": ["null", {"type": "array", "items":
"string"}] },
-{ "name": "map1", "type": ["null", {"type": "map", "values": "int"}] },
-{ "name": "struct1", "type": ["null", {"type": "record", "name":
"struct1_name", "fields": [
- { "name": "sInt", "type": "int" },
- { "name": "sBoolean", "type": "boolean" },
- { "name": "sString", "type": "string" }
-]}] },
-{ "name": "enum1", "type": ["null", {"type": "enum", "name":
"enum1_values", "symbols": ["BLUE", "RED", "GREEN"]}] },
-{ "name": "nullableint", "type": ["null", "int"] },
-{ "name": "bytes1", "type": ["null", "bytes"] },
-{ "name": "fixed1", "type": ["null", {"type": "fixed", "name":
"threebytes", "size": 3}] }
- ]
-}'
- )
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@as_avro
-POSTHOOK: query: CREATE TABLE as_avro
- ROW FORMAT
- SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
- STORED AS
- INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
- OUTP
[23/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/join0.q.out -- diff --git a/ql/src/test/results/clientpositive/join0.q.out b/ql/src/test/results/clientpositive/join0.q.out new file mode 100644 index 000..59122e2 --- /dev/null +++ b/ql/src/test/results/clientpositive/join0.q.out @@ -0,0 +1,238 @@ +Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 +JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN +SELECT src1.key as k1, src1.value as v1, + src2.key as k2, src2.value as v2 FROM + (SELECT * FROM src WHERE src.key < 10) src1 +JOIN + (SELECT * FROM src WHERE src.key < 10) src2 + SORT BY k1, v1, k2, v2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: src +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: string), value (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + TableScan +alias: src +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (key < 10) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: string), value (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + sort order: + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Reduce Operator Tree: +Join Operator + condition map: + Inner Join 0 to 1 + keys: +0 +1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 +Map Reduce + Map Operator Tree: + TableScan +Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) + sort order: + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: +Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey2 (type: string), KEY.reducesinkkey3 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 +Fetch Operator + limit: -1 + Processor Tree: +ListSink + +Warning: Shuffle Join JOIN[8][tables = [src1, src2]] in Stage 'Stage-1:MAPRED' is a cross product +PREHOOK: query:
[22/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.8.out
--
diff --git
a/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.8.out
b/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.8.out
deleted file mode 100644
index 00a6235..000
--- a/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.8.out
+++ /dev/null
@@ -1,424 +0,0 @@
-PREHOOK: query: -- Ensure it works if skewed column is not the first column in
the table columns
-
--- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns.
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (value) on ('val_466','val_287','val_82')
-stored as DIRECTORIES
-STORED AS RCFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@list_bucketing_static_part
-POSTHOOK: query: -- Ensure it works if skewed column is not the first column
in the table columns
-
--- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns.
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (value) on ('val_466','val_287','val_82')
-stored as DIRECTORIES
-STORED AS RCFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@list_bucketing_static_part
-PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from src
-PREHOOK: type: QUERY
-POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from src
-POSTHOOK: type: QUERY
-ABSTRACT SYNTAX TREE:
-
-TOK_QUERY
- TOK_FROM
- TOK_TABREF
- TOK_TABNAME
-src
- TOK_INSERT
- TOK_DESTINATION
- TOK_TAB
-TOK_TABNAME
- list_bucketing_static_part
-TOK_PARTSPEC
- TOK_PARTVAL
- ds
- '2008-04-08'
- TOK_PARTVAL
- hr
- '11'
- TOK_SELECT
- TOK_SELEXPR
-TOK_TABLE_OR_COL
- key
- TOK_SELEXPR
-TOK_TABLE_OR_COL
- value
-
-
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
-
-STAGE PLANS:
- Stage: Stage-1
-Map Reduce
- Map Operator Tree:
- TableScan
-alias: src
-Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE
Column stats: NONE
-GatherStats: false
-Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE
Column stats: NONE
- File Output Operator
-compressed: false
-GlobalTableId: 1
- A masked pattern was here
-NumFilesPerFileSink: 1
-Static Partition Specification: ds=2008-04-08/hr=11/
-Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
- A masked pattern was here
-table:
-input format:
org.apache.hadoop.hive.ql.io.RCFileInputFormat
-output format:
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
-properties:
- bucket_count -1
- columns key,value
- columns.comments
- columns.types string:string
- A masked pattern was here
- name default.list_bucketing_static_part
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct list_bucketing_static_part {
string key, string value}
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- A masked pattern was here
-serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-name: default.list_bucketing_static_part
-TotalFiles: 1
-GatherStats: true
-MultiFileSpray: false
- Path -> Alias:
-
[06/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.java1.8.out -- diff --git a/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.java1.8.out b/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.java1.8.out deleted file mode 100644 index 1bfdba2..000 --- a/ql/src/test/results/clientpositive/spark/subquery_multiinsert.q.java1.8.out +++ /dev/null @@ -1,890 +0,0 @@ -PREHOOK: query: -- SORT_QUERY_RESULTS --- JAVA_VERSION_SPECIFIC_OUTPUT - -CREATE TABLE src_4( - key STRING, - value STRING -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src_4 -POSTHOOK: query: -- SORT_QUERY_RESULTS --- JAVA_VERSION_SPECIFIC_OUTPUT - -CREATE TABLE src_4( - key STRING, - value STRING -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src_4 -RUN: Stage-0:DDL -PREHOOK: query: CREATE TABLE src_5( - key STRING, - value STRING -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src_5 -POSTHOOK: query: CREATE TABLE src_5( - key STRING, - value STRING -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src_5 -RUN: Stage-0:DDL -Warning: Shuffle Join JOIN[31][tables = [sq_2_notin_nullcheck]] in Work 'Reducer 2' is a cross product -PREHOOK: query: explain -from src b -INSERT OVERWRITE TABLE src_4 - select * - where b.key in - (select a.key -from src a -where b.value = a.value and a.key > '9' - ) -INSERT OVERWRITE TABLE src_5 - select * - where b.key not in ( select key from src s1 where s1.key > '2') - order by key -PREHOOK: type: QUERY -POSTHOOK: query: explain -from src b -INSERT OVERWRITE TABLE src_4 - select * - where b.key in - (select a.key -from src a -where b.value = a.value and a.key > '9' - ) -INSERT OVERWRITE TABLE src_5 - select * - where b.key not in ( select key from src s1 where s1.key > '2') - order by key -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-2 is a root stage - Stage-1 depends on stages: Stage-2 - Stage-3 depends on stages: Stage-1 - Stage-0 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-2 -Spark - Edges: -Reducer 2 <- Map 10 (PARTITION-LEVEL SORT, 1), Reducer 9 (PARTITION-LEVEL SORT, 1) -Reducer 3 <- Map 7 (PARTITION-LEVEL SORT, 2), Reducer 2 (PARTITION-LEVEL SORT, 2) -Reducer 5 <- Map 11 (PARTITION-LEVEL SORT, 2), Map 6 (PARTITION-LEVEL SORT, 2) -Reducer 9 <- Map 8 (GROUP, 1) -Reducer 4 <- Reducer 3 (SORT, 1) - A masked pattern was here - Vertices: -Map 10 -Map Operator Tree: -TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -sort order: -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -value expressions: key (type: string), value (type: string) -Map 11 -Map Operator Tree: -TableScan - alias: b - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: key (type: string), value (type: string) -sort order: ++ -Map-reduce partition columns: key (type: string), value (type: string) -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Map 6 -Map Operator Tree: -TableScan - alias: a - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: ((key > '9') and value is not null) (type: boolean) -Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Group By Operator -keys: _col0 (type: string), _col1 (type: string) -mode: hash -outputColumnNames: _col0, _col1 -Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE -Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type:
[03/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out -- diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out deleted file mode 100644 index 70f9591..000 --- a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out +++ /dev/null @@ -1,766 +0,0 @@ -Warning: Shuffle Join JOIN[21][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-2:MAPRED' is a cross product -PREHOOK: query: -- non agg, non corr --- JAVA_VERSION_SPECIFIC_OUTPUT - -explain -select key, count(*) -from src -group by key -having key not in - ( select key from src s1 -where s1.key > '12' - ) -PREHOOK: type: QUERY -POSTHOOK: query: -- non agg, non corr --- JAVA_VERSION_SPECIFIC_OUTPUT - -explain -select key, count(*) -from src -group by key -having key not in - ( select key from src s1 -where s1.key > '12' - ) -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-2 depends on stages: Stage-1, Stage-4 - Stage-3 depends on stages: Stage-2 - Stage-4 is a root stage - Stage-0 depends on stages: Stage-3 - -STAGE PLANS: - Stage: Stage-1 -Map Reduce - Map Operator Tree: - TableScan -alias: src -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: key (type: string) - outputColumnNames: key - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Group By Operator -aggregations: count() -keys: key (type: string) -mode: hash -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - Reduce Operator Tree: -Group By Operator - aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1 - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - File Output Operator -compressed: false -table: -input format: org.apache.hadoop.mapred.SequenceFileInputFormat -output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat -serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 -Map Reduce - Map Operator Tree: - TableScan -Reduce Output Operator - sort order: - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) - TableScan -Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Operator Tree: -Join Operator - condition map: - Inner Join 0 to 1 - keys: -0 -1 - outputColumnNames: _col0, _col1 - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - File Output Operator -compressed: false -table: -input format: org.apache.hadoop.mapred.SequenceFileInputFormat -output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat -serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-3 -Map Reduce - Map Operator Tree: - TableScan -Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: bigint) - TableScan -alias: src -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Filter Operator - predicate: (key > '12') (type: boolean) - Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: string) -o
[20/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.7.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.7.out
b/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.7.out
deleted file mode 100644
index dcfbec0..000
--- a/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.7.out
+++ /dev/null
@@ -1,591 +0,0 @@
-PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns.
--- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- 5263 00_0
--- 5263 01_0
--- ds=2008-04-08/hr=11/key=103/value=val_103:
--- 99 00_0
--- 99 01_0
--- ds=2008-04-08/hr=11/key=484/value=val_484:
--- 87 00_0
--- 87 01_0
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
-stored as DIRECTORIES
-STORED AS RCFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@list_bucketing_static_part
-POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
--- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
--- list bucketing DML: static partition. multiple skewed columns.
--- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
--- 5263 00_0
--- 5263 01_0
--- ds=2008-04-08/hr=11/key=103/value=val_103:
--- 99 00_0
--- 99 01_0
--- ds=2008-04-08/hr=11/key=484/value=val_484:
--- 87 00_0
--- 87 01_0
-
--- create a skewed table
-create table list_bucketing_static_part (key String, value String)
-partitioned by (ds String, hr String)
-skewed by (key, value) on
(('484','val_484'),('51','val_14'),('103','val_103'))
-stored as DIRECTORIES
-STORED AS RCFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@list_bucketing_static_part
-PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate
a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from srcpart where ds = '2008-04-08'
-PREHOOK: type: QUERY
-POSTHOOK: query: -- list bucketing DML without merge. use bucketize to
generate a few small files.
-explain extended
-insert overwrite table list_bucketing_static_part partition (ds =
'2008-04-08', hr = '11')
-select key, value from srcpart where ds = '2008-04-08'
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
- Stage-2 depends on stages: Stage-0
-
-STAGE PLANS:
- Stage: Stage-1
-Map Reduce
- Map Operator Tree:
- TableScan
-alias: srcpart
-Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE
Column stats: NONE
-GatherStats: false
-Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
- File Output Operator
-compressed: false
-GlobalTableId: 1
- A masked pattern was here
-NumFilesPerFileSink: 1
-Static Partition Specification: ds=2008-04-08/hr=11/
-Statistics: Num rows: 1000 Data size: 10624 Basic stats:
COMPLETE Column stats: NONE
- A masked pattern was here
-table:
-input format:
org.apache.hadoop.hive.ql.io.RCFileInputFormat
-output format:
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
-properties:
- bucket_count -1
- columns key,value
- columns.comments
- columns.types string:string
- A masked pattern was here
- name default.list_bucketing_static_part
- partition_columns ds/hr
- partition_columns.types string:string
- serialization.ddl struct list_bucketing_static_part {
string key, string value}
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
- A masked pattern was here
-serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
-name: default.list_bucketing_static_part
-TotalFiles: 1
-GatherStats: true
-MultiFileSpray: false
- Path -> Alias:
- A masked pattern was here
- Path -> Partition:
- A masked
[27/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/060aa579 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/060aa579 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/060aa579 Branch: refs/heads/java8 Commit: 060aa579982fb181a0e7e856f203a24b2c44a9ad Parents: 2695a63 Author: Mohit Sabharwal Authored: Fri May 20 11:14:13 2016 -0500 Committer: Sergio Pena Committed: Fri May 20 11:14:13 2016 -0500 -- .../columnstats_partlvl_invalid_values.q|1 - .../clientpositive/authorization_explain.q |1 - ql/src/test/queries/clientpositive/avro_date.q |1 - .../clientpositive/avro_deserialize_map_null.q |1 - .../clientpositive/avro_nullable_fields.q |1 - .../queries/clientpositive/avro_timestamp.q |1 - .../clientpositive/cbo_rp_outer_join_ppr.q |1 - ql/src/test/queries/clientpositive/char_udf1.q |1 - ql/src/test/queries/clientpositive/input4.q |1 - ql/src/test/queries/clientpositive/join0.q |1 - .../queries/clientpositive/list_bucket_dml_10.q |1 - .../queries/clientpositive/list_bucket_dml_11.q |1 - .../queries/clientpositive/list_bucket_dml_12.q |1 - .../queries/clientpositive/list_bucket_dml_13.q |1 - .../queries/clientpositive/list_bucket_dml_2.q |1 - .../queries/clientpositive/list_bucket_dml_4.q |1 - .../queries/clientpositive/list_bucket_dml_5.q |1 - .../queries/clientpositive/list_bucket_dml_6.q |1 - .../queries/clientpositive/list_bucket_dml_8.q |1 - .../queries/clientpositive/list_bucket_dml_9.q |1 - .../queries/clientpositive/outer_join_ppr.q |1 - .../queries/clientpositive/parquet_map_null.q |1 - ql/src/test/queries/clientpositive/plan_json.q |1 - .../queries/clientpositive/stats_list_bucket.q |1 - ql/src/test/queries/clientpositive/str_to_map.q |1 - .../clientpositive/subquery_multiinsert.q |1 - .../clientpositive/subquery_notin_having.q |1 - .../test/queries/clientpositive/varchar_udf1.q |1 - .../clientpositive/vector_cast_constant.q |1 - ...mnstats_partlvl_invalid_values.q.java1.7.out | 73 -- ...mnstats_partlvl_invalid_values.q.java1.8.out | 73 -- .../columnstats_partlvl_invalid_values.q.out| 69 ++ .../authorization_explain.q.java1.7.out | 44 - .../authorization_explain.q.java1.8.out | 47 - .../clientpositive/authorization_explain.q.out | 40 + .../clientpositive/avro_date.q.java1.7.out | 130 -- .../clientpositive/avro_date.q.java1.8.out | 130 -- .../test/results/clientpositive/avro_date.q.out | 126 ++ .../avro_deserialize_map_null.q.java1.7.out | 57 - .../avro_deserialize_map_null.q.java1.8.out | 57 - .../avro_deserialize_map_null.q.out | 55 + .../avro_nullable_fields.q.java1.7.out | 179 --- .../avro_nullable_fields.q.java1.8.out | 179 --- .../clientpositive/avro_nullable_fields.q.out | 177 +++ .../clientpositive/avro_timestamp.q.java1.7.out | 134 --- .../clientpositive/avro_timestamp.q.java1.8.out | 134 --- .../results/clientpositive/avro_timestamp.q.out | 132 +++ .../cbo_rp_outer_join_ppr.q.java1.7.out | 693 --- .../clientpositive/cbo_rp_outer_join_ppr.q.out | 691 +++ .../clientpositive/char_udf1.q.java1.7.out | 463 .../clientpositive/char_udf1.q.java1.8.out | 457 --- .../test/results/clientpositive/char_udf1.q.out | 459 +++ .../results/clientpositive/input4.q.java1.7.out | 559 - .../results/clientpositive/input4.q.java1.8.out | 559 - ql/src/test/results/clientpositive/input4.q.out | 555 + .../results/clientpositive/join0.q.java1.7.out | 240 .../results/clientpositive/join0.q.java1.8.out | 240 ql/src/test/results/clientpositive/join0.q.out | 238 .../list_bucket_dml_10.q.java1.7.out| 361 -- .../list_bucket_dml_10.q.java1.8.out| 389 -- .../clientpositive/list_bucket_dml_10.q.out | 359 ++ .../list_bucket_dml_11.q.java1.7.out| 329 - .../list_bucket_dml_11.q.java1.8.out| 424 --- .../clientpositive/list_bucket_dml_11.q.out | 327 + .../list_bucket_dml_12.q.java1.7.out| 426 --- .../list_bucket_dml_12.q.java1.8.out| 596 -- .../clientpositive/list_bucket_dml_12.q.out | 424 +++ .../list_bucket_dml_13.q.java1.7.out| 337 -- .../list_bucket_dml_13.q.java1.8.out| 439 --- .../clientpositive/list_bucket_dml_13.q.out | 335 ++ .../list_bucket_dml_2.q.java1.7.out | 591 - .../list_bucket_dml_2.q.java1.8.out | 692
[08/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.7.out
--
diff --git
a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.7.out
b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.7.out
deleted file mode 100644
index 68943e1..000
--- a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.java1.7.out
+++ /dev/null
@@ -1,709 +0,0 @@
-PREHOOK: query: -- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
-EXPLAIN EXTENDED
- FROM
- src a
- FULL OUTER JOIN
- srcpart b
- ON (a.key = b.key AND b.ds = '2008-04-08')
- SELECT a.key, a.value, b.key, b.value
- WHERE a.key > 10 AND a.key < 20 AND b.key > 15 AND b.key < 25
-PREHOOK: type: QUERY
-POSTHOOK: query: -- SORT_QUERY_RESULTS
--- JAVA_VERSION_SPECIFIC_OUTPUT
-
-EXPLAIN EXTENDED
- FROM
- src a
- FULL OUTER JOIN
- srcpart b
- ON (a.key = b.key AND b.ds = '2008-04-08')
- SELECT a.key, a.value, b.key, b.value
- WHERE a.key > 10 AND a.key < 20 AND b.key > 15 AND b.key < 25
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
-Spark
- Edges:
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Map 3 (PARTITION-LEVEL
SORT, 2)
- A masked pattern was here
- Vertices:
-Map 1
-Map Operator Tree:
-TableScan
- alias: a
- Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
- GatherStats: false
- Select Operator
-expressions: key (type: string), value (type: string)
-outputColumnNames: _col0, _col1
-Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
-Reduce Output Operator
- key expressions: _col0 (type: string)
- null sort order: a
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
- tag: 0
- value expressions: _col1 (type: string)
- auto parallelism: false
-Path -> Alias:
- A masked pattern was here
-Path -> Partition:
- A masked pattern was here
-Partition
- base file name: src
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
-COLUMN_STATS_ACCURATE
{"COLUMN_STATS":{"key":"true","value":"true"},"BASIC_STATS":"true"}
-bucket_count -1
-columns key,value
-columns.comments 'default','default'
-columns.types string:string
- A masked pattern was here
-name default.src
-numFiles 1
-numRows 500
-rawDataSize 5312
-serialization.ddl struct src { string key, string value}
-serialization.format 1
-serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-totalSize 5812
- A masked pattern was here
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
-input format: org.apache.hadoop.mapred.TextInputFormat
-output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-properties:
- COLUMN_STATS_ACCURATE
{"COLUMN_STATS":{"key":"true","value":"true"},"BASIC_STATS":"true"}
- bucket_count -1
- columns key,value
- columns.comments 'default','default'
- columns.types string:string
- A masked pattern was here
- name default.src
- numFiles 1
- numRows 500
- rawDataSize 5312
- serialization.ddl struct src { string key, string value}
- serialization.format 1
- serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
- A masked pattern was here
-serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-name: default.src
- name: default.src
-Truncated Path -> Alias:
- /src [a]
-Map 3
-Map Operator Tree:
-TableScan
- a
[21/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out
--
diff --git a/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out
b/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out
new file mode 100644
index 000..0e11f3f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out
@@ -0,0 +1,424 @@
+PREHOOK: query: -- Ensure it works if skewed column is not the first column in
the table columns
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- test where the skewed values are more than 1 say columns no. 2 and 4 in a
table with 5 columns
+create table list_bucketing_mul_col (col1 String, col2 String, col3 String,
col4 String, col5 string)
+partitioned by (ds String, hr String)
+skewed by (col2, col4) on
(('466','val_466'),('287','val_287'),('82','val_82'))
+stored as DIRECTORIES
+STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@list_bucketing_mul_col
+POSTHOOK: query: -- Ensure it works if skewed column is not the first column
in the table columns
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+
+-- test where the skewed values are more than 1 say columns no. 2 and 4 in a
table with 5 columns
+create table list_bucketing_mul_col (col1 String, col2 String, col3 String,
col4 String, col5 string)
+partitioned by (ds String, hr String)
+skewed by (col2, col4) on
(('466','val_466'),('287','val_287'),('82','val_82'))
+stored as DIRECTORIES
+STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@list_bucketing_mul_col
+PREHOOK: query: -- list bucketing DML
+explain extended
+insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08',
hr = '11')
+select 1, key, 1, value, 1 from src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- list bucketing DML
+explain extended
+insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08',
hr = '11')
+select 1, key, 1, value, 1 from src
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+Map Reduce
+ Map Operator Tree:
+ TableScan
+alias: src
+Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE
Column stats: NONE
+GatherStats: false
+Select Operator
+ expressions: '1' (type: string), key (type: string), '1' (type:
string), value (type: string), '1' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE
Column stats: NONE
+ File Output Operator
+compressed: false
+GlobalTableId: 1
+ A masked pattern was here
+NumFilesPerFileSink: 1
+Static Partition Specification: ds=2008-04-08/hr=11/
+Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ A masked pattern was here
+table:
+input format:
org.apache.hadoop.hive.ql.io.RCFileInputFormat
+output format:
org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+properties:
+ bucket_count -1
+ columns col1,col2,col3,col4,col5
+ columns.comments
+ columns.types string:string:string:string:string
+ A masked pattern was here
+ name default.list_bucketing_mul_col
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_mul_col { string
col1, string col2, string col3, string col4, string col5}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ A masked pattern was here
+serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+name: default.list_bucketing_mul_col
+TotalFiles: 1
+GatherStats: true
+MultiFileSpray: false
+ Path -> Alias:
+ A masked pattern was here
+ Path -> Partition:
+ A masked pattern was here
+ Partition
+base file name: src
+input format: org.apache.hadoop.mapred.TextInputFormat
+output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+properties:
+ COLUMN_STATS_ACCURATE
{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+
[04/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/subquery_multiinsert.q.java1.8.out -- diff --git a/ql/src/test/results/clientpositive/subquery_multiinsert.q.java1.8.out b/ql/src/test/results/clientpositive/subquery_multiinsert.q.java1.8.out deleted file mode 100644 index 899723f..000 --- a/ql/src/test/results/clientpositive/subquery_multiinsert.q.java1.8.out +++ /dev/null @@ -1,999 +0,0 @@ -PREHOOK: query: -- SORT_QUERY_RESULTS --- JAVA_VERSION_SPECIFIC_OUTPUT - -CREATE TABLE src_4( - key STRING, - value STRING -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src_4 -POSTHOOK: query: -- SORT_QUERY_RESULTS --- JAVA_VERSION_SPECIFIC_OUTPUT - -CREATE TABLE src_4( - key STRING, - value STRING -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src_4 -RUN: Stage-0:DDL -PREHOOK: query: CREATE TABLE src_5( - key STRING, - value STRING -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@src_5 -POSTHOOK: query: CREATE TABLE src_5( - key STRING, - value STRING -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@src_5 -RUN: Stage-0:DDL -Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 'Stage-2:MAPRED' is a cross product -PREHOOK: query: explain -from src b -INSERT OVERWRITE TABLE src_4 - select * - where b.key in - (select a.key -from src a -where b.value = a.value and a.key > '9' - ) -INSERT OVERWRITE TABLE src_5 - select * - where b.key not in ( select key from src s1 where s1.key > '2') - order by key -PREHOOK: type: QUERY -POSTHOOK: query: explain -from src b -INSERT OVERWRITE TABLE src_4 - select * - where b.key in - (select a.key -from src a -where b.value = a.value and a.key > '9' - ) -INSERT OVERWRITE TABLE src_5 - select * - where b.key not in ( select key from src s1 where s1.key > '2') - order by key -POSTHOOK: type: QUERY -STAGE DEPENDENCIES: - Stage-10 is a root stage - Stage-2 depends on stages: Stage-10 - Stage-3 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-4 - Stage-5 depends on stages: Stage-1 - Stage-6 depends on stages: Stage-2 - Stage-0 depends on stages: Stage-6 - Stage-7 depends on stages: Stage-0 - -STAGE PLANS: - Stage: Stage-10 -Map Reduce - Map Operator Tree: - TableScan -alias: s1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Filter Operator - predicate: ((key > '2') and key is null) (type: boolean) - Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - Select Operator -Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE -Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -sort order: -Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: bigint) - Reduce Operator Tree: -Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (_col0 = 0) (type: boolean) -Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -Select Operator - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Group By Operator -keys: 0 (type: bigint) -mode: hash -outputColumnNames: _col0 -Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE -File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe - - Stage: Stage-2 -Map Reduce - Map Operator Tree: - TableScan -alias: b -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Reduce Output Operator - sort order: - Statistics: Num rows: 500 Data
[10/27] hive git commit: HIVE-13549: Remove jdk version specific out files from Hive2 (Mohit Sabharwal, reviewed by Sergio Pena)
http://git-wip-us.apache.org/repos/asf/hive/blob/060aa579/ql/src/test/results/clientpositive/outer_join_ppr.q.out
--
diff --git a/ql/src/test/results/clientpositive/outer_join_ppr.q.out
b/ql/src/test/results/clientpositive/outer_join_ppr.q.out
new file mode 100644
index 000..cf20851
--- /dev/null
+++ b/ql/src/test/results/clientpositive/outer_join_ppr.q.out
@@ -0,0 +1,683 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN EXTENDED
+ FROM
+ src a
+ FULL OUTER JOIN
+ srcpart b
+ ON (a.key = b.key AND b.ds = '2008-04-08')
+ SELECT a.key, a.value, b.key, b.value
+ WHERE a.key > 10 AND a.key < 20 AND b.key > 15 AND b.key < 25
+PREHOOK: type: QUERY
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+EXPLAIN EXTENDED
+ FROM
+ src a
+ FULL OUTER JOIN
+ srcpart b
+ ON (a.key = b.key AND b.ds = '2008-04-08')
+ SELECT a.key, a.value, b.key, b.value
+ WHERE a.key > 10 AND a.key < 20 AND b.key > 15 AND b.key < 25
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+Map Reduce
+ Map Operator Tree:
+ TableScan
+alias: a
+Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE
Column stats: NONE
+GatherStats: false
+Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Output Operator
+key expressions: _col0 (type: string)
+null sort order: a
+sort order: +
+Map-reduce partition columns: _col0 (type: string)
+Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+tag: 0
+value expressions: _col1 (type: string)
+auto parallelism: false
+ TableScan
+alias: b
+Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE
Column stats: NONE
+GatherStats: false
+Select Operator
+ expressions: key (type: string), value (type: string), ds (type:
string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+key expressions: _col0 (type: string)
+null sort order: a
+sort order: +
+Map-reduce partition columns: _col0 (type: string)
+Statistics: Num rows: 2000 Data size: 21248 Basic stats:
COMPLETE Column stats: NONE
+tag: 1
+value expressions: _col1 (type: string), _col2 (type: string)
+auto parallelism: false
+ Path -> Alias:
+ A masked pattern was here
+ Path -> Partition:
+ A masked pattern was here
+ Partition
+base file name: src
+input format: org.apache.hadoop.mapred.TextInputFormat
+output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+properties:
+ COLUMN_STATS_ACCURATE
{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+ A masked pattern was here
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+ A masked pattern was here
+serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+COLUMN_STATS_ACCURATE
{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
+bucket_count -1
+columns key,value
+columns.comments 'default','default'
+columns.types string:string
+ A masked pattern was here
+name default.src
+numFiles 1
+numRows 500
+rawDataSize 5312
+serialization.ddl struct src { string key, string value}
+serialization.format 1
+serialization.lib
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+totalSize 5812
+ A masked pattern was here
+ serde
[1/3] hive git commit: HIVE-13249 : Hard upper bound on number of open transactions (Wei Zheng, reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/master 360dfa0ff -> 259e8be1d
http://git-wip-us.apache.org/repos/asf/hive/blob/259e8be1/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
new file mode 100644
index 000..2804e21
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -0,0 +1,1484 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
+import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
+import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
+import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
+import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
+import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
+import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
+import org.apache.hadoop.hive.metastore.api.TxnInfo;
+import org.apache.hadoop.hive.metastore.api.TxnOpenException;
+import org.apache.hadoop.hive.metastore.api.TxnState;
+import org.apache.hadoop.hive.metastore.api.UnlockRequest;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
+
+/**
+ * Tests for TxnHandler.
+ */
+public class TestTxnHandler {
+ static final private String CLASS_NAME = TxnHandler.class.getName();
+ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
+
+ private HiveConf conf = new HiveConf();
+ private TxnStore txnHandler;
+
+ public TestTxnHandler() throws Exception {
+TxnDbUtil.setConfValues(conf);
+LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+Conf
[3/3] hive git commit: HIVE-13249 : Hard upper bound on number of open transactions (Wei Zheng, reviewed by Eugene Koifman)
HIVE-13249 : Hard upper bound on number of open transactions (Wei Zheng,
reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/259e8be1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/259e8be1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/259e8be1
Branch: refs/heads/master
Commit: 259e8be1d4486c6a17b8c240e43154c5a839524e
Parents: 360dfa0
Author: Wei Zheng
Authored: Fri May 20 09:50:44 2016 -0700
Committer: Wei Zheng
Committed: Fri May 20 09:50:44 2016 -0700
--
.../org/apache/hadoop/hive/conf/HiveConf.java |6 +
.../hadoop/hive/metastore/txn/TxnHandler.java | 79 +
.../hadoop/hive/metastore/txn/TxnStore.java |6 +
.../metastore/txn/TestCompactionTxnHandler.java | 466 --
.../hive/metastore/txn/TestTxnHandler.java | 1484 --
.../hive/ql/txn/AcidOpenTxnsCounterService.java | 69 +
.../metastore/txn/TestCompactionTxnHandler.java | 466 ++
.../hive/metastore/txn/TestTxnHandler.java | 1484 ++
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 41 +-
9 files changed, 2150 insertions(+), 1951 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/259e8be1/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 9cc8fbe..4cfa5f1 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1681,6 +1681,12 @@ public class HiveConf extends Configuration {
" of the lock manager is dumped to log file. This is for debugging.
See also " +
"hive.lock.numretries and hive.lock.sleep.between.retries."),
+HIVE_MAX_OPEN_TXNS("hive.max.open.txns", 10, "Maximum number of open
transactions. If \n" +
+"current open transactions reach this limit, future open transaction
requests will be \n" +
+"rejected, until this number goes below the limit."),
+HIVE_COUNT_OPEN_TXNS_INTERVAL("hive.count.open.txns.interval", "1s",
+new TimeValidator(TimeUnit.SECONDS), "Time in seconds between checks
to count open transactions."),
+
HIVE_TXN_MAX_OPEN_BATCH("hive.txn.max.open.batch", 1000,
"Maximum number of transactions that can be fetched in one call to
open_txns().\n" +
"This controls how many transactions streaming agents such as Flume or
Storm open\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/259e8be1/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index abaff34..82d685d 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -28,6 +28,7 @@ import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.hive.common.ServerUtils;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.HouseKeeperService;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -167,6 +168,15 @@ abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
}
}
+ // Maximum number of open transactions that's allowed
+ private static volatile int maxOpenTxns = 0;
+ // Current number of open txns
+ private static volatile long numOpenTxns = 0;
+ // Whether number of open transactions reaches the threshold
+ private static volatile boolean tooManyOpenTxns = false;
+ // The AcidHouseKeeperService for counting open transactions
+ private static volatile HouseKeeperService openTxnsCounter = null;
+
/**
* Number of consecutive deadlocks we have seen
*/
@@ -236,6 +246,7 @@ abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
TimeUnit.MILLISECONDS);
retryLimit = HiveConf.getIntVar(conf,
HiveConf.ConfVars.HMSHANDLERATTEMPTS);
deadlockRetryInterval = retryInterval / 10;
+maxOpenTxns = HiveConf.getIntVar(conf,
HiveConf.ConfVars.HIVE_MAX_OPEN_TXNS);
}
public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException {
@@ -362,7 +373,45 @@ abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
return getOpenTxns();
}
}
+
+ private static void startHouseKeeperService(HiveConf conf, Class c){
+tr
[2/3] hive git commit: HIVE-13249 : Hard upper bound on number of open transactions (Wei Zheng, reviewed by Eugene Koifman)
http://git-wip-us.apache.org/repos/asf/hive/blob/259e8be1/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
--
diff --git
a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
deleted file mode 100644
index 2804e21..000
---
a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ /dev/null
@@ -1,1484 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
-import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
-import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
-import org.apache.hadoop.hive.metastore.api.CompactionRequest;
-import org.apache.hadoop.hive.metastore.api.CompactionType;
-import org.apache.hadoop.hive.metastore.api.DataOperationType;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
-import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
-import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeRequest;
-import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
-import org.apache.hadoop.hive.metastore.api.LockComponent;
-import org.apache.hadoop.hive.metastore.api.LockLevel;
-import org.apache.hadoop.hive.metastore.api.LockRequest;
-import org.apache.hadoop.hive.metastore.api.LockResponse;
-import org.apache.hadoop.hive.metastore.api.LockState;
-import org.apache.hadoop.hive.metastore.api.LockType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
-import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
-import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
-import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
-import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
-import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
-import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
-import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
-import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
-import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
-import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
-import org.apache.hadoop.hive.metastore.api.TxnInfo;
-import org.apache.hadoop.hive.metastore.api.TxnOpenException;
-import org.apache.hadoop.hive.metastore.api.TxnState;
-import org.apache.hadoop.hive.metastore.api.UnlockRequest;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.core.LoggerContext;
-import org.apache.logging.log4j.core.config.Configuration;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertNull;
-import static junit.framework.Assert.assertTrue;
-import static junit.framework.Assert.fail;
-
-/**
- * Tests for TxnHandler.
- */
-public class TestTxnHandler {
- static final private String CLASS_NAME = TxnHandler.class.getName();
- private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
-
- private HiveConf conf = new HiveConf();
- private TxnStore txnHandler;
-
- public TestTxnHandler() throws Exception {
-TxnDbUtil.setConfValues(conf);
-LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
-Configuration conf = ctx.getConfiguration();
-c
[1/2] hive git commit: HIVE-13249 : Hard upper bound on number of open transactions (Wei Zheng, reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/branch-1 5fe252b93 -> cb3636f3f
http://git-wip-us.apache.org/repos/asf/hive/blob/cb3636f3/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidOpenTxnsCounterService.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidOpenTxnsCounterService.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidOpenTxnsCounterService.java
new file mode 100644
index 000..f5eb8a1
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidOpenTxnsCounterService.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.txn;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.ql.txn.compactor.HouseKeeperServiceBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+/**
+ * Background running thread, periodically updating number of open
transactions.
+ * Runs inside Hive Metastore Service.
+ */
+public class AcidOpenTxnsCounterService extends HouseKeeperServiceBase {
+ private static final Logger LOG =
LoggerFactory.getLogger(AcidOpenTxnsCounterService.class);
+ @Override
+ protected long getStartDelayMs() {
+return 100; // in miliseconds
+ }
+ @Override
+ protected long getIntervalMs() {
+return
hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_COUNT_OPEN_TXNS_INTERVAL,
TimeUnit.MILLISECONDS);
+ }
+ @Override
+ protected Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger
isAliveCounter) {
+return new OpenTxnsCounter(hiveConf, isAliveCounter);
+ }
+ @Override
+ public String getServiceDescription() {
+return "Count number of open transactions";
+ }
+ private static final class OpenTxnsCounter implements Runnable {
+private final TxnStore txnHandler;
+private final AtomicInteger isAliveCounter;
+private OpenTxnsCounter(HiveConf hiveConf, AtomicInteger isAliveCounter) {
+ txnHandler = TxnUtils.getTxnStore(hiveConf);
+ this.isAliveCounter = isAliveCounter;
+}
+@Override
+public void run() {
+ try {
+long startTime = System.currentTimeMillis();
+txnHandler.countOpenTxns();
+int count = isAliveCounter.incrementAndGet();
+LOG.info("OpenTxnsCounter ran for " + (System.currentTimeMillis() -
startTime)/1000 + "seconds. isAliveCounter=" + count);
+ }
+ catch(Throwable t) {
+LOG.error("Serious error in {}", Thread.currentThread().getName(), ":
{}" + t.getMessage(), t);
+ }
+}
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/cb3636f3/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
new file mode 100644
index 000..23ad54e
--- /dev/null
+++
b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
@@ -0,0 +1,447 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hi
[2/2] hive git commit: HIVE-13249 : Hard upper bound on number of open transactions (Wei Zheng, reviewed by Eugene Koifman)
HIVE-13249 : Hard upper bound on number of open transactions (Wei Zheng,
reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb3636f3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb3636f3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb3636f3
Branch: refs/heads/branch-1
Commit: cb3636f3fe3e45744eed23a542de05f77a3dd356
Parents: 5fe252b
Author: Wei Zheng
Authored: Fri May 20 10:25:07 2016 -0700
Committer: Wei Zheng
Committed: Fri May 20 10:25:07 2016 -0700
--
.../org/apache/hadoop/hive/conf/HiveConf.java |6 +
.../hadoop/hive/metastore/txn/TxnHandler.java | 77 +
.../hadoop/hive/metastore/txn/TxnStore.java |6 +
.../metastore/txn/TestCompactionTxnHandler.java | 447 -
.../hive/metastore/txn/TestTxnHandler.java | 1521 --
.../hive/ql/txn/AcidOpenTxnsCounterService.java | 69 +
.../metastore/txn/TestCompactionTxnHandler.java | 447 +
.../hive/metastore/txn/TestTxnHandler.java | 1521 ++
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 41 +-
9 files changed, 2166 insertions(+), 1969 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/cb3636f3/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4c6aa71..c63c2ca 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1485,6 +1485,12 @@ public class HiveConf extends Configuration {
" of the lock manager is dumped to log file. This is for debugging.
See also " +
"hive.lock.numretries and hive.lock.sleep.between.retries."),
+HIVE_MAX_OPEN_TXNS("hive.max.open.txns", 10, "Maximum number of open
transactions. If \n" +
+"current open transactions reach this limit, future open transaction
requests will be \n" +
+"rejected, until this number goes below the limit."),
+HIVE_COUNT_OPEN_TXNS_INTERVAL("hive.count.open.txns.interval", "1s",
+new TimeValidator(TimeUnit.SECONDS), "Time in seconds between checks
to count open transactions."),
+
HIVE_TXN_MAX_OPEN_BATCH("hive.txn.max.open.batch", 1000,
"Maximum number of transactions that can be fetched in one call to
open_txns().\n" +
"This controls how many transactions streaming agents such as Flume or
Storm open\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/cb3636f3/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
--
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 4da5542..27fa820 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.ValidReadTxnList;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HouseKeeperService;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -169,6 +170,15 @@ abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
}
}
+ // Maximum number of open transactions that's allowed
+ private static volatile int maxOpenTxns = 0;
+ // Current number of open txns
+ private static volatile long numOpenTxns = 0;
+ // Whether number of open transactions reaches the threshold
+ private static volatile boolean tooManyOpenTxns = false;
+ // The AcidHouseKeeperService for counting open transactions
+ private static volatile HouseKeeperService openTxnsCounter = null;
+
/**
* Number of consecutive deadlocks we have seen
*/
@@ -234,6 +244,7 @@ abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
TimeUnit.MILLISECONDS);
retryLimit = HiveConf.getIntVar(conf,
HiveConf.ConfVars.HMSHANDLERATTEMPTS);
deadlockRetryInterval = retryInterval / 10;
+maxOpenTxns = HiveConf.getIntVar(conf,
HiveConf.ConfVars.HIVE_MAX_OPEN_TXNS);
}
public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException {
@@ -383,7 +394,43 @@ abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
return new ValidReadTxnList(exceptions, highWater);
}
+ private static void startHouseKeeperService(HiveConf conf, Class c)
hive git commit: HIVE-13783: Display a secondary prompt on beeline for multi-line statements (Vihang Karajgaonkar, reviewed by Reuben Kuhnert and Sergio Pena)
Repository: hive
Updated Branches:
refs/heads/master 259e8be1d -> fd06601eb
HIVE-13783: Display a secondary prompt on beeline for multi-line statements
(Vihang Karajgaonkar, reviewed by Reuben Kuhnert and Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fd06601e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fd06601e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fd06601e
Branch: refs/heads/master
Commit: fd06601eb81b39e3ca4d5604ec4f2aebc91c5c5b
Parents: 259e8be
Author: Vihang Karajgaonkar
Authored: Fri May 20 12:57:41 2016 -0500
Committer: Sergio Pena
Committed: Fri May 20 12:59:58 2016 -0500
--
beeline/src/java/org/apache/hive/beeline/Commands.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/fd06601e/beeline/src/java/org/apache/hive/beeline/Commands.java
--
diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java
b/beeline/src/java/org/apache/hive/beeline/Commands.java
index 32c1275..80703ff 100644
--- a/beeline/src/java/org/apache/hive/beeline/Commands.java
+++ b/beeline/src/java/org/apache/hive/beeline/Commands.java
@@ -1033,8 +1033,8 @@ public class Commands {
while (beeLine.getConsoleReader() != null && !(line.trim().endsWith(";"))
&& beeLine.getOpts()
.isAllowMultiLineCommand()) {
+ StringBuilder prompt = new StringBuilder(beeLine.getPrompt());
if (!beeLine.getOpts().isSilent()) {
-StringBuilder prompt = new StringBuilder(beeLine.getPrompt());
for (int i = 0; i < prompt.length() - 1; i++) {
if (prompt.charAt(i) != '>') {
prompt.setCharAt(i, i % 2 == 0 ? '.' : ' ');
@@ -1046,7 +1046,7 @@ public class Commands {
if (beeLine.getOpts().isSilent() && beeLine.getOpts().getScriptFile() !=
null) {
extra = beeLine.getConsoleReader().readLine(null,
jline.console.ConsoleReader.NULL_MASK);
} else {
-extra = beeLine.getConsoleReader().readLine(beeLine.getPrompt());
+extra = beeLine.getConsoleReader().readLine(prompt.toString());
}
if (extra == null) { //it happens when using -f and the line of cmds
does not end with ;
[27/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j) Fixes #72. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ffb79509 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ffb79509 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ffb79509 Branch: refs/heads/master Commit: ffb79509bcaefb9e7f916930edb022371b9d810f Parents: fd06601 Author: Owen O'Malley Authored: Fri May 20 14:20:00 2016 -0700 Committer: Owen O'Malley Committed: Fri May 20 14:21:39 2016 -0700 -- bin/ext/orcfiledump.cmd |2 +- bin/ext/orcfiledump.sh |2 +- .../hive/hcatalog/streaming/TestStreaming.java |9 +- .../llap/io/decode/OrcEncodedDataConsumer.java |2 +- .../llap/io/encoded/OrcEncodedDataReader.java | 15 +- .../hive/llap/io/metadata/OrcFileMetadata.java |4 +- orc/pom.xml | 27 + .../org/apache/orc/FileFormatException.java | 30 + orc/src/java/org/apache/orc/OrcFile.java|6 + orc/src/java/org/apache/orc/Reader.java |2 +- .../java/org/apache/orc/TypeDescription.java| 18 +- orc/src/java/org/apache/orc/impl/AcidStats.java | 60 + .../orc/impl/ConvertTreeReaderFactory.java | 2840 + .../java/org/apache/orc/impl/HadoopShims.java | 79 + .../org/apache/orc/impl/HadoopShimsCurrent.java | 30 + .../org/apache/orc/impl/HadoopShims_2_2.java| 71 +- .../java/org/apache/orc/impl/IntegerReader.java |3 +- .../java/org/apache/orc/impl/OrcAcidUtils.java | 85 + .../java/org/apache/orc/impl/ReaderImpl.java| 758 .../org/apache/orc/impl/RecordReaderImpl.java | 1215 ++ .../org/apache/orc/impl/RecordReaderUtils.java | 578 +++ .../org/apache/orc/impl/SchemaEvolution.java| 190 + .../org/apache/orc/impl/TreeReaderFactory.java | 2093 ++ .../java/org/apache/orc/impl/ZeroCopyShims.java | 89 + orc/src/java/org/apache/orc/tools/FileDump.java | 934 + .../java/org/apache/orc/tools/JsonFileDump.java | 406 ++ .../org/apache/orc/TestColumnStatistics.java| 364 ++ .../org/apache/orc/TestNewIntegerEncoding.java | 1373 +++ .../org/apache/orc/TestOrcNullOptimization.java | 415 ++ .../test/org/apache/orc/TestOrcTimezone1.java | 189 + .../test/org/apache/orc/TestOrcTimezone2.java | 143 + .../org/apache/orc/TestStringDictionary.java| 290 ++ .../org/apache/orc/TestTypeDescription.java | 68 + .../org/apache/orc/TestUnrolledBitPack.java | 114 + .../test/org/apache/orc/TestVectorOrcFile.java | 2782 + .../org/apache/orc/impl/TestOrcWideTable.java | 64 + orc/src/test/org/apache/orc/impl/TestRLEv2.java | 307 ++ .../org/apache/orc/impl/TestReaderImpl.java | 152 + .../apache/orc/impl/TestRecordReaderImpl.java | 1691 .../org/apache/orc/impl/TestStreamName.java | 49 + .../test/org/apache/orc/tools/TestFileDump.java | 486 +++ .../org/apache/orc/tools/TestJsonFileDump.java | 150 + orc/src/test/resources/orc-file-11-format.orc | Bin 0 -> 373336 bytes .../resources/orc-file-dump-bloomfilter.out | 179 + .../resources/orc-file-dump-bloomfilter2.out| 179 + .../orc-file-dump-dictionary-threshold.out | 190 + orc/src/test/resources/orc-file-dump.json | 1355 +++ orc/src/test/resources/orc-file-dump.out| 195 + orc/src/test/resources/orc-file-has-null.out| 112 + .../expressions/CastDecimalToTimestamp.java |8 +- .../expressions/CastDoubleToTimestamp.java | 13 +- .../vector/expressions/CastLongToTimestamp.java |4 +- .../CastMillisecondsLongToTimestamp.java|7 +- .../ql/exec/vector/expressions/StringExpr.java | 354 -- .../hive/ql/hooks/PostExecOrcFileDump.java |4 +- .../hadoop/hive/ql/io/FileFormatException.java | 30 - .../ql/io/orc/ConvertTreeReaderFactory.java | 3750 -- .../apache/hadoop/hive/ql/io/orc/FileDump.java | 884 - .../hadoop/hive/ql/io/orc/JsonFileDump.java | 401 -- .../hive/ql/io/orc/OrcRawRecordMerger.java | 35 +- .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 71 +- .../hadoop/hive/ql/io/orc/ReaderImpl.java | 509 +-- .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 1823 - .../hive/ql/io/orc/RecordReaderUtils.java | 586 --- .../hadoop/hive/ql/io/orc/SchemaEvolution.java | 190 - .../hive/ql/io/orc/TreeReaderFactory.java | 2525 .../ql/io/orc/encoded/EncodedReaderImpl.java|2 +- .../orc/encoded/EncodedTreeReaderFactory.java |2 +- .../apache/hadoop/hive/ql/TestTxnCommands.java |2 - .../TestTimestampWritableAndColumnVector.java |7 +- .../vector/expressions/TestVectorTypeCasts.java | 10 +- .../exec/vecto
[12/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
index 2199b11..e46ca51 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
@@ -18,1218 +18,923 @@
package org.apache.hadoop.hive.ql.io.orc;
import java.io.IOException;
-import java.math.BigDecimal;
-import java.sql.Date;
-import java.sql.Timestamp;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.orc.BooleanColumnStatistics;
-import org.apache.orc.impl.BufferChunk;
-import org.apache.orc.ColumnStatistics;
-import org.apache.orc.impl.ColumnStatisticsImpl;
-import org.apache.orc.CompressionCodec;
-import org.apache.orc.DataReader;
-import org.apache.orc.DateColumnStatistics;
-import org.apache.orc.DecimalColumnStatistics;
-import org.apache.orc.DoubleColumnStatistics;
-import org.apache.orc.impl.DataReaderProperties;
-import org.apache.orc.impl.InStream;
-import org.apache.orc.IntegerColumnStatistics;
-import org.apache.orc.OrcConf;
-import org.apache.orc.impl.OrcIndex;
-import org.apache.orc.impl.PositionProvider;
-import org.apache.orc.impl.StreamName;
-import org.apache.orc.StringColumnStatistics;
-import org.apache.orc.StripeInformation;
-import org.apache.orc.TimestampColumnStatistics;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.io.DiskRange;
-import org.apache.hadoop.hive.common.io.DiskRangeList;
-import org.apache.hadoop.hive.common.io.DiskRangeList.CreateHelper;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.orc.BloomFilterIO;
-import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-import org.apache.orc.OrcProto;
+import org.apache.orc.TypeDescription;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-public class RecordReaderImpl implements RecordReader {
+public class RecordReaderImpl extends org.apache.orc.impl.RecordReaderImpl
+ implements RecordReader {
static final Logger LOG = LoggerFactory.getLogger(RecordReaderImpl.class);
- private static final boolean isLogDebugEnabled = LOG.isDebugEnabled();
- private static final Object UNKNOWN_VALUE = new Object();
- private final Path path;
- private final long firstRow;
- private final List stripes =
- new ArrayList();
- private OrcProto.StripeFooter stripeFooter;
- private final long totalRowCount;
- private final CompressionCodec codec;
- private final List types;
- private final int bufferSize;
- private final boolean[] included;
- private final long rowIndexStride;
- private long rowInStripe = 0;
- private int currentStripe = -1;
- private long rowBaseInStripe = 0;
- private long rowCountInStripe = 0;
- private final Map streams =
- new HashMap();
- DiskRangeList bufferChunks = null;
- private final TreeReaderFactory.TreeReader reader;
- private final OrcProto.RowIndex[] indexes;
-
[01/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
Repository: hive
Updated Branches:
refs/heads/master fd06601eb -> ffb79509b
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
--
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
index 932ae0b..6415bf8 100644
---
a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
@@ -27,6 +27,7 @@ import java.sql.Timestamp;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.hive.ql.util.TimestampUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.common.type.HiveChar;
@@ -1088,13 +1089,13 @@ public final class PrimitiveObjectInspectorUtils {
result = TimestampWritable.longToTimestamp(longValue,
intToTimestampInSeconds);
break;
case FLOAT:
- result = TimestampWritable.doubleToTimestamp(((FloatObjectInspector)
inputOI).get(o));
+ result = TimestampUtils.doubleToTimestamp(((FloatObjectInspector)
inputOI).get(o));
break;
case DOUBLE:
- result = TimestampWritable.doubleToTimestamp(((DoubleObjectInspector)
inputOI).get(o));
+ result = TimestampUtils.doubleToTimestamp(((DoubleObjectInspector)
inputOI).get(o));
break;
case DECIMAL:
- result =
TimestampWritable.decimalToTimestamp(((HiveDecimalObjectInspector) inputOI)
+ result = TimestampUtils.decimalToTimestamp(((HiveDecimalObjectInspector)
inputOI)
.getPrimitiveJavaObject(o));
break;
case STRING:
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java
--
diff --git
a/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java
b/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java
index 6c763bc..7619efa 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritable.java
@@ -35,6 +35,7 @@ import java.util.List;
import java.util.Random;
import java.util.TimeZone;
+import org.apache.hadoop.hive.ql.util.TimestampUtils;
import org.junit.*;
import static org.junit.Assert.*;
@@ -70,7 +71,7 @@ public class TestTimestampWritable {
long seconds = (ts.getTime() - ts.getNanos() / 100) / 1000;
// It should also be possible to calculate this based on ts.getTime() only.
-assertEquals(seconds, TimestampWritable.millisToSeconds(ts.getTime()));
+assertEquals(seconds, TimestampUtils.millisToSeconds(ts.getTime()));
return seconds;
}
@@ -335,10 +336,10 @@ public class TestTimestampWritable {
Math.pow(10, 9 - nanosPrecision));
assertEquals(String.format("Invalid nanosecond part recovered from
%f", asDouble),
nanos, recoveredNanos);
-assertEquals(ts, TimestampWritable.doubleToTimestamp(asDouble));
+assertEquals(ts, TimestampUtils.doubleToTimestamp(asDouble));
// decimalToTimestamp should be consistent with doubleToTimestamp for
this level of
// precision.
-assertEquals(ts, TimestampWritable.decimalToTimestamp(
+assertEquals(ts, TimestampUtils.decimalToTimestamp(
HiveDecimal.create(BigDecimal.valueOf(asDouble;
}
}
@@ -358,7 +359,7 @@ public class TestTimestampWritable {
Timestamp ts = new Timestamp(
randomMillis(MIN_FOUR_DIGIT_YEAR_MILLIS, MAX_FOUR_DIGIT_YEAR_MILLIS,
rand));
ts.setNanos(randomNanos(rand, 9)); // full precision
- assertEquals(ts,
TimestampWritable.decimalToTimestamp(timestampToDecimal(ts)));
+ assertEquals(ts,
TimestampUtils.decimalToTimestamp(timestampToDecimal(ts)));
}
}
@@ -371,8 +372,8 @@ public class TestTimestampWritable {
for (int nanos : new int[] { 10, 90, 99910, 0 }) {
ts.setNanos(nanos);
HiveDecimal d = timestampToDecimal(ts);
- assertEquals(ts, TimestampWritable.decimalToTimestamp(d));
- assertEquals(ts,
TimestampWritable.doubleToTimestamp(d.bigDecimalValue().doubleValue()));
+ assertEquals(ts, TimestampUtils.decimalToTimestamp(d));
+ assertEquals(ts,
TimestampUtils.doubleToTimestamp(d.bigDecimalValue().doubleValue()));
}
}
@@ -435,20 +436,20 @@ public class TestTimestampWritable {
@Concurrent(count=4)
@Repeating(repetition=100)
public void testMillisToSeconds() {
-assertE
[03/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/resources/orc-file-dump-bloomfilter.out -- diff --git a/ql/src/test/resources/orc-file-dump-bloomfilter.out b/ql/src/test/resources/orc-file-dump-bloomfilter.out deleted file mode 100644 index 18fd2fb..000 --- a/ql/src/test/resources/orc-file-dump-bloomfilter.out +++ /dev/null @@ -1,179 +0,0 @@ -Structure for TestFileDump.testDump.orc -File Version: 0.12 with HIVE_13083 -Rows: 21000 -Compression: ZLIB -Compression size: 4096 -Type: struct - -Stripe Statistics: - Stripe 1: -Column 0: count: 5000 hasNull: false -Column 1: count: 5000 hasNull: false min: -2146021688 max: 2147223299 sum: 515792826 -Column 2: count: 5000 hasNull: false min: -9218592812243954469 max: 9221614132680747961 -Column 3: count: 5000 hasNull: false min: Darkness, max: worst sum: 19280 - Stripe 2: -Column 0: count: 5000 hasNull: false -Column 1: count: 5000 hasNull: false min: -2146733128 max: 2147001622 sum: 7673427 -Column 2: count: 5000 hasNull: false min: -9220818777591257749 max: 959462014003839 -Column 3: count: 5000 hasNull: false min: Darkness, max: worst sum: 19504 - Stripe 3: -Column 0: count: 5000 hasNull: false -Column 1: count: 5000 hasNull: false min: -2146993718 max: 2147378179 sum: 132660742551 -Column 2: count: 5000 hasNull: false min: -9218342074710552826 max: 9222303228623055266 -Column 3: count: 5000 hasNull: false min: Darkness, max: worst sum: 19641 - Stripe 4: -Column 0: count: 5000 hasNull: false -Column 1: count: 5000 hasNull: false min: -2146658006 max: 2145520931 sum: 8533549236 -Column 2: count: 5000 hasNull: false min: -9222758097219661129 max: 9221043130193737406 -Column 3: count: 5000 hasNull: false min: Darkness, max: worst sum: 19470 - Stripe 5: -Column 0: count: 1000 hasNull: false -Column 1: count: 1000 hasNull: false min: -2146245500 max: 2146378640 sum: 51299706363 -Column 2: count: 1000 hasNull: false min: -9208193203370316142 max: 9218567213558056476 -Column 3: count: 1000 hasNull: false min: Darkness, max: worst sum: 3866 - -File Statistics: - Column 0: count: 21000 hasNull: false - Column 1: count: 21000 hasNull: false min: -2146993718 max: 2147378179 sum: 193017464403 - Column 2: count: 21000 hasNull: false min: -9222758097219661129 max: 9222303228623055266 - Column 3: count: 21000 hasNull: false min: Darkness, max: worst sum: 81761 - -Stripes: - Stripe: offset: 3 data: 63786 rows: 5000 tail: 86 index: 951 -Stream: column 0 section ROW_INDEX start: 3 length 17 -Stream: column 1 section ROW_INDEX start: 20 length 166 -Stream: column 2 section ROW_INDEX start: 186 length 169 -Stream: column 3 section ROW_INDEX start: 355 length 87 -Stream: column 3 section BLOOM_FILTER start: 442 length 512 -Stream: column 1 section DATA start: 954 length 20035 -Stream: column 2 section DATA start: 20989 length 40050 -Stream: column 3 section DATA start: 61039 length 3543 -Stream: column 3 section LENGTH start: 64582 length 25 -Stream: column 3 section DICTIONARY_DATA start: 64607 length 133 -Encoding column 0: DIRECT -Encoding column 1: DIRECT_V2 -Encoding column 2: DIRECT_V2 -Encoding column 3: DICTIONARY_V2[35] -Row group indices for column 3: - Entry 0: count: 1000 hasNull: false min: Darkness, max: worst sum: 3862 positions: 0,0,0 - Entry 1: count: 1000 hasNull: false min: Darkness, max: worst sum: 3884 positions: 0,659,149 - Entry 2: count: 1000 hasNull: false min: Darkness, max: worst sum: 3893 positions: 0,1531,3 - Entry 3: count: 1000 hasNull: false min: Darkness, max: worst sum: 3798 positions: 0,2281,32 - Entry 4: count: 1000 hasNull: false min: Darkness, max: worst sum: 3843 positions: 0,3033,45 -Bloom filters for column 3: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 138 loadFactor: 0.022 expectedFpp: 2.343647E-7 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 138 loadFactor: 0.022 expectedFpp: 2.343647E-7 - Entry 2: numHashFunctions: 4 bitCount: 6272 popCount: 138 loadFactor: 0.022 expectedFpp: 2.343647E-7 - Entry 3: numHashFunctions: 4 bitCount: 6272 popCount: 138 loadFactor: 0.022 expectedFpp: 2.343647E-7 - Entry 4: numHashFunctions: 4 bitCount: 6272 popCount: 138 loadFactor: 0.022 expectedFpp: 2.343647E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 138 loadFactor: 0.022 expectedFpp: 2.343647E-7 - Stripe: offset: 64826 data: 63775 rows: 5000 tail: 86 index: 944 -Stream: column 0 section ROW_INDEX start: 64826 length 17 -Stream: column 1 section ROW_INDEX start: 64843 length 164 -Stream: column 2 section ROW_INDEX start: 65007 length 168 -Stream: column 3 section ROW_INDEX start: 65175 length 83 -Stream: column 3 section BLOOM_FILTER start: 65258 length 512 -Stre
[21/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/test/org/apache/orc/TestNewIntegerEncoding.java
--
diff --git a/orc/src/test/org/apache/orc/TestNewIntegerEncoding.java
b/orc/src/test/org/apache/orc/TestNewIntegerEncoding.java
new file mode 100644
index 000..526dd81
--- /dev/null
+++ b/orc/src/test/org/apache/orc/TestNewIntegerEncoding.java
@@ -0,0 +1,1373 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import static junit.framework.Assert.assertEquals;
+
+import java.io.File;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Longs;
+
+@RunWith(value = Parameterized.class)
+public class TestNewIntegerEncoding {
+
+ private OrcFile.EncodingStrategy encodingStrategy;
+
+ public TestNewIntegerEncoding( OrcFile.EncodingStrategy es) {
+this.encodingStrategy = es;
+ }
+
+ @Parameters
+ public static Collection data() {
+Object[][] data = new Object[][] { { OrcFile.EncodingStrategy.COMPRESSION
},
+{ OrcFile.EncodingStrategy.SPEED } };
+return Arrays.asList(data);
+ }
+
+ public static class TSRow {
+Timestamp ts;
+
+public TSRow(Timestamp ts) {
+ this.ts = ts;
+}
+ }
+
+ public static TypeDescription getRowSchema() {
+return TypeDescription.createStruct()
+.addField("int1", TypeDescription.createInt())
+.addField("long1", TypeDescription.createLong());
+ }
+
+ public static void appendRow(VectorizedRowBatch batch,
+ int int1, long long1) {
+int row = batch.size++;
+((LongColumnVector) batch.cols[0]).vector[row] = int1;
+((LongColumnVector) batch.cols[1]).vector[row] = long1;
+ }
+
+ public static void appendLong(VectorizedRowBatch batch,
+long long1) {
+int row = batch.size++;
+((LongColumnVector) batch.cols[0]).vector[row] = long1;
+ }
+
+ Path workDir = new Path(System.getProperty("test.tmp.dir", "target"
+ + File.separator + "test" + File.separator + "tmp"));
+
+ Configuration conf;
+ FileSystem fs;
+ Path testFilePath;
+
+ @Rule
+ public TestName testCaseName = new TestName();
+
+ @Before
+ public void openFileSystem() throws Exception {
+conf = new Configuration();
+fs = FileSystem.getLocal(conf);
+testFilePath = new Path(workDir, "TestOrcFile."
++ testCaseName.getMethodName() + ".orc");
+fs.delete(testFilePath, false);
+ }
+
+ @Test
+ public void testBasicRow() throws Exception {
+TypeDescription schema= getRowSchema();
+Writer writer = OrcFile.createWriter(testFilePath,
+ OrcFile.writerOptions(conf)
+ .setSchema(schema)
+ .stripeSize(10)
+ .compress(CompressionKind.NONE)
+ .bufferSize(1)
+ .encodingStrategy(encodingStrategy));
+VectorizedRowBatch batch = schema.createRowBatch();
+appendRow(batch, 111, L);
+appendRow(batch, 111, L);
+appendRow(batch, 111, L);
+writer.addRowBatch(batch);
+writer.close();
+
+Reader reader = OrcFile.createReader(testFilePath,
+OrcFile.readerOptions(conf).filesystem(fs));
+RecordReader rows = reader.rows();
+batch = reader.getSchema().createRowBatch();
+while (rows.nextBatch(batch)) {
+
[18/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/test/org/apache/orc/impl/TestOrcWideTable.java
--
diff --git a/orc/src/test/org/apache/orc/impl/TestOrcWideTable.java
b/orc/src/test/org/apache/orc/impl/TestOrcWideTable.java
new file mode 100644
index 000..289a86e
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestOrcWideTable.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+public class TestOrcWideTable {
+
+ @Test
+ public void testBufferSizeFor1Col() throws IOException {
+assertEquals(128 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 *
1024,
+1, 128*1024));
+ }
+
+ @Test
+ public void testBufferSizeFor50Col() throws IOException {
+assertEquals(256 * 1024, WriterImpl.getEstimatedBufferSize(256 * 1024 *
1024,
+50, 256*1024));
+ }
+
+ @Test
+ public void testBufferSizeFor1000Col() throws IOException {
+assertEquals(32 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 *
1024,
+1000, 128*1024));
+ }
+
+ @Test
+ public void testBufferSizeFor2000Col() throws IOException {
+assertEquals(16 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 *
1024,
+2000, 256*1024));
+ }
+
+ @Test
+ public void testBufferSizeFor4000Col() throws IOException {
+assertEquals(8 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 * 1024,
+4000, 256*1024));
+ }
+
+ @Test
+ public void testBufferSizeFor25000Col() throws IOException {
+assertEquals(4 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 * 1024,
+25000, 256*1024));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/test/org/apache/orc/impl/TestRLEv2.java
--
diff --git a/orc/src/test/org/apache/orc/impl/TestRLEv2.java
b/orc/src/test/org/apache/orc/impl/TestRLEv2.java
new file mode 100644
index 000..e139619
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestRLEv2.java
@@ -0,0 +1,307 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.PrintStream;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcFile;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.apache.orc.tools.FileDump;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+public class TestRLEv2 {
+ Path workDir = new Path(System.getProperty("test.tmp.dir",
+ "target" + File.separator + "test" + File.separator + "tmp"));
+ Path testFilePath;
+ Configuration conf;
+ FileSystem fs;
+
+ @Rule
+ public TestName testCaseName = new TestName();
+
+ @Before
+ public void openFileSystem () throws Exception {
+conf = new Configuration();
+fs = FileSystem.getLocal(conf);
+testFilePath = new Pat
[05/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
deleted file mode 100644
index 41a211b..000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestStringDictionary.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.io.Text;
-import org.apache.orc.CompressionKind;
-import org.apache.orc.OrcProto;
-
-import org.apache.orc.StripeInformation;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-public class TestStringDictionary {
-
- Path workDir = new Path(System.getProperty("test.tmp.dir", "target" +
File.separator + "test"
- + File.separator + "tmp"));
-
- Configuration conf;
- FileSystem fs;
- Path testFilePath;
-
- @Rule
- public TestName testCaseName = new TestName();
-
- @Before
- public void openFileSystem() throws Exception {
-conf = new Configuration();
-fs = FileSystem.getLocal(conf);
-testFilePath = new Path(workDir, "TestOrcFile." +
testCaseName.getMethodName() + ".orc");
-fs.delete(testFilePath, false);
- }
-
- @Test
- public void testTooManyDistinct() throws Exception {
-ObjectInspector inspector;
-synchronized (TestOrcFile.class) {
- inspector =
ObjectInspectorFactory.getReflectionObjectInspector(Text.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
-}
-
-Writer writer = OrcFile.createWriter(
-testFilePath,
-
OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.NONE)
-.bufferSize(1));
-for (int i = 0; i < 2; i++) {
- writer.addRow(new Text(String.valueOf(i)));
-}
-writer.close();
-
-Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
-RecordReader rows = reader.rows();
-int idx = 0;
-while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(new Text(String.valueOf(idx++)), row);
-}
-
-// make sure the encoding type is correct
-for (StripeInformation stripe : reader.getStripes()) {
- // hacky but does the job, this casting will work as long this test
resides
- // within the same package as ORC reader
- OrcProto.StripeFooter footer = ((RecordReaderImpl)
rows).readStripeFooter(stripe);
- for (int i = 0; i < footer.getColumnsCount(); ++i) {
-OrcProto.ColumnEncoding encoding = footer.getColumns(i);
-assertEquals(OrcProto.ColumnEncoding.Kind.DIRECT_V2,
encoding.getKind());
- }
-}
- }
-
- @Test
- public void testHalfDistinct() throws Exception {
-ObjectInspector inspector;
-synchronized (TestOrcFile.class) {
- inspector =
ObjectInspectorFactory.getReflectionObjectInspector(Text.class,
- ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
-}
-
-Writer writer = OrcFile.createWriter(
-testFilePath,
-
OrcFile.writerOptions(conf).inspector(inspector).compress(CompressionKind.NONE)
-.bufferSize(1));
-Random rand = new Random(123);
-int[] input = new int[2];
-for (int i = 0; i < 2; i++) {
- input[i] = rand.nextInt(1);
-}
-
-for (int i = 0; i < 2; i++) {
- writer.addRow(new Text(String.valueOf(input[i])));
-}
-writer.close();
-
-Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
-RecordReader rows = r
[23/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/java/org/apache/orc/impl/TreeReaderFactory.java
--
diff --git a/orc/src/java/org/apache/orc/impl/TreeReaderFactory.java
b/orc/src/java/org/apache/orc/impl/TreeReaderFactory.java
new file mode 100644
index 000..6c8ecfd
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/TreeReaderFactory.java
@@ -0,0 +1,2093 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.OrcProto;
+
+/**
+ * Factory for creating ORC tree readers.
+ */
+public class TreeReaderFactory {
+
+ public abstract static class TreeReader {
+protected final int columnId;
+protected BitFieldReader present = null;
+protected boolean valuePresent = false;
+protected int vectorColumnCount;
+
+TreeReader(int columnId) throws IOException {
+ this(columnId, null);
+}
+
+protected TreeReader(int columnId, InStream in) throws IOException {
+ this.columnId = columnId;
+ if (in == null) {
+present = null;
+valuePresent = true;
+ } else {
+present = new BitFieldReader(in, 1);
+ }
+ vectorColumnCount = -1;
+}
+
+void setVectorColumnCount(int vectorColumnCount) {
+ this.vectorColumnCount = vectorColumnCount;
+}
+
+void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) {
+throw new IOException("Unknown encoding " + encoding + " in column " +
+columnId);
+ }
+}
+
+static IntegerReader createIntegerReader(OrcProto.ColumnEncoding.Kind kind,
+InStream in,
+boolean signed, boolean skipCorrupt) throws IOException {
+ switch (kind) {
+case DIRECT_V2:
+case DICTIONARY_V2:
+ return new RunLengthIntegerReaderV2(in, signed, skipCorrupt);
+case DIRECT:
+case DICTIONARY:
+ return new RunLengthIntegerReader(in, signed);
+default:
+ throw new IllegalArgumentException("Unknown encoding " + kind);
+ }
+}
+
+void startStripe(Map streams,
+OrcProto.StripeFooter stripeFooter
+) throws IOException {
+ checkEncoding(stripeFooter.getColumnsList().get(columnId));
+ InStream in = streams.get(new StreamName(columnId,
+ OrcProto.Stream.Kind.PRESENT));
+ if (in == null) {
+present = null;
+valuePresent = true;
+ } else {
+present = new BitFieldReader(in, 1);
+ }
+}
+
+/**
+ * Seek to the given position.
+ *
+ * @param index the indexes loaded from the file
+ * @throws IOException
+ */
+void seek(PositionProvider[] index) throws IOException {
+ seek(index[columnId]);
+}
+
+public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+present.seek(index);
+ }
+}
+
+protected long countNonNulls(long rows) throws IOException {
+ i
[20/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/test/org/apache/orc/TestStringDictionary.java
--
diff --git a/orc/src/test/org/apache/orc/TestStringDictionary.java
b/orc/src/test/org/apache/orc/TestStringDictionary.java
new file mode 100644
index 000..46209bb
--- /dev/null
+++ b/orc/src/test/org/apache/orc/TestStringDictionary.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+
+import org.apache.orc.impl.RecordReaderImpl;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+public class TestStringDictionary {
+
+ Path workDir = new Path(System.getProperty("test.tmp.dir", "target" +
File.separator + "test"
+ + File.separator + "tmp"));
+
+ Configuration conf;
+ FileSystem fs;
+ Path testFilePath;
+
+ @Rule
+ public TestName testCaseName = new TestName();
+
+ @Before
+ public void openFileSystem() throws Exception {
+conf = new Configuration();
+fs = FileSystem.getLocal(conf);
+testFilePath = new Path(workDir, "TestOrcFile." +
testCaseName.getMethodName() + ".orc");
+fs.delete(testFilePath, false);
+ }
+
+ @Test
+ public void testTooManyDistinct() throws Exception {
+TypeDescription schema = TypeDescription.createString();
+
+Writer writer = OrcFile.createWriter(
+testFilePath,
+OrcFile.writerOptions(conf).setSchema(schema)
+ .compress(CompressionKind.NONE)
+ .bufferSize(1));
+VectorizedRowBatch batch = schema.createRowBatch();
+BytesColumnVector col = (BytesColumnVector) batch.cols[0];
+for (int i = 0; i < 2; i++) {
+ if (batch.size == batch.getMaxSize()) {
+writer.addRowBatch(batch);
+batch.reset();
+ }
+ col.setVal(batch.size++, String.valueOf(i).getBytes());
+}
+writer.addRowBatch(batch);
+writer.close();
+
+Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
+RecordReader rows = reader.rows();
+batch = reader.getSchema().createRowBatch();
+col = (BytesColumnVector) batch.cols[0];
+int idx = 0;
+while (rows.nextBatch(batch)) {
+ for(int r=0; r < batch.size; ++r) {
+assertEquals(String.valueOf(idx++), col.toString(r));
+ }
+}
+
+// make sure the encoding type is correct
+for (StripeInformation stripe : reader.getStripes()) {
+ // hacky but does the job, this casting will work as long this test
resides
+ // within the same package as ORC reader
+ OrcProto.StripeFooter footer = ((RecordReaderImpl)
rows).readStripeFooter(stripe);
+ for (int i = 0; i < footer.getColumnsCount(); ++i) {
+OrcProto.ColumnEncoding encoding = footer.getColumns(i);
+assertEquals(OrcProto.ColumnEncoding.Kind.DIRECT_V2,
encoding.getKind());
+ }
+}
+ }
+
+ @Test
+ public void testHalfDistinct() throws Exception {
+TypeDescription schema = TypeDescription.createString();
+
+Writer writer = OrcFile.createWriter(
+testFilePath,
+
OrcFile.writerOptions(conf).setSchema(schema).compress(CompressionKind.NONE)
+.bufferSize(1));
+Random rand = new Random(123);
+int[] input = new int[2];
+for (int i = 0; i < 2; i++) {
+ input[i] = rand.nextInt(1);
+}
+
+VectorizedRowBatch batch = schema.createRowBatch();
+BytesColumnVector col = (BytesColumnVector) batch.cols[0];
+for (int i = 0; i < 2; i++) {
+ if (batch.size == batch.getMaxSize()) {
+writer.addRowBatch(batch);
+batch.reset();
+ }
+ col.setVal(batch.size++, String.valueOf(input[i]).getBytes());
+}
+writer.addRowBatch(batch);
+writer.
[16/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/test/org/apache/orc/tools/TestFileDump.java
--
diff --git a/orc/src/test/org/apache/orc/tools/TestFileDump.java
b/orc/src/test/org/apache/orc/tools/TestFileDump.java
new file mode 100644
index 000..ce3381e
--- /dev/null
+++ b/orc/src/test/org/apache/orc/tools/TestFileDump.java
@@ -0,0 +1,486 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.tools;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.PrintStream;
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcConf;
+import org.apache.orc.OrcFile;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestFileDump {
+
+ Path workDir = new Path(System.getProperty("test.tmp.dir"));
+ Configuration conf;
+ FileSystem fs;
+ Path testFilePath;
+
+ @Before
+ public void openFileSystem () throws Exception {
+conf = new Configuration();
+fs = FileSystem.getLocal(conf);
+fs.setWorkingDirectory(workDir);
+testFilePath = new Path("TestFileDump.testDump.orc");
+fs.delete(testFilePath, false);
+ }
+
+ static TypeDescription getMyRecordType() {
+return TypeDescription.createStruct()
+.addField("i", TypeDescription.createInt())
+.addField("l", TypeDescription.createLong())
+.addField("s", TypeDescription.createString());
+ }
+
+ static void appendMyRecord(VectorizedRowBatch batch,
+ int i,
+ long l,
+ String str) {
+((LongColumnVector) batch.cols[0]).vector[batch.size] = i;
+((LongColumnVector) batch.cols[1]).vector[batch.size] = l;
+if (str == null) {
+ batch.cols[2].noNulls = false;
+ batch.cols[2].isNull[batch.size] = true;
+} else {
+ ((BytesColumnVector) batch.cols[2]).setVal(batch.size,
+ str.getBytes());
+}
+batch.size += 1;
+ }
+
+ static TypeDescription getAllTypesType() {
+return TypeDescription.createStruct()
+.addField("b", TypeDescription.createBoolean())
+.addField("bt", TypeDescription.createByte())
+.addField("s", TypeDescription.createShort())
+.addField("i", TypeDescription.createInt())
+.addField("l", TypeDescription.createLong())
+.addField("f", TypeDescription.createFloat())
+.addField("d", TypeDescription.createDouble())
+.addField("de", TypeDescription.createDecimal())
+.addField("t", TypeDescription.createTimestamp())
+.addField("dt", TypeDescription.createDate())
+.addField("str", TypeDescription.createString())
+.addField("c", TypeDescription.createChar().withMaxLength(5))
+.addField("vc", TypeDescription.createVarchar().withMaxLength(10))
+.addF
[08/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
deleted file mode 100644
index f41a7ba..000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
+++ /dev/null
@@ -1,1342 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-
-import java.io.File;
-import java.sql.Timestamp;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.orc.CompressionKind;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Longs;
-
-@RunWith(value = Parameterized.class)
-public class TestNewIntegerEncoding {
-
- private OrcFile.EncodingStrategy encodingStrategy;
-
- public TestNewIntegerEncoding( OrcFile.EncodingStrategy es) {
-this.encodingStrategy = es;
- }
-
- @Parameters
- public static Collection data() {
-Object[][] data = new Object[][] { { OrcFile.EncodingStrategy.COMPRESSION
},
-{ OrcFile.EncodingStrategy.SPEED } };
-return Arrays.asList(data);
- }
-
- public static class TSRow {
-Timestamp ts;
-
-public TSRow(Timestamp ts) {
- this.ts = ts;
-}
- }
-
- public static class Row {
-Integer int1;
-Long long1;
-
-public Row(int val, long l) {
- this.int1 = val;
- this.long1 = l;
-}
- }
-
- Path workDir = new Path(System.getProperty("test.tmp.dir", "target"
- + File.separator + "test" + File.separator + "tmp"));
-
- Configuration conf;
- FileSystem fs;
- Path testFilePath;
-
- @Rule
- public TestName testCaseName = new TestName();
-
- @Before
- public void openFileSystem() throws Exception {
-conf = new Configuration();
-fs = FileSystem.getLocal(conf);
-testFilePath = new Path(workDir, "TestOrcFile."
-+ testCaseName.getMethodName() + ".orc");
-fs.delete(testFilePath, false);
- }
-
- @Test
- public void testBasicRow() throws Exception {
-ObjectInspector inspector;
-synchronized (TestOrcFile.class) {
- inspector = ObjectInspectorFactory.getReflectionObjectInspector(
- Row.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
-}
-
-Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .stripeSize(10)
- .compress(CompressionKind.NONE)
- .bufferSize(1)
- .encodingStrategy(encodingStrategy));
-writer.addRow(new Row(111, L));
-writer.addRow(new Row(111, L));
-writer.addRow(new Row(111, L));
-writer.close();
-
-Reader reader = OrcFile.createReader(testFilePath,
-OrcFile.readerOptions(conf).filesystem(fs));
-RecordReader rows = reader.rows();
-while (rows.hasNext()) {
- Object row = rows.next(null);
- assertEquals(new IntWritable(111), ((OrcStruct) row).getFieldValue(0));
- assertEquals(new LongWritable(), ((OrcStruct) row).getFieldValue(1));
-}
- }
-
- @Test
- public void testBasicO
[11/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
deleted file mode 100644
index 4192588..000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
+++ /dev/null
@@ -1,586 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import com.google.common.collect.Lists;
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.io.DiskRange;
-import org.apache.hadoop.hive.common.io.DiskRangeList;
-import org.apache.hadoop.hive.common.io.DiskRangeList.CreateHelper;
-import org.apache.hadoop.hive.common.io.DiskRangeList.MutateHelper;
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.hive.shims.HadoopShims.ByteBufferPoolShim;
-import org.apache.hadoop.hive.shims.HadoopShims.ZeroCopyReaderShim;
-import org.apache.orc.StripeInformation;
-import org.apache.orc.impl.BufferChunk;
-import org.apache.orc.CompressionCodec;
-import org.apache.orc.DataReader;
-import org.apache.orc.impl.DataReaderProperties;
-import org.apache.orc.impl.DirectDecompressionCodec;
-import org.apache.orc.OrcProto;
-
-import com.google.common.collect.ComparisonChain;
-import org.apache.orc.impl.InStream;
-import org.apache.orc.impl.OrcIndex;
-import org.apache.orc.impl.OutStream;
-
-/**
- * Stateless methods shared between RecordReaderImpl and EncodedReaderImpl.
- */
-public class RecordReaderUtils {
- private static final HadoopShims SHIMS = ShimLoader.getHadoopShims();
-
- private static class DefaultDataReader implements DataReader {
-private FSDataInputStream file = null;
-private final ByteBufferAllocatorPool pool;
-private ZeroCopyReaderShim zcr = null;
-private final FileSystem fs;
-private final Path path;
-private final boolean useZeroCopy;
-private final CompressionCodec codec;
-private final int bufferSize;
-private final int typeCount;
-
-private DefaultDataReader(DefaultDataReader other) {
- this.pool = other.pool;
- this.zcr = other.zcr;
- this.bufferSize = other.bufferSize;
- this.typeCount = other.typeCount;
- this.fs = other.fs;
- this.path = other.path;
- this.useZeroCopy = other.useZeroCopy;
- this.codec = other.codec;
-}
-
-private DefaultDataReader(DataReaderProperties properties) {
- this.fs = properties.getFileSystem();
- this.path = properties.getPath();
- this.useZeroCopy = properties.getZeroCopy();
- this.codec = WriterImpl.createCodec(properties.getCompression());
- this.bufferSize = properties.getBufferSize();
- this.typeCount = properties.getTypeCount();
- if (useZeroCopy) {
-this.pool = new ByteBufferAllocatorPool();
- } else {
-this.pool = null;
- }
-}
-
-@Override
-public void open() throws IOException {
- this.file = fs.open(path);
- if (useZeroCopy) {
-zcr = RecordReaderUtils.createZeroCopyShim(file, codec, pool);
- } else {
-zcr = null;
- }
-}
-
-@Override
-public OrcIndex readRowIndex(StripeInformation stripe,
- OrcProto.StripeFooter footer,
- boolean[] included,
- OrcProto.RowIndex[] indexes,
- boolean[] sargColumns,
- OrcProto.BloomFilterIndex[] bloomFilterIndices
- ) throws IOException {
- if (file == null) {
-open();
- }
- if (footer == null) {
-footer = readStripe
[14/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ConvertTreeReaderFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ConvertTreeReaderFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ConvertTreeReaderFactory.java
deleted file mode 100644
index 74a097e..000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ConvertTreeReaderFactory.java
+++ /dev/null
@@ -1,3750 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.StandardCharsets;
-import java.sql.Date;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.EnumMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
-import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.orc.OrcProto;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.TypeDescription.Category;
-import org.apache.orc.impl.InStream;
-import org.apache.orc.impl.PositionProvider;
-import org.apache.orc.impl.StreamName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Convert ORC tree readers.
- */
-public class ConvertTreeReaderFactory extends TreeReaderFactory {
-
- private static final Logger LOG =
-LoggerFactory.getLogger(TreeReaderFactory.class);
-
- /**
- * Override methods like checkEncoding to pass-thru to the convert
TreeReader.
- */
- public static class ConvertTreeReader extends TreeReader {
-
-private TreeReader convertTreeReader;
-
-ConvertTreeReader(int columnId) throws IOException {
- super(columnId);
-}
-
-private static List numericTypeList = new
ArrayList();
-
-// The ordering of types here is used to determine which numeric types
-// are common/convertible to one another. Probably better to rely on the
-// ordering explicitly defined here than to assume that the enum values
-// that were arbitrarily assigned in PrimitiveCategory work for our
purposes.
-private static EnumMap numericTypes =
-new EnumMap(TypeDescription.Category.class);
-
-static {
- registerNumericType(TypeDescription.Category.BOOLEAN, 1);
- registerNumericType(TypeDescription.Category.BYTE, 2);
- registerNumericType(TypeDescription.Category.SHORT, 3);
- registerNumericType(TypeDescription.Category.INT, 4);
- registerNumericType(TypeDescription.Category.LONG, 5);
- registerNumericType(TypeDescription.Category.FLOAT, 6);
- registerNumericType(TypeDescription.Category.DOUBLE, 7);
- registerNumericType(TypeDescription.Category.DECIMAL, 8);
-}
-
-private static void registerNumericType(TypeDescription.Category kind, int
level) {
- numericTypeList.add(kind);
- numericTypes.put(kind, level);
-}
-
-protected void setConvertTreeReader(TreeReader convertTreeR
[25/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/java/org/apache/orc/impl/HadoopShims.java
--
diff --git a/orc/src/java/org/apache/orc/impl/HadoopShims.java
b/orc/src/java/org/apache/orc/impl/HadoopShims.java
index 2980d71..ef7d70f 100644
--- a/orc/src/java/org/apache/orc/impl/HadoopShims.java
+++ b/orc/src/java/org/apache/orc/impl/HadoopShims.java
@@ -18,9 +18,13 @@
package org.apache.orc.impl;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.VersionInfo;
+import java.io.Closeable;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.ByteBuffer;
public interface HadoopShims {
@@ -43,6 +47,81 @@ public interface HadoopShims {
*/
DirectDecompressor getDirectDecompressor(DirectCompressionType codec);
+ /**
+ * a hadoop.io ByteBufferPool shim.
+ */
+ public interface ByteBufferPoolShim {
+/**
+ * Get a new ByteBuffer from the pool. The pool can provide this from
+ * removing a buffer from its internal cache, or by allocating a
+ * new buffer.
+ *
+ * @param direct Whether the buffer should be direct.
+ * @param length The minimum length the buffer will have.
+ * @return A new ByteBuffer. Its capacity can be less
+ * than what was requested, but must be at
+ * least 1 byte.
+ */
+ByteBuffer getBuffer(boolean direct, int length);
+
+/**
+ * Release a buffer back to the pool.
+ * The pool may choose to put this buffer into its cache/free it.
+ *
+ * @param buffera direct bytebuffer
+ */
+void putBuffer(ByteBuffer buffer);
+ }
+
+ /**
+ * Provides an HDFS ZeroCopyReader shim.
+ * @param in FSDataInputStream to read from (where the cached/mmap buffers
are tied to)
+ * @param in ByteBufferPoolShim to allocate fallback buffers with
+ *
+ * @return returns null if not supported
+ */
+ public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in,
ByteBufferPoolShim pool) throws IOException;
+
+ public interface ZeroCopyReaderShim extends Closeable {
+/**
+ * Get a ByteBuffer from the FSDataInputStream - this can be either a
HeapByteBuffer or an MappedByteBuffer.
+ * Also move the in stream by that amount. The data read can be small than
maxLength.
+ *
+ * @return ByteBuffer read from the stream,
+ */
+public ByteBuffer readBuffer(int maxLength, boolean verifyChecksums)
throws IOException;
+/**
+ * Release a ByteBuffer obtained from a read on the
+ * Also move the in stream by that amount. The data read can be small than
maxLength.
+ *
+ */
+public void releaseBuffer(ByteBuffer buffer);
+
+/**
+ * Close the underlying stream.
+ * @throws IOException
+ */
+public void close() throws IOException;
+ }
+ /**
+ * Read data into a Text object in the fastest way possible
+ */
+ public interface TextReaderShim {
+/**
+ * @param txt
+ * @param size
+ * @return bytes read
+ * @throws IOException
+ */
+void read(Text txt, int size) throws IOException;
+ }
+
+ /**
+ * Wrap a TextReaderShim around an input stream. The reader shim will not
+ * buffer any reads from the underlying stream and will only consume bytes
+ * which are required for TextReaderShim.read() input.
+ */
+ public TextReaderShim getTextReaderShim(InputStream input) throws
IOException;
class Factory {
private static HadoopShims SHIMS = null;
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
--
diff --git a/orc/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
b/orc/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
index 3b9371d..5c53f74 100644
--- a/orc/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
+++ b/orc/src/java/org/apache/orc/impl/HadoopShimsCurrent.java
@@ -18,10 +18,14 @@
package org.apache.orc.impl;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
import org.apache.hadoop.io.compress.zlib.ZlibDecompressor;
+import java.io.DataInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.ByteBuffer;
/**
@@ -59,4 +63,30 @@ public class HadoopShimsCurrent implements HadoopShims {
return null;
}
}
+
+ @Override
+ public ZeroCopyReaderShim getZeroCopyReader(FSDataInputStream in,
+ ByteBufferPoolShim pool
+ ) throws IOException {
+return ZeroCopyShims.getZeroCopyReader(in, pool);
+ }
+
+ private final class FastTextReaderShim implements TextReaderShim {
+private final DataInputStream din;
[17/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
--
diff --git a/orc/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
b/orc/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
new file mode 100644
index 000..cdd62ac
--- /dev/null
+++ b/orc/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
@@ -0,0 +1,1691 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import static junit.framework.Assert.assertEquals;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl;
+import org.apache.orc.BloomFilterIO;
+import org.apache.orc.DataReader;
+import org.apache.orc.RecordReader;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.apache.orc.impl.RecordReaderImpl.Location;
+import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.OrcFile;
+import org.apache.orc.Reader;
+import org.apache.orc.OrcProto;
+
+import org.junit.Test;
+import org.mockito.MockSettings;
+import org.mockito.Mockito;
+
+public class TestRecordReaderImpl {
+ /**
+ * Create a predicate leaf. This is used by another test.
+ */
+ public static PredicateLeaf createPredicateLeaf(PredicateLeaf.Operator
operator,
+ PredicateLeaf.Type type,
+ String columnName,
+ Object literal,
+ List literalList) {
+return new SearchArgumentImpl.PredicateLeafImpl(operator, type, columnName,
+literal, literalList);
+ }
+
+ // can add .verboseLogging() to cause Mockito to log invocations
+ private final MockSettings settings =
Mockito.withSettings().verboseLogging();
+
+ static class BufferInStream
+ extends InputStream implements PositionedReadable, Seekable {
+private final byte[] buffer;
+private final int length;
+private int position = 0;
+
+BufferInStream(byte[] bytes, int length) {
+ this.buffer = bytes;
+ this.length = length;
+}
+
+@Override
+public int read() {
+ if (position < length) {
+return buffer[position++];
+ }
+ return -1;
+}
+
+@Override
+public int read(byte[] bytes, int offset, int length) {
+ int lengthToRead = Math.min(length, this.length - this.position);
+ if (lengthToRead >= 0) {
+for(int i=0; i < lengthToRead; ++i) {
+ bytes[offset + i] = buffer[position++];
+}
+return lengthToRead;
+ } else {
+return -1;
+ }
+}
+
+@Override
+public int read(long position, byte[] bytes, int offset, int length) {
+ this.position = (int) position;
+ return read(bytes, offset, length);
+}
+
+@Override
+public void readFully(long position, byte[] bytes, int offset,
+
[02/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/resources/orc-file-dump.json
--
diff --git a/ql/src/test/resources/orc-file-dump.json
b/ql/src/test/resources/orc-file-dump.json
deleted file mode 100644
index bf654a1..000
--- a/ql/src/test/resources/orc-file-dump.json
+++ /dev/null
@@ -1,1355 +0,0 @@
-{
- "fileName": "TestFileDump.testDump.orc",
- "fileVersion": "0.12",
- "writerVersion": "HIVE_13083",
- "numberOfRows": 21000,
- "compression": "ZLIB",
- "compressionBufferSize": 4096,
- "schemaString": "struct",
- "schema": [
-{
- "columnId": 0,
- "columnType": "STRUCT",
- "childColumnNames": [
-"i",
-"l",
-"s"
- ],
- "childColumnIds": [
-1,
-2,
-3
- ]
-},
-{
- "columnId": 1,
- "columnType": "INT"
-},
-{
- "columnId": 2,
- "columnType": "LONG"
-},
-{
- "columnId": 3,
- "columnType": "STRING"
-}
- ],
- "stripeStatistics": [
-{
- "stripeNumber": 1,
- "columnStatistics": [
-{
- "columnId": 0,
- "count": 5000,
- "hasNull": false
-},
-{
- "columnId": 1,
- "count": 5000,
- "hasNull": false,
- "min": -2147115959,
- "max": 2145210552,
- "sum": 50111854553,
- "type": "LONG"
-},
-{
- "columnId": 2,
- "count": 5000,
- "hasNull": false,
- "min": -9223180583305557329,
- "max": 9221614132680747961,
- "type": "LONG"
-},
-{
- "columnId": 3,
- "count": 4950,
- "hasNull": true,
- "min": "Darkness,",
- "max": "worst",
- "totalLength": 19283,
- "type": "STRING"
-}
- ]
-},
-{
- "stripeNumber": 2,
- "columnStatistics": [
-{
- "columnId": 0,
- "count": 5000,
- "hasNull": false
-},
-{
- "columnId": 1,
- "count": 5000,
- "hasNull": false,
- "min": -2147390285,
- "max": 2147224606,
- "sum": -22290798217,
- "type": "LONG"
-},
-{
- "columnId": 2,
- "count": 5000,
- "hasNull": false,
- "min": -9219295160509160427,
- "max": 9217571024994660020,
- "type": "LONG"
-},
-{
- "columnId": 3,
- "count": 4950,
- "hasNull": true,
- "min": "Darkness,",
- "max": "worst",
- "totalLength": 19397,
- "type": "STRING"
-}
- ]
-},
-{
- "stripeNumber": 3,
- "columnStatistics": [
-{
- "columnId": 0,
- "count": 5000,
- "hasNull": false
-},
-{
- "columnId": 1,
- "count": 5000,
- "hasNull": false,
- "min": -2146954065,
- "max": 2146722468,
- "sum": 20639652136,
- "type": "LONG"
-},
-{
- "columnId": 2,
- "count": 5000,
- "hasNull": false,
- "min": -9214076359988107846,
- "max": 9222919052987871506,
- "type": "LONG"
-},
-{
- "columnId": 3,
- "count": 4950,
- "hasNull": true,
- "min": "Darkness,",
- "max": "worst",
- "totalLength": 19031,
- "type": "STRING"
-}
- ]
-},
-{
- "stripeNumber": 4,
- "columnStatistics": [
-{
- "columnId": 0,
- "count": 5000,
- "hasNull": false
-},
-{
- "columnId": 1,
- "count": 5000,
- "hasNull": false,
- "min": -2146969085,
- "max": 2146025044,
- "sum": -5156814387,
- "type": "LONG"
-},
-{
- "columnId": 2,
- "count": 5000,
- "hasNull": false,
- "min": -9222731174895935707,
- "max": 9220625004936875965,
- "type": "LONG"
-},
-{
- "columnId": 3,
- "count": 4950,
- "hasNull": true,
- "min": "Darkness,",
- "max": "worst",
- "totalLength": 19459,
- "type": "STRING"
-}
- ]
-},
-{
- "stripeNumber": 5,
- "columnStatistics": [
-{
- "columnId": 0,
- "count": 1000,
- "hasNull": false
-},
-{
- "columnId": 1,
- "count": 1000,
- "hasNull": false,
- "min": -2144303438,
- "max": 2127599049,
- "sum": 62841564778,
- "type": "LONG"
-},
-{
- "columnId": 2,
- "count": 1000,
- "hasNull": false,
- "min": -9195133638801798919,
- "max": 9218626063131504414,
- "type": "LONG"
[26/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
--
diff --git a/orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
b/orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
new file mode 100644
index 000..3ba56f7
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -0,0 +1,2840 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.EnumMap;
+import java.util.Map;
+
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
+import org.apache.hadoop.hive.ql.util.TimestampUtils;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.orc.OrcProto;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.TypeDescription.Category;
+
+/**
+ * Convert ORC tree readers.
+ */
+public class ConvertTreeReaderFactory extends TreeReaderFactory {
+
+ /**
+ * Override methods like checkEncoding to pass-thru to the convert
TreeReader.
+ */
+ public static class ConvertTreeReader extends TreeReader {
+
+private TreeReader convertTreeReader;
+
+ConvertTreeReader(int columnId) throws IOException {
+ super(columnId);
+}
+
+// The ordering of types here is used to determine which numeric types
+// are common/convertible to one another. Probably better to rely on the
+// ordering explicitly defined here than to assume that the enum values
+// that were arbitrarily assigned in PrimitiveCategory work for our
purposes.
+private static EnumMap numericTypes =
+new EnumMap<>(TypeDescription.Category.class);
+
+static {
+ registerNumericType(TypeDescription.Category.BOOLEAN, 1);
+ registerNumericType(TypeDescription.Category.BYTE, 2);
+ registerNumericType(TypeDescription.Category.SHORT, 3);
+ registerNumericType(TypeDescription.Category.INT, 4);
+ registerNumericType(TypeDescription.Category.LONG, 5);
+ registerNumericType(TypeDescription.Category.FLOAT, 6);
+ registerNumericType(TypeDescription.Category.DOUBLE, 7);
+ registerNumericType(TypeDescription.Category.DECIMAL, 8);
+}
+
+private static void registerNumericType(TypeDescription.Category kind, int
level) {
+ numericTypes.put(kind, level);
+}
+
+protected void setConvertTreeReader(TreeReader convertTreeReader) {
+ this.convertTreeReader = convertTreeReader;
+}
+
+protected TreeReader getStringGroupTreeReader(int columnId,
+TypeDescription fileType) throws IOException {
+ switch (fileType.getCategory()) {
+ case STRING:
+return new StringTreeReader(columnId);
+ case CHAR:
+return new CharTreeReader(columnId, fileType.getMaxLength());
+ case VARCHAR:
+return new VarcharTreeReader(columnId, fileType.getMaxLength());
+ default:
+throw new RuntimeException("Unexpected type kind " +
fileType.getCategory().name());
+ }
+}
+
+protected void assignStringGroupVectorEntry(BytesColumnVector
bytesColVector,
+int elementNum, TypeDescription readerType, byte[] bytes) {
+ assignStringGroupVectorEntry(bytesColVector,
+ elementNum, readerType, bytes, 0, bytes.length);
+}
+
+/*
+ * Assign a BytesColumnVector entry when we have a byte array, start, and
+ * length for the string group which can be (STRING, CHAR,
[10/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TreeReaderFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TreeReaderFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TreeReaderFactory.java
deleted file mode 100644
index 6d1c256..000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TreeReaderFactory.java
+++ /dev/null
@@ -1,2525 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.sql.Timestamp;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
-
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
-import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
-import org.apache.hadoop.hive.shims.HadoopShims.TextReaderShim;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.impl.BitFieldReader;
-import org.apache.orc.impl.DynamicByteArray;
-import org.apache.orc.impl.InStream;
-import org.apache.orc.impl.IntegerReader;
-import org.apache.orc.OrcProto;
-import org.apache.orc.impl.PositionProvider;
-import org.apache.orc.impl.RunLengthByteReader;
-import org.apache.orc.impl.RunLengthIntegerReader;
-import org.apache.orc.impl.RunLengthIntegerReaderV2;
-import org.apache.orc.impl.SerializationUtils;
-import org.apache.orc.impl.StreamName;
-
-/**
- * Factory for creating ORC tree readers.
- */
-public class TreeReaderFactory {
-
- public abstract static class TreeReader {
-protected final int columnId;
-protected BitFieldReader present = null;
-protected boolean valuePresent = false;
-protected int vectorColumnCount;
-
-TreeReader(int columnId) throws IOException {
- this(columnId, null);
-}
-
-protected TreeReader(int columnId, InStream in) throws IOException {
- this.columnId = columnId;
- if (in == null) {
-present = null;
-valuePresent = true;
- } else {
-present = new BitFieldReader(in, 1);
- }
- vectorColumnCount = -1;
-}
-
-void setVectorColumnCount(int vectorColumnCount) {
- this.vectorColumnCount = vectorColumnCount;
-}
-
-void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
- if (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) {
-throw new IOException("Unknown encoding " + encoding + " in column " +
-
[13/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
deleted file mode 100644
index 9c2f88f..000
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java
+++ /dev/null
@@ -1,884 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.PrintStream;
-import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.orc.BloomFilterIO;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.orc.ColumnStatistics;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.impl.ColumnStatisticsImpl;
-import org.apache.orc.impl.OrcIndex;
-import org.apache.orc.OrcProto;
-import org.apache.orc.StripeInformation;
-import org.apache.orc.StripeStatistics;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONWriter;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-/**
- * A tool for printing out the file structure of ORC files.
- */
-public final class FileDump {
- public static final String UNKNOWN = "UNKNOWN";
- public static final String SEPARATOR = Strings.repeat("_", 120) + "\n";
- public static final int DEFAULT_BLOCK_SIZE = 256 * 1024 * 1024;
- public static final String DEFAULT_BACKUP_PATH =
System.getProperty("java.io.tmpdir");
- public static final PathFilter HIDDEN_AND_SIDE_FILE_FILTER = new
PathFilter() {
-public boolean accept(Path p) {
- String name = p.getName();
- return !name.startsWith("_") && !name.startsWith(".") && !name.endsWith(
- AcidUtils.DELTA_SIDE_FILE_SUFFIX);
-}
- };
-
- // not used
- private FileDump() {
- }
-
- public static void main(String[] args) throws Exception {
-Configuration conf = new Configuration();
-
-List rowIndexCols = null;
-Options opts = createOptions();
-CommandLine cli = new GnuParser().parse(opts, args);
-
-if (cli.hasOption('h')) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("orcfiledump", opts);
- return;
-}
-
-boolean dumpData = cli.hasOption('d');
-boolean recover = cli.hasOption("recover");
-boolean skipDump = cli.hasOption("skip-dump");
-String backupPath = DEFAULT_BACKUP_PATH;
-if (cli.hasOption("backup-path")) {
- backupPath = cli.getOptionValue("backup-path");
-}
-
-if (cli.hasOption("r")) {
- String[] colStrs = cli.getOptionValue("r").split(",");
- rowIndexCols = new ArrayList(colStrs.length);
- for (String colStr : colStrs) {
-rowIndexCols.add(Integer.parseInt(colStr));
- }
-}
-
-boolean printTimeZone = cli.hasOption('t');
-boolean jsonFormat = cli.hasOption('j');
-String[] files = cli.getArgs();
-if (files.length == 0) {
- System.err.println("Error : ORC files are not specified");
- return;
-
[09/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index 40cc86f..dad35e3 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -38,7 +38,7 @@ import org.apache.orc.CompressionCodec;
import org.apache.orc.DataReader;
import org.apache.orc.OrcConf;
import org.apache.orc.impl.OutStream;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils;
+import org.apache.orc.impl.RecordReaderUtils;
import org.apache.orc.impl.StreamName;
import org.apache.orc.StripeInformation;
import org.apache.orc.impl.BufferChunk;
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
index fe46446..b44da06 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java
@@ -25,7 +25,7 @@ import
org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamD
import org.apache.orc.CompressionCodec;
import org.apache.orc.impl.PositionProvider;
import org.apache.orc.impl.SettableUncompressedStream;
-import org.apache.hadoop.hive.ql.io.orc.TreeReaderFactory;
+import org.apache.orc.impl.TreeReaderFactory;
import org.apache.orc.OrcProto;
public class EncodedTreeReaderFactory extends TreeReaderFactory {
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index b20ce28..e4cbd5f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.io.orc.FileDump;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService;
@@ -36,7 +35,6 @@ import org.junit.Test;
import org.junit.rules.TestName;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampWritableAndColumnVector.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampWritableAndColumnVector.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampWritableAndColumnVector.java
index 6c46257..2fa9ab2 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampWritableAndColumnVector.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestTimestampWritableAndColumnVector.java
@@ -20,14 +20,11 @@ package org.apache.hadoop.hive.ql.exec.vector;
import org.junit.Test;
-import java.math.BigDecimal;
-import java.math.RoundingMode;
import java.sql.Timestamp;
-import java.util.Date;
import java.util.Random;
import org.apache.hadoop.hive.common.type.RandomTypeUtil;
-import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.ql.util.TimestampUtils;
import static org.junit.Assert.*;
@@ -58,7 +55,7 @@ public class TestTimestampWritableAndColumnVector {
if (!retrievedTimestamp.equals(randTimestamp)) {
assertTrue(false);
}
- double randDouble = TimestampWritable.getDouble(randTimestamp);
+ double randDouble = TimestampUtils.getDouble(randTimestamp);
double retrievedDouble = timestampColVector.getDouble(i);
if (randDouble != retrievedDouble) {
assertTrue(false);
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/ex
[15/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/test/resources/orc-file-dump.json
--
diff --git a/orc/src/test/resources/orc-file-dump.json
b/orc/src/test/resources/orc-file-dump.json
new file mode 100644
index 000..bf654a1
--- /dev/null
+++ b/orc/src/test/resources/orc-file-dump.json
@@ -0,0 +1,1355 @@
+{
+ "fileName": "TestFileDump.testDump.orc",
+ "fileVersion": "0.12",
+ "writerVersion": "HIVE_13083",
+ "numberOfRows": 21000,
+ "compression": "ZLIB",
+ "compressionBufferSize": 4096,
+ "schemaString": "struct",
+ "schema": [
+{
+ "columnId": 0,
+ "columnType": "STRUCT",
+ "childColumnNames": [
+"i",
+"l",
+"s"
+ ],
+ "childColumnIds": [
+1,
+2,
+3
+ ]
+},
+{
+ "columnId": 1,
+ "columnType": "INT"
+},
+{
+ "columnId": 2,
+ "columnType": "LONG"
+},
+{
+ "columnId": 3,
+ "columnType": "STRING"
+}
+ ],
+ "stripeStatistics": [
+{
+ "stripeNumber": 1,
+ "columnStatistics": [
+{
+ "columnId": 0,
+ "count": 5000,
+ "hasNull": false
+},
+{
+ "columnId": 1,
+ "count": 5000,
+ "hasNull": false,
+ "min": -2147115959,
+ "max": 2145210552,
+ "sum": 50111854553,
+ "type": "LONG"
+},
+{
+ "columnId": 2,
+ "count": 5000,
+ "hasNull": false,
+ "min": -9223180583305557329,
+ "max": 9221614132680747961,
+ "type": "LONG"
+},
+{
+ "columnId": 3,
+ "count": 4950,
+ "hasNull": true,
+ "min": "Darkness,",
+ "max": "worst",
+ "totalLength": 19283,
+ "type": "STRING"
+}
+ ]
+},
+{
+ "stripeNumber": 2,
+ "columnStatistics": [
+{
+ "columnId": 0,
+ "count": 5000,
+ "hasNull": false
+},
+{
+ "columnId": 1,
+ "count": 5000,
+ "hasNull": false,
+ "min": -2147390285,
+ "max": 2147224606,
+ "sum": -22290798217,
+ "type": "LONG"
+},
+{
+ "columnId": 2,
+ "count": 5000,
+ "hasNull": false,
+ "min": -9219295160509160427,
+ "max": 9217571024994660020,
+ "type": "LONG"
+},
+{
+ "columnId": 3,
+ "count": 4950,
+ "hasNull": true,
+ "min": "Darkness,",
+ "max": "worst",
+ "totalLength": 19397,
+ "type": "STRING"
+}
+ ]
+},
+{
+ "stripeNumber": 3,
+ "columnStatistics": [
+{
+ "columnId": 0,
+ "count": 5000,
+ "hasNull": false
+},
+{
+ "columnId": 1,
+ "count": 5000,
+ "hasNull": false,
+ "min": -2146954065,
+ "max": 2146722468,
+ "sum": 20639652136,
+ "type": "LONG"
+},
+{
+ "columnId": 2,
+ "count": 5000,
+ "hasNull": false,
+ "min": -9214076359988107846,
+ "max": 9222919052987871506,
+ "type": "LONG"
+},
+{
+ "columnId": 3,
+ "count": 4950,
+ "hasNull": true,
+ "min": "Darkness,",
+ "max": "worst",
+ "totalLength": 19031,
+ "type": "STRING"
+}
+ ]
+},
+{
+ "stripeNumber": 4,
+ "columnStatistics": [
+{
+ "columnId": 0,
+ "count": 5000,
+ "hasNull": false
+},
+{
+ "columnId": 1,
+ "count": 5000,
+ "hasNull": false,
+ "min": -2146969085,
+ "max": 2146025044,
+ "sum": -5156814387,
+ "type": "LONG"
+},
+{
+ "columnId": 2,
+ "count": 5000,
+ "hasNull": false,
+ "min": -9222731174895935707,
+ "max": 9220625004936875965,
+ "type": "LONG"
+},
+{
+ "columnId": 3,
+ "count": 4950,
+ "hasNull": true,
+ "min": "Darkness,",
+ "max": "worst",
+ "totalLength": 19459,
+ "type": "STRING"
+}
+ ]
+},
+{
+ "stripeNumber": 5,
+ "columnStatistics": [
+{
+ "columnId": 0,
+ "count": 1000,
+ "hasNull": false
+},
+{
+ "columnId": 1,
+ "count": 1000,
+ "hasNull": false,
+ "min": -2144303438,
+ "max": 2127599049,
+ "sum": 62841564778,
+ "type": "LONG"
+},
+{
+ "columnId": 2,
+ "count": 1000,
+ "hasNull": false,
+ "min": -9195133638801798919,
+ "max": 9218626063131504414,
+ "type": "LONG"
[19/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/test/org/apache/orc/TestVectorOrcFile.java
--
diff --git a/orc/src/test/org/apache/orc/TestVectorOrcFile.java
b/orc/src/test/org/apache/orc/TestVectorOrcFile.java
new file mode 100644
index 000..112edb9
--- /dev/null
+++ b/orc/src/test/org/apache/orc/TestVectorOrcFile.java
@@ -0,0 +1,2782 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import com.google.common.collect.Lists;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.orc.impl.DataReaderProperties;
+import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.impl.RecordReaderImpl;
+import org.apache.orc.impl.RecordReaderUtils;
+import org.apache.orc.tools.TestJsonFileDump;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import static junit.framework.TestCase.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for the vectorized reader and writer for ORC files.
+ */
+public class TestVectorOrcFile {
+
+ public static class InnerStruct {
+int int1;
+Text string1 = new Text();
+InnerStruct(int int1, Text string1) {
+ this.int1 = int1;
+ this.string1.set(string1);
+}
+InnerStruct(int int1, String string1) {
+ this.int1 = int1;
+ this.string1.set(string1);
+}
+
+public String toString() {
+ return "{" + int1 + ", " + string1 + "}";
+}
+ }
+
+ public static class MiddleStruct {
+List list = new ArrayList();
+
+MiddleStruct(InnerStruct... items) {
+ list.clear();
+ list.addAll(Arrays.asList(items));
+}
+ }
+
+ private static InnerStruct inner(int i, String s) {
+return new InnerStruct(i, s);
+ }
+
+ private static Map map(InnerStruct... items) {
+Map result = new HashMap();
+for(InnerStruct i: items) {
+ result.put(i.string1.toString(), i);
+}
+return result;
+ }
+
+ private static List list(InnerStruct... items) {
+List result = new ArrayList();
+result.addAll(Arrays.asList(items));
+return result;
+ }
+
+ private static BytesWritable bytes(int... items) {
+BytesWritable result = new BytesWritable();
+result.setSize(items.length);
+for(int i=0; i < items.length; ++i) {
+ result.getBytes()[i] = (byte) items[i];
+}
+return result;
+ }
+
+ private static byte[] bytesArray(int... items) {
+byte[] result = new byte[items.length];
+
[06/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
deleted file mode 100644
index 8731be0..000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
+++ /dev/null
@@ -1,1678 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static junit.framework.Assert.assertEquals;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.atLeastOnce;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PositionedReadable;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.hive.common.io.DiskRangeList;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hive.common.util.HiveTestUtils;
-import org.apache.orc.BloomFilterIO;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.Location;
-import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
-import org.apache.hadoop.hive.ql.io.sarg.TestSearchArgumentImpl;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.orc.ColumnStatistics;
-import org.apache.orc.DataReader;
-import org.apache.orc.StripeInformation;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.impl.ColumnStatisticsImpl;
-import org.apache.orc.OrcProto;
-
-import org.junit.Test;
-import org.mockito.MockSettings;
-import org.mockito.Mockito;
-
-public class TestRecordReaderImpl {
-
- // can add .verboseLogging() to cause Mockito to log invocations
- private final MockSettings settings =
Mockito.withSettings().verboseLogging();
-
- static class BufferInStream
- extends InputStream implements PositionedReadable, Seekable {
-private final byte[] buffer;
-private final int length;
-private int position = 0;
-
-BufferInStream(byte[] bytes, int length) {
- this.buffer = bytes;
- this.length = length;
-}
-
-@Override
-public int read() {
- if (position < length) {
-return buffer[position++];
- }
- return -1;
-}
-
-@Override
-public int read(byte[] bytes, int offset, int length) {
- int lengthToRead = Math.min(length, this.length - this.position);
- if (lengthToRead >= 0) {
-for(int i=0; i < lengthToRead; ++i) {
- bytes[offset + i] = buffer[position++];
-}
-return lengthToRead;
- } else {
-return -1;
- }
-}
-
-@Override
-public int read(long position, byte[] bytes, int offset, int length) {
- this.position = (int) position;
- return read(bytes, offset, length);
-}
-
-@Override
-public void readFully(long position, byte[] bytes, int offset,
- int length) throws IOException {
- this.position = (int) position;
- while (length > 0) {
-int result = read(bytes, offset, length);
-offset += result;
-length -= result;
-if (result < 0) {
- throw new IOException("Read past end of buffer at " + offset);
-}
- }
-}
-
-@Override
-public void readFully(long position, byte[] bytes) throws IOException {
- readFully(position, bytes, 0, bytes.length);
-}
-
-
[24/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/java/org/apache/orc/impl/RecordReaderUtils.java
--
diff --git a/orc/src/java/org/apache/orc/impl/RecordReaderUtils.java
b/orc/src/java/org/apache/orc/impl/RecordReaderUtils.java
new file mode 100644
index 000..1067957
--- /dev/null
+++ b/orc/src/java/org/apache/orc/impl/RecordReaderUtils.java
@@ -0,0 +1,578 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.impl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hadoop.hive.common.io.DiskRangeList.CreateHelper;
+import org.apache.hadoop.hive.common.io.DiskRangeList.MutateHelper;
+import org.apache.orc.CompressionCodec;
+import org.apache.orc.DataReader;
+import org.apache.orc.OrcProto;
+
+import com.google.common.collect.ComparisonChain;
+import org.apache.orc.StripeInformation;
+
+/**
+ * Stateless methods shared between RecordReaderImpl and EncodedReaderImpl.
+ */
+public class RecordReaderUtils {
+ private static final HadoopShims SHIMS = HadoopShims.Factory.get();
+
+ private static class DefaultDataReader implements DataReader {
+private FSDataInputStream file = null;
+private final ByteBufferAllocatorPool pool;
+private HadoopShims.ZeroCopyReaderShim zcr = null;
+private final FileSystem fs;
+private final Path path;
+private final boolean useZeroCopy;
+private final CompressionCodec codec;
+private final int bufferSize;
+private final int typeCount;
+
+private DefaultDataReader(DefaultDataReader other) {
+ this.pool = other.pool;
+ this.bufferSize = other.bufferSize;
+ this.typeCount = other.typeCount;
+ this.fs = other.fs;
+ this.path = other.path;
+ this.useZeroCopy = other.useZeroCopy;
+ this.codec = other.codec;
+}
+
+private DefaultDataReader(DataReaderProperties properties) {
+ this.fs = properties.getFileSystem();
+ this.path = properties.getPath();
+ this.useZeroCopy = properties.getZeroCopy();
+ this.codec = WriterImpl.createCodec(properties.getCompression());
+ this.bufferSize = properties.getBufferSize();
+ this.typeCount = properties.getTypeCount();
+ if (useZeroCopy) {
+this.pool = new ByteBufferAllocatorPool();
+ } else {
+this.pool = null;
+ }
+}
+
+@Override
+public void open() throws IOException {
+ this.file = fs.open(path);
+ if (useZeroCopy) {
+zcr = RecordReaderUtils.createZeroCopyShim(file, codec, pool);
+ } else {
+zcr = null;
+ }
+}
+
+@Override
+public OrcIndex readRowIndex(StripeInformation stripe,
+ OrcProto.StripeFooter footer,
+ boolean[] included,
+ OrcProto.RowIndex[] indexes,
+ boolean[] sargColumns,
+ OrcProto.BloomFilterIndex[] bloomFilterIndices
+ ) throws IOException {
+ if (file == null) {
+open();
+ }
+ if (footer == null) {
+footer = readStripeFooter(stripe);
+ }
+ if (indexes == null) {
+indexes = new OrcProto.RowIndex[typeCount];
+ }
+ if (bloomFilterIndices == null) {
+bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
+ }
+ long offset = stripe.getOffset();
+ List streams = footer.getStreamsList();
+ for (int i = 0; i < streams.size(); i++) {
+OrcProto.Stream stream = streams.get(i);
+OrcProto.Stream nextStream = null;
+if (i < streams.size() - 1) {
+ nextStream = streams.get(i+1);
+}
+int col =
[07/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcWideTable.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcWideTable.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcWideTable.java
deleted file mode 100644
index da2c681..000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcWideTable.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.junit.Test;
-
-public class TestOrcWideTable {
-
- @Test
- public void testBufferSizeFor1Col() throws IOException {
-assertEquals(128 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 *
1024,
-1, 128*1024));
- }
-
- @Test
- public void testBufferSizeFor50Col() throws IOException {
-assertEquals(256 * 1024, WriterImpl.getEstimatedBufferSize(256 * 1024 *
1024,
-50, 256*1024));
- }
-
- @Test
- public void testBufferSizeFor1000Col() throws IOException {
-assertEquals(32 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 *
1024,
-1000, 128*1024));
- }
-
- @Test
- public void testBufferSizeFor2000Col() throws IOException {
-assertEquals(16 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 *
1024,
-2000, 256*1024));
- }
-
- @Test
- public void testBufferSizeFor4000Col() throws IOException {
-assertEquals(8 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 * 1024,
-4000, 256*1024));
- }
-
- @Test
- public void testBufferSizeFor25000Col() throws IOException {
-assertEquals(4 * 1024, WriterImpl.getEstimatedBufferSize(512 * 1024 * 1024,
-25000, 256*1024));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRLEv2.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRLEv2.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRLEv2.java
deleted file mode 100644
index 1a3559e..000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRLEv2.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.PrintStream;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-public class TestRLEv2 {
- Path workDir = new Path(System.getProperty("test.tmp.dir",
- "target" + File.separator + "test" + File.separator + "tmp"));
- Path testFilePath;
- Configuration conf;
- FileSystem fs;
-
- @Rule
- public TestName testCaseName = new TestName();
-
- @Before
- public void openFileSystem () throws Exception {
-conf = new Configuration();
-fs = FileSystem.getLocal(conf);
-testFilePath = new Path(wor
[22/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/orc/src/java/org/apache/orc/tools/FileDump.java
--
diff --git a/orc/src/java/org/apache/orc/tools/FileDump.java
b/orc/src/java/org/apache/orc/tools/FileDump.java
new file mode 100644
index 000..e32027f
--- /dev/null
+++ b/orc/src/java/org/apache/orc/tools/FileDump.java
@@ -0,0 +1,934 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.orc.tools;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.orc.BloomFilterIO;
+import org.apache.orc.ColumnStatistics;
+import org.apache.orc.CompressionKind;
+import org.apache.orc.OrcFile;
+import org.apache.orc.Reader;
+import org.apache.orc.RecordReader;
+import org.apache.orc.TypeDescription;
+import org.apache.orc.Writer;
+import org.apache.orc.impl.AcidStats;
+import org.apache.orc.impl.ColumnStatisticsImpl;
+import org.apache.orc.impl.OrcAcidUtils;
+import org.apache.orc.impl.OrcIndex;
+import org.apache.orc.OrcProto;
+import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
+import org.apache.orc.impl.RecordReaderImpl;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONWriter;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+
+/**
+ * A tool for printing out the file structure of ORC files.
+ */
+public final class FileDump {
+ public static final String UNKNOWN = "UNKNOWN";
+ public static final String SEPARATOR = Strings.repeat("_", 120) + "\n";
+ public static final int DEFAULT_BLOCK_SIZE = 256 * 1024 * 1024;
+ public static final String DEFAULT_BACKUP_PATH =
System.getProperty("java.io.tmpdir");
+ public static final PathFilter HIDDEN_AND_SIDE_FILE_FILTER = new
PathFilter() {
+public boolean accept(Path p) {
+ String name = p.getName();
+ return !name.startsWith("_") && !name.startsWith(".") && !name.endsWith(
+ OrcAcidUtils.DELTA_SIDE_FILE_SUFFIX);
+}
+ };
+
+ // not used
+ private FileDump() {
+ }
+
+ public static void main(String[] args) throws Exception {
+Configuration conf = new Configuration();
+
+List rowIndexCols = null;
+Options opts = createOptions();
+CommandLine cli = new GnuParser().parse(opts, args);
+
+if (cli.hasOption('h')) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("orcfiledump", opts);
+ return;
+}
+
+boolean dumpData = cli.hasOption('d');
+boolean recover = cli.hasOption("recover");
+boolean skipDump = cli.hasOption("skip-dump");
+String backupPath = DEFAULT_BACKUP_PATH;
+if (c
[04/27] hive git commit: HIVE-11417. Move the ReaderImpl and RowReaderImpl to the ORC module, by making shims for the row by row reader. (omalley reviewed by prasanth_j)
http://git-wip-us.apache.org/repos/asf/hive/blob/ffb79509/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
deleted file mode 100644
index 6589692..000
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorOrcFile.java
+++ /dev/null
@@ -1,2791 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.io.orc;
-
-import com.google.common.collect.Lists;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hive.common.util.HiveTestUtils;
-import org.apache.orc.BinaryColumnStatistics;
-import org.apache.orc.BooleanColumnStatistics;
-import org.apache.orc.ColumnStatistics;
-import org.apache.orc.CompressionKind;
-import org.apache.orc.DataReader;
-import org.apache.orc.DecimalColumnStatistics;
-import org.apache.orc.DoubleColumnStatistics;
-import org.apache.orc.IntegerColumnStatistics;
-import org.apache.orc.impl.DataReaderProperties;
-import org.apache.orc.impl.MemoryManager;
-import org.apache.orc.impl.OrcIndex;
-import org.apache.orc.OrcProto;
-import org.apache.orc.OrcUtils;
-import org.apache.orc.StringColumnStatistics;
-import org.apache.orc.StripeInformation;
-import org.apache.orc.StripeStatistics;
-import org.apache.orc.TypeDescription;
-import org.apache.orc.Writer;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
-
-import java.io.File;
-import java.io.IOException;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.sql.Date;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertNull;
-import static junit.framework.Assert.assertTrue;
-
-/**
- * Tests for the vectorized reader and writer for ORC files.
- */
-public class TestVectorOrcFile {
-
- public static class InnerStruct {
-int int1;
-Text string1 = new Text();
-InnerStruct(int int1, Text string1) {
- this.int1 = int1;
- this.string1.set(string1);
-}
-InnerStruct(int int1, String string1) {
- this.int1 = int1;
- this.string1.set(string1);
-}
-
-public String toString() {
- return "{" + int1 + ", " + string1 + "}";
-}
- }
-
- public static class MiddleStruct {
-List list = new ArrayList();
-
-MiddleStruct(InnerStruct... items) {
- list.clear();
- list.addAll(Arrays.asList(items));
-}
- }
-
- private static InnerStruct inner(int i, String s) {
-return new InnerStruct(i, s);
- }
-
- private static Map map(InnerStruct... items) {
-Map result = new HashMap();
-for(InnerStruct i
hive git commit: HIVE-13197 : Add adapted constprog2.q and constprog_partitioner.q tests back (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Repository: hive Updated Branches: refs/heads/master ffb79509b -> f68cbcbfb HIVE-13197 : Add adapted constprog2.q and constprog_partitioner.q tests back (Ashutosh Chauhan via Jesus Camacho Rodriguez) Signed-off-by: Ashutosh Chauhan Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f68cbcbf Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f68cbcbf Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f68cbcbf Branch: refs/heads/master Commit: f68cbcbfb4892ed954948967a978b03f2e754227 Parents: ffb7950 Author: Ashutosh Chauhan Authored: Mon Mar 21 18:52:44 2016 -0700 Committer: Ashutosh Chauhan Committed: Fri May 20 17:47:50 2016 -0700 -- .../test/resources/testconfiguration.properties | 1 - ql/src/test/queries/clientpositive/constprog2.q | 12 +++ .../clientpositive/constprog_partitioner.q | 18 .../results/clientpositive/constprog2.q.out | 107 +++ .../clientpositive/constprog_partitioner.q.out | 77 - 5 files changed, 93 insertions(+), 122 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f68cbcbf/itests/src/test/resources/testconfiguration.properties -- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index c891d40..1ab914d 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -9,7 +9,6 @@ minimr.query.files=auto_sortmerge_join_16.q,\ bucketizedhiveinputformat.q,\ bucketmapjoin6.q,\ bucketmapjoin7.q,\ - constprog_partitioner.q,\ disable_merge_for_bucketing.q,\ empty_dir_in_table.q,\ exchgpartition2lel.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/f68cbcbf/ql/src/test/queries/clientpositive/constprog2.q -- diff --git a/ql/src/test/queries/clientpositive/constprog2.q b/ql/src/test/queries/clientpositive/constprog2.q new file mode 100644 index 000..dbebd34 --- /dev/null +++ b/ql/src/test/queries/clientpositive/constprog2.q @@ -0,0 +1,12 @@ +set hive.mapred.mode=nonstrict; +set hive.fetch.task.conversion=more; +set hive.optimize.constant.propagation=true; + +EXPLAIN +SELECT src1.key, src1.key + 1, src2.value + FROM srcbucket src1 join srcbucket src2 ON src1.key = src2.key AND src1.key = 86; + +EXPLAIN +SELECT src1.key, src1.key + 1, src2.value + FROM srcbucket src1 join srcbucket src2 ON src1.key = src2.key AND cast(src1.key as double) = 86; + http://git-wip-us.apache.org/repos/asf/hive/blob/f68cbcbf/ql/src/test/queries/clientpositive/constprog_partitioner.q -- diff --git a/ql/src/test/queries/clientpositive/constprog_partitioner.q b/ql/src/test/queries/clientpositive/constprog_partitioner.q new file mode 100644 index 000..ba55031 --- /dev/null +++ b/ql/src/test/queries/clientpositive/constprog_partitioner.q @@ -0,0 +1,18 @@ +set hive.mapred.mode=nonstrict; +set hive.fetch.task.conversion=more; +set hive.optimize.constant.propagation=true; + +set mapred.reduce.tasks=4; + +EXPLAIN +SELECT src1.key, src1.key + 1, src2.value + FROM srcbucket src1 join srcbucket src2 ON src1.key = src2.key AND src1.key = 100; + +EXPLAIN +SELECT l_partkey, l_suppkey +FROM lineitem li +WHERE li.l_linenumber = 1 AND + li.l_orderkey IN (SELECT l_orderkey FROM lineitem WHERE l_shipmode = 'AIR' AND l_linenumber = li.l_linenumber) +; + + http://git-wip-us.apache.org/repos/asf/hive/blob/f68cbcbf/ql/src/test/results/clientpositive/constprog2.q.out -- diff --git a/ql/src/test/results/clientpositive/constprog2.q.out b/ql/src/test/results/clientpositive/constprog2.q.out index d49e50b..0d76fc8 100644 --- a/ql/src/test/results/clientpositive/constprog2.q.out +++ b/ql/src/test/results/clientpositive/constprog2.q.out @@ -1,10 +1,11 @@ +Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product PREHOOK: query: EXPLAIN SELECT src1.key, src1.key + 1, src2.value - FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 86 + FROM srcbucket src1 join srcbucket src2 ON src1.key = src2.key AND src1.key = 86 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT src1.key, src1.key + 1, src2.value - FROM src src1 join src src2 ON src1.key = src2.key AND src1.key = 86 + FROM srcbucket src1 join srcbucket src2 ON src1.key = src2.key AND src1.key = 86 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -16,49 +17,45 @@ STAGE PLANS: Map Operator Tree: TableScan
hive git commit: HIVE-13699: Make JavaDataModel#get thread safe for parallel compilation (Peter Slawski via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master f68cbcbfb -> 2c3ebf8f2
HIVE-13699: Make JavaDataModel#get thread safe for parallel compilation (Peter
Slawski via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2c3ebf8f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2c3ebf8f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2c3ebf8f
Branch: refs/heads/master
Commit: 2c3ebf8f2259a46d102b598df3512f7226b6522a
Parents: f68cbcb
Author: Peter Slawski
Authored: Tue Apr 5 14:21:55 2016 -0700
Committer: Ashutosh Chauhan
Committed: Fri May 20 18:04:14 2016 -0700
--
.../hadoop/hive/ql/util/JavaDataModel.java | 34 +++
.../hadoop/hive/ql/util/JavaDataModelTest.java | 59
2 files changed, 82 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/2c3ebf8f/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
--
diff --git
a/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
index 151f30d..33b70c2 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
@@ -18,6 +18,11 @@
package org.apache.hadoop.hive.ql.util;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Estimation of memory footprint of object
*/
@@ -229,6 +234,8 @@ public enum JavaDataModel {
return (value + align - 1) & ~(align - 1);
}
+ private static final Logger LOG =
LoggerFactory.getLogger(JavaDataModel.class);
+
public static final int JAVA32_META = 12;
public static final int JAVA32_ARRAY_META = 16;
public static final int JAVA32_REF = 4;
@@ -246,22 +253,27 @@ public enum JavaDataModel {
public static final int PRIMITIVE_BYTE = 1;// byte
- private static JavaDataModel current;
+ private static final class LazyHolder {
+private static final JavaDataModel MODEL_FOR_SYSTEM = getModelForSystem();
+ }
- public static JavaDataModel get() {
-if (current != null) {
- return current;
-}
+ @VisibleForTesting
+ static JavaDataModel getModelForSystem() {
+String props = null;
try {
- String props = System.getProperty("sun.arch.data.model");
- if ("32".equals(props)) {
-return current = JAVA32;
- }
+ props = System.getProperty("sun.arch.data.model");
} catch (Exception e) {
- // ignore
+ LOG.warn("Failed to determine java data model, defaulting to 64", e);
+}
+if ("32".equals(props)) {
+ return JAVA32;
}
// TODO: separate model is needed for compressedOops, which can be guessed
from memory size.
-return current = JAVA64;
+return JAVA64;
+ }
+
+ public static JavaDataModel get() {
+return LazyHolder.MODEL_FOR_SYSTEM;
}
public static int round(int size) {
http://git-wip-us.apache.org/repos/asf/hive/blob/2c3ebf8f/storage-api/src/test/org/apache/hadoop/hive/ql/util/JavaDataModelTest.java
--
diff --git
a/storage-api/src/test/org/apache/hadoop/hive/ql/util/JavaDataModelTest.java
b/storage-api/src/test/org/apache/hadoop/hive/ql/util/JavaDataModelTest.java
new file mode 100644
index 000..35976cc
--- /dev/null
+++ b/storage-api/src/test/org/apache/hadoop/hive/ql/util/JavaDataModelTest.java
@@ -0,0 +1,59 @@
+package org.apache.hadoop.hive.ql.util;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+
+public final class JavaDataModelTest {
+
+ private static final String DATA_MODEL_PROPERTY = "sun.arch.data.model";
+
+ private String previousModelSetting;
+
+ @Before
+ public void setUp() throws Exception {
+previousModelSetting = System.getProperty(DATA_MODEL_PROPERTY);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+if (previousModelSetting != null) {
+ System.setProperty(DATA_MODEL_PROPERTY, previousModelSetting);
+} else {
+ System.clearProperty(DATA_MODEL_PROPERTY);
+}
+ }
+
+ @Test
+ public void testGetDoesNotReturnNull() throws Exception {
+JavaDataModel model = JavaDataModel.get();
+assertNotNull(model);
+ }
+
+ @Test
+ public void testGetModelForSystemWhenSetTo32() throws Exception {
+System.setProperty(DATA_MODEL_PROPERTY, "32");
+assertSame(JavaDataModel.JAVA32, JavaDataModel.getModelForSystem());
+ }
+
+ @Test
+ public void testGetModelForSystemWhen
