TAJO-806: CreateTableNode in CTAS uses a wrong schema as output schema and table schema. (hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/cde1bcaf Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/cde1bcaf Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/cde1bcaf Branch: refs/heads/window_function Commit: cde1bcaf563befb8889cafc23e72850782725488 Parents: d5347c6 Author: Hyunsik Choi <[email protected]> Authored: Wed Apr 30 18:25:55 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Wed Apr 30 18:25:55 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../apache/tajo/cli/ConnectDatabaseCommand.java | 2 +- .../tajo/engine/planner/LogicalPlanner.java | 7 +- .../java/org/apache/tajo/QueryTestCaseBase.java | 2 +- .../tajo/engine/query/TestInsertQuery.java | 197 ++++++++++--------- .../TestInsertQuery/full_table_csv_ddl.sql | 1 + .../TestInsertQuery/full_table_parquet_ddl.sql | 1 + .../queries/TestInsertQuery/table1_ddl.sql | 1 + .../TestInsertQuery/testInsertOverwrite.sql | 1 + .../testInsertOverwriteLocation.sql | 1 + ...stInsertOverwriteLocationWithCompression.sql | 1 + .../testInsertOverwriteSmallerColumns.sql | 1 + .../testInsertOverwriteWithAsterisk.sql | 1 + .../testInsertOverwriteWithCompression.sql | 1 + .../testInsertOverwriteWithCompression_ddl.sql | 4 + .../testInsertOverwriteWithTargetColumns.sql | 1 + ...sertOverwriteWithAsteriskUsingParquet.result | 4 + ...ertOverwriteWithAsteriskUsingParquet2.result | 4 + .../tajo/storage/parquet/ParquetScanner.java | 3 +- .../tajo/storage/parquet/TajoReadSupport.java | 16 +- .../tajo/storage/parquet/TajoWriteSupport.java | 4 - 21 files changed, 139 insertions(+), 117 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 40f5806..b71cbc6 100644 --- a/CHANGES +++ b/CHANGES @@ -29,6 +29,9 @@ Release 0.9.0 - unreleased BUG FIXES + TAJO-806: CreateTableNode in CTAS uses a wrong schema as output schema and + table schema. (hyunsik) + TAJO-805: Multiple constant in selection emits some columns. (Hyoungjun Kim via hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-client/src/main/java/org/apache/tajo/cli/ConnectDatabaseCommand.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/ConnectDatabaseCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/ConnectDatabaseCommand.java index 4ec4611..cb0bfdb 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/ConnectDatabaseCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/ConnectDatabaseCommand.java @@ -39,7 +39,7 @@ public class ConnectDatabaseCommand extends TajoShellCommand { } else if (cmd.length == 2) { if (!client.existDatabase(cmd[1])) { - context.getOutput().write(cmd[1] + " database not found\n"); + context.getOutput().write("Database '" + cmd[1] + "' not found\n"); } else { try { if (client.selectDatabase(cmd[1])) { http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java index f2ddf13..1c63354 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java @@ -1371,8 +1371,11 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex createTableNode.setOutSchema(tableSchema); createTableNode.setTableSchema(tableSchema); } else { - createTableNode.setOutSchema(subQuery.getOutSchema()); - createTableNode.setTableSchema(subQuery.getOutSchema()); + // Convert the schema of subquery into the target table's one. + Schema schema = new Schema(subQuery.getOutSchema()); + schema.setQualifier(createTableNode.getTableName()); + createTableNode.setOutSchema(schema); + createTableNode.setTableSchema(schema); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 961184c..bf5891a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -224,7 +224,7 @@ public class QueryTestCaseBase { } protected ResultSet executeString(String sql) throws Exception { - return testBase.execute(sql); + return client.executeQueryAndGetResult(sql); } /** http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java index e058943..c7df75c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java @@ -25,90 +25,77 @@ import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.apache.hadoop.io.compress.DeflateCodec; import org.apache.tajo.IntegrationTest; -import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.TpchTestBase; +import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import java.io.IOException; import java.sql.ResultSet; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.*; @Category(IntegrationTest.class) -public class TestInsertQuery { - private static TpchTestBase tpch; - public TestInsertQuery() throws IOException { - super(); - } - - @BeforeClass - public static void setUp() throws Exception { - tpch = TpchTestBase.getInstance(); - } +public class TestInsertQuery extends QueryTestCaseBase { @Test public final void testInsertOverwrite() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("InsertOverwrite"); - ResultSet res = tpch.execute("create table " + tableName +" (col1 int4, col2 int4, col3 float8)"); + ResultSet res = executeFile("table1_ddl.sql"); res.close(); - TajoTestingCluster cluster = tpch.getTestingCluster(); - CatalogService catalog = cluster.getMaster().getCatalog(); - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - res = tpch.execute("insert overwrite into " + tableName - + " select l_orderkey, l_partkey, l_quantity from lineitem"); + CatalogService catalog = testingCluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(getCurrentDatabase(), "table1")); + + res = executeFile("testInsertOverwrite.sql"); res.close(); - TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!cluster.isHCatalogStoreRunning()) { + TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "table1"); + if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } + + executeString("DROP TABLE table1 PURGE"); } @Test public final void testInsertOverwriteSmallerColumns() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("insertoverwritesmallercolumns"); - ResultSet res = tpch.execute("create table " + tableName + " (col1 int4, col2 int4, col3 float8)"); + ResultSet res = executeFile("table1_ddl.sql"); res.close(); - TajoTestingCluster cluster = tpch.getTestingCluster(); - CatalogService catalog = cluster.getMaster().getCatalog(); - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - TableDesc originalDesc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - res = tpch.execute("insert overwrite into " + tableName + " select l_orderkey from lineitem"); + CatalogService catalog = testingCluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(getCurrentDatabase(), "table1")); + TableDesc originalDesc = catalog.getTableDesc(getCurrentDatabase(), "table1"); + + res = executeFile("testInsertOverwriteSmallerColumns.sql"); res.close(); - TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!cluster.isHCatalogStoreRunning()) { + TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "table1"); + if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } assertEquals(originalDesc.getSchema(), desc.getSchema()); + + executeString("DROP TABLE table1 PURGE"); } @Test public final void testInsertOverwriteWithTargetColumns() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("InsertOverwriteWithTargetColumns"); - ResultSet res = tpch.execute("create table " + tableName + " (col1 int4, col2 int4, col3 float8)"); + ResultSet res = executeFile("table1_ddl.sql"); res.close(); - TajoTestingCluster cluster = tpch.getTestingCluster(); - CatalogService catalog = cluster.getMaster().getCatalog(); - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - TableDesc originalDesc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - res = tpch.execute( - "insert overwrite into " + tableName + " (col1, col3) select l_orderkey, l_quantity from lineitem"); + CatalogService catalog = testingCluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(getCurrentDatabase(), "table1")); + TableDesc originalDesc = catalog.getTableDesc(getCurrentDatabase(), "table1"); + + res = executeFile("testInsertOverwriteWithTargetColumns.sql"); res.close(); - TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!cluster.isHCatalogStoreRunning()) { + TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "table1"); + if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(5, desc.getStats().getNumRows().intValue()); } - res = tpch.execute("select * from " + tableName); + res = executeString("select * from " + CatalogUtil.denormalizeIdentifier(getCurrentDatabase()) + ".table1"); assertTrue(res.next()); assertEquals(1, res.getLong(1)); @@ -143,21 +130,22 @@ public class TestInsertQuery { res.close(); assertEquals(originalDesc.getSchema(), desc.getSchema()); + + executeString("DROP TABLE table1 PURGE"); } @Test public final void testInsertOverwriteWithAsterisk() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testinsertoverwritewithasterisk"); - ResultSet res = tpch.execute("create table " + tableName + " as select * from lineitem"); + ResultSet res = executeFile("full_table_csv_ddl.sql"); res.close(); - TajoTestingCluster cluster = tpch.getTestingCluster(); - CatalogService catalog = cluster.getMaster().getCatalog(); - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - res = tpch.execute("insert overwrite into " + tableName + " select * from lineitem where l_orderkey = 3"); + CatalogService catalog = testingCluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(getCurrentDatabase(), "full_table_csv")); + + res = executeString("insert overwrite into full_table_csv select * from default.lineitem where l_orderkey = 3"); res.close(); - TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!cluster.isHCatalogStoreRunning()) { + TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "full_table_csv"); + if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(2, desc.getStats().getNumRows().intValue()); } } @@ -165,28 +153,26 @@ public class TestInsertQuery { @Test public final void testInsertOverwriteIntoSelect() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("insertoverwriteintoselect"); - ResultSet res = tpch.execute( - "create table " + tableName + " as select l_orderkey from lineitem"); + ResultSet res = executeString("create table " + tableName + " as select l_orderkey from default.lineitem"); assertFalse(res.next()); res.close(); - TajoTestingCluster cluster = tpch.getTestingCluster(); - CatalogService catalog = cluster.getMaster().getCatalog(); - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - TableDesc orderKeys = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!cluster.isHCatalogStoreRunning()) { + + CatalogService catalog = testingCluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(getCurrentDatabase(), tableName)); + TableDesc orderKeys = catalog.getTableDesc(getCurrentDatabase(), tableName); + if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(5, orderKeys.getStats().getNumRows().intValue()); } // this query will result in the two rows. - res = tpch.execute( - "insert overwrite into " + tableName + " select l_orderkey from lineitem where l_orderkey = 3"); + res = executeString("insert overwrite into " + tableName + " select l_orderkey from default.lineitem where l_orderkey = 3"); assertFalse(res.next()); res.close(); - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - orderKeys = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!cluster.isHCatalogStoreRunning()) { + assertTrue(catalog.existsTable(getCurrentDatabase(), tableName)); + orderKeys = catalog.getTableDesc(getCurrentDatabase(), tableName); + if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(2, orderKeys.getStats().getNumRows().intValue()); } } @@ -194,26 +180,25 @@ public class TestInsertQuery { @Test public final void testInsertOverwriteCapitalTableName() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testInsertOverwriteCapitalTableName"); - ResultSet res = tpch.execute("create table " + tableName + " as select * from lineitem"); + ResultSet res = executeString("create table " + tableName + " as select * from default.lineitem"); res.close(); - TajoTestingCluster cluster = tpch.getTestingCluster(); - CatalogService catalog = cluster.getMaster().getCatalog(); - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - res = tpch.execute("insert overwrite into " + tableName + " select * from lineitem where l_orderkey = 3"); + CatalogService catalog = testingCluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(getCurrentDatabase(), tableName)); + + res = executeString("insert overwrite into " + tableName + " select * from default.lineitem where l_orderkey = 3"); res.close(); - TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!cluster.isHCatalogStoreRunning()) { + TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), tableName); + if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(2, desc.getStats().getNumRows().intValue()); } } @Test public final void testInsertOverwriteLocation() throws Exception { - ResultSet res = - tpch.execute("insert overwrite into location '/tajo-data/testInsertOverwriteCapitalTableName' select * from lineitem where l_orderkey = 3"); + ResultSet res = executeQuery(); res.close(); - FileSystem fs = FileSystem.get(tpch.getTestingCluster().getConfiguration()); + FileSystem fs = FileSystem.get(testingCluster.getConfiguration()); assertTrue(fs.exists(new Path("/tajo-data/testInsertOverwriteCapitalTableName"))); assertEquals(1, fs.listStatus(new Path("/tajo-data/testInsertOverwriteCapitalTableName")).length); } @@ -221,22 +206,22 @@ public class TestInsertQuery { @Test public final void testInsertOverwriteWithCompression() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testInsertOverwriteWithCompression"); - ResultSet res = tpch.execute("create table " + tableName + " (col1 int4, col2 int4, col3 float8) USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec')"); + ResultSet res = executeFile("testInsertOverwriteWithCompression_ddl.sql"); res.close(); - TajoTestingCluster cluster = tpch.getTestingCluster(); - CatalogService catalog = cluster.getMaster().getCatalog(); - assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); - res = tpch.execute("insert overwrite into " + tableName + " select l_orderkey, l_partkey, l_quantity from lineitem where l_orderkey = 3"); + CatalogService catalog = testingCluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(getCurrentDatabase(), tableName)); + + res = executeQuery(); res.close(); - TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); - if (!cluster.isHCatalogStoreRunning()) { + TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), tableName); + if (!testingCluster.isHCatalogStoreRunning()) { assertEquals(2, desc.getStats().getNumRows().intValue()); } - FileSystem fs = FileSystem.get(tpch.getTestingCluster().getConfiguration()); + FileSystem fs = FileSystem.get(testingCluster.getConfiguration()); assertTrue(fs.exists(desc.getPath())); - CompressionCodecFactory factory = new CompressionCodecFactory(tpch.getTestingCluster().getConfiguration()); + CompressionCodecFactory factory = new CompressionCodecFactory(testingCluster.getConfiguration()); for (FileStatus file : fs.listStatus(desc.getPath())) { CompressionCodec codec = factory.getCodec(file.getPath()); @@ -246,17 +231,45 @@ public class TestInsertQuery { @Test public final void testInsertOverwriteLocationWithCompression() throws Exception { - ResultSet res = tpch.execute("insert overwrite into location '/tajo-data/testInsertOverwriteLocationWithCompression' USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') select * from lineitem where l_orderkey = 3"); - res.close(); - FileSystem fs = FileSystem.get(tpch.getTestingCluster().getConfiguration()); - Path path = new Path("/tajo-data/testInsertOverwriteLocationWithCompression"); - assertTrue(fs.exists(path)); - assertEquals(1, fs.listStatus(path).length); + if (!testingCluster.isHCatalogStoreRunning()) { + ResultSet res = executeQuery(); + res.close(); + FileSystem fs = FileSystem.get(testingCluster.getConfiguration()); + Path path = new Path("/tajo-data/testInsertOverwriteLocationWithCompression"); + assertTrue(fs.exists(path)); + assertEquals(1, fs.listStatus(path).length); + + CompressionCodecFactory factory = new CompressionCodecFactory(testingCluster.getConfiguration()); + for (FileStatus file : fs.listStatus(path)){ + CompressionCodec codec = factory.getCodec(file.getPath()); + assertTrue(codec instanceof DeflateCodec); + } + } + } - CompressionCodecFactory factory = new CompressionCodecFactory(tpch.getTestingCluster().getConfiguration()); - for (FileStatus file : fs.listStatus(path)){ - CompressionCodec codec = factory.getCodec(file.getPath()); - assertTrue(codec instanceof DeflateCodec); + @Test + public final void testInsertOverwriteWithAsteriskUsingParquet() throws Exception { + if (!testingCluster.isHCatalogStoreRunning()) { + ResultSet res = executeFile("full_table_parquet_ddl.sql"); + res.close(); + + CatalogService catalog = testingCluster.getMaster().getCatalog(); + assertTrue(catalog.existsTable(getCurrentDatabase(), "full_table_parquet")); + + res = executeString("insert overwrite into full_table_parquet select * from default.lineitem where l_orderkey = 3"); + res.close(); + TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "full_table_parquet"); + if (!testingCluster.isHCatalogStoreRunning()) { + assertEquals(2, desc.getStats().getNumRows().intValue()); + } + + res = executeString("select * from full_table_parquet;"); + assertResultSet(res); + + res = executeString("select l_orderkey, l_partkey from full_table_parquet;"); + assertResultSet(res, "testInsertOverwriteWithAsteriskUsingParquet2.result"); + + executeString("DROP TABLE full_table_parquet_ddl PURGE"); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/full_table_csv_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/full_table_csv_ddl.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/full_table_csv_ddl.sql new file mode 100644 index 0000000..729ca83 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/full_table_csv_ddl.sql @@ -0,0 +1 @@ +create table full_table_csv as select * from default.lineitem \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/full_table_parquet_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/full_table_parquet_ddl.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/full_table_parquet_ddl.sql new file mode 100644 index 0000000..1025b3b --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/full_table_parquet_ddl.sql @@ -0,0 +1 @@ +create table full_table_parquet using parquet as select * from default.lineitem; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/table1_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/table1_ddl.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/table1_ddl.sql new file mode 100644 index 0000000..b8333c4 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/table1_ddl.sql @@ -0,0 +1 @@ +create table table1 (col1 int4, col2 int4, col3 float8) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwrite.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwrite.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwrite.sql new file mode 100644 index 0000000..db18c0e --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwrite.sql @@ -0,0 +1 @@ +insert overwrite into table1 select l_orderkey, l_partkey, l_quantity from default.lineitem; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocation.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocation.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocation.sql new file mode 100644 index 0000000..28ac95c --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocation.sql @@ -0,0 +1 @@ +insert overwrite into location '/tajo-data/testInsertOverwriteCapitalTableName' select * from default.lineitem where l_orderkey = 3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocationWithCompression.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocationWithCompression.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocationWithCompression.sql new file mode 100644 index 0000000..c3c67c7 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteLocationWithCompression.sql @@ -0,0 +1 @@ +insert overwrite into location '/tajo-data/testInsertOverwriteLocationWithCompression' USING csv WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') select * from default.lineitem where l_orderkey = 3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteSmallerColumns.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteSmallerColumns.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteSmallerColumns.sql new file mode 100644 index 0000000..ce3fc0d --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteSmallerColumns.sql @@ -0,0 +1 @@ +insert overwrite into table1 select l_orderkey from default.lineitem; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithAsterisk.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithAsterisk.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithAsterisk.sql new file mode 100644 index 0000000..4856e18 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithAsterisk.sql @@ -0,0 +1 @@ +insert overwrite into full_table_csv select * from default.lineitem where l_orderkey = 3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression.sql new file mode 100644 index 0000000..8157c1d --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression.sql @@ -0,0 +1 @@ +insert overwrite into testInsertOverwriteWithCompression select l_orderkey, l_partkey, l_quantity from default.lineitem where l_orderkey = 3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression_ddl.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression_ddl.sql new file mode 100644 index 0000000..dd49f84 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithCompression_ddl.sql @@ -0,0 +1,4 @@ +create table + testInsertOverwriteWithCompression (col1 int4, col2 int4, col3 float8) +USING csv +WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec'); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithTargetColumns.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithTargetColumns.sql b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithTargetColumns.sql new file mode 100644 index 0000000..67f9773 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestInsertQuery/testInsertOverwriteWithTargetColumns.sql @@ -0,0 +1 @@ +insert overwrite into table1 (col1, col3) select l_orderkey, l_quantity from default.lineitem; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskUsingParquet.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskUsingParquet.result b/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskUsingParquet.result new file mode 100644 index 0000000..fd79944 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskUsingParquet.result @@ -0,0 +1,4 @@ +l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment +------------------------------- +3,2,1798,1,45.0,54058.05,0.06,0.0,R,F,1994-02-02,1994-01-04,1994-02-23,NONE,AIR,ongside of the furiously brave acco +3,3,6540,2,49.0,46796.47,0.1,0.0,R,F,1993-11-09,1993-12-20,1993-11-24,TAKE BACK RETURN,RAIL, unusual accounts. eve \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskUsingParquet2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskUsingParquet2.result b/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskUsingParquet2.result new file mode 100644 index 0000000..1bbb067 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestInsertQuery/testInsertOverwriteWithAsteriskUsingParquet2.result @@ -0,0 +1,4 @@ +l_orderkey,l_partkey +------------------------------- +3,2 +3,3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java index 38d8ca4..36b89b8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/ParquetScanner.java @@ -55,8 +55,7 @@ public class ParquetScanner extends FileScanner { if (targets == null) { targets = schema.toArray(); } - reader = new TajoParquetReader(fragment.getPath(), schema, - new Schema(targets)); + reader = new TajoParquetReader(fragment.getPath(), schema, new Schema(targets)); super.init(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java index 4733a2f..269f782 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoReadSupport.java @@ -42,11 +42,6 @@ public class TajoReadSupport extends ReadSupport<Tuple> { private Schema requestedSchema; /** - * The key for the Tajo schema stored in the Parquet file metadata. - */ - public static final String TAJO_SCHEMA_METADATA_KEY = "tajo.schema"; - - /** * Creates a new TajoReadSupport. * * @param requestedSchema The Tajo schema of the requested projection passed @@ -100,16 +95,7 @@ public class TajoReadSupport extends ReadSupport<Tuple> { Map<String, String> keyValueMetaData, MessageType fileSchema, ReadContext readContext) { - Schema tajoReadSchema = null; - String metadataReadSchema = keyValueMetaData.get(TAJO_SCHEMA_METADATA_KEY); - if (metadataReadSchema != null) { - tajoReadSchema = CatalogGsonHelper.fromJson( - metadataReadSchema, Schema.class); - } else { - tajoReadSchema = readSchema; - } MessageType parquetRequestedSchema = readContext.getRequestedSchema(); - return new TajoRecordMaterializer(parquetRequestedSchema, requestedSchema, - tajoReadSchema); + return new TajoRecordMaterializer(parquetRequestedSchema, requestedSchema, readSchema); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/cde1bcaf/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java index 83b2f7b..35165de 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java @@ -41,8 +41,6 @@ import org.apache.tajo.datum.Datum; * Users should use {@link ParquetAppender} and not this class directly. */ public class TajoWriteSupport extends WriteSupport<Tuple> { - private static final String TAJO_SCHEMA = "parquet.tajo.schema"; - private RecordConsumer recordConsumer; private MessageType rootSchema; private Schema rootTajoSchema; @@ -66,8 +64,6 @@ public class TajoWriteSupport extends WriteSupport<Tuple> { @Override public WriteContext init(Configuration configuration) { Map<String, String> extraMetaData = new HashMap<String, String>(); - extraMetaData.put(TajoReadSupport.TAJO_SCHEMA_METADATA_KEY, - rootTajoSchema.toJson()); return new WriteContext(rootSchema, extraMetaData); }
