[CARBONDATA-1609] Thrift Changes to support Pre-aggregate This closes #1425
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/037f630a Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/037f630a Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/037f630a Branch: refs/heads/pre-aggregate Commit: 037f630a5ab331a9f45a6413e9052d4672c8d675 Parents: 80195da Author: kumarvishal <kumarvishal.1...@gmail.com> Authored: Sat Oct 21 16:31:15 2017 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Sat Nov 11 17:29:56 2017 +0530 ---------------------------------------------------------------------- .../metadata/converter/SchemaConverter.java | 10 ++ .../ThriftWrapperSchemaConverterImpl.java | 147 +++++++++++++++++-- .../core/metadata/schema/table/CarbonTable.java | 24 +-- .../metadata/schema/table/DataMapSchema.java | 121 +++++++++++++++ .../schema/table/RelationIdentifier.java | 88 +++++++++++ .../core/metadata/schema/table/TableInfo.java | 83 ++++++++++- .../core/metadata/schema/table/TableSchema.java | 26 ++++ .../schema/table/column/ColumnSchema.java | 48 +++++- .../util/AbstractDataFileFooterConverter.java | 28 ++++ format/src/main/thrift/schema.thrift | 31 ++++ 10 files changed, 580 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java index 4faa017..bfbb6f7 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/SchemaConverter.java @@ -18,6 +18,7 @@ package org.apache.carbondata.core.metadata.converter; import org.apache.carbondata.core.metadata.schema.SchemaEvolution; import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry; +import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; import org.apache.carbondata.core.metadata.schema.table.TableInfo; import org.apache.carbondata.core.metadata.schema.table.TableSchema; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; @@ -100,4 +101,13 @@ public interface SchemaConverter { */ TableInfo fromExternalToWrapperTableInfo(org.apache.carbondata.format.TableInfo externalTableInfo, String dbName, String tableName, String storePath); + + /** + * method to convert thrift datamap schema object to wrapper + * data map object + * @param thriftchildSchema + * @return DataMapSchema + */ + DataMapSchema fromExternalToWrapperDataMapSchema( + org.apache.carbondata.format.DataMapSchema thriftchildSchema); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java index adcac7d..5ffc612 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java @@ -30,9 +30,12 @@ import org.apache.carbondata.core.metadata.schema.PartitionInfo; import org.apache.carbondata.core.metadata.schema.SchemaEvolution; import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry; import org.apache.carbondata.core.metadata.schema.partition.PartitionType; +import org.apache.carbondata.core.metadata.schema.table.DataMapSchema; +import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier; import org.apache.carbondata.core.metadata.schema.table.TableInfo; import org.apache.carbondata.core.metadata.schema.table.TableSchema; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; +import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation; /** * Thrift schema to carbon schema converter and vice versa @@ -40,11 +43,11 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { /* (non-Javadoc) - * Converts from wrapper to thrift schema evolution entry - */ + * Converts from wrapper to thrift schema evolution entry + */ @Override public org.apache.carbondata.format.SchemaEvolutionEntry - fromWrapperToExternalSchemaEvolutionEntry(SchemaEvolutionEntry wrapperSchemaEvolutionEntry) { + fromWrapperToExternalSchemaEvolutionEntry(SchemaEvolutionEntry wrapperSchemaEvolutionEntry) { org.apache.carbondata.format.SchemaEvolutionEntry thriftSchemaEvolutionEntry = new org.apache.carbondata.format.SchemaEvolutionEntry( wrapperSchemaEvolutionEntry.getTimeStamp()); @@ -159,7 +162,6 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { */ @Override public org.apache.carbondata.format.ColumnSchema fromWrapperToExternalColumnSchema( ColumnSchema wrapperColumnSchema) { - List<org.apache.carbondata.format.Encoding> encoders = new ArrayList<org.apache.carbondata.format.Encoding>(); for (Encoding encoder : wrapperColumnSchema.getEncodingList()) { @@ -188,13 +190,18 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { thriftColumnSchema.setInvisible(wrapperColumnSchema.isInvisible()); thriftColumnSchema.setColumnReferenceId(wrapperColumnSchema.getColumnReferenceId()); thriftColumnSchema.setSchemaOrdinal(wrapperColumnSchema.getSchemaOrdinal()); - if (wrapperColumnSchema.isSortColumn()) { Map<String, String> properties = new HashMap<String, String>(); properties.put(CarbonCommonConstants.SORT_COLUMNS, "true"); thriftColumnSchema.setColumnProperties(properties); } - + thriftColumnSchema.setAggregate_function(wrapperColumnSchema.getAggFunction()); + List<ParentColumnTableRelation> parentColumnTableRelations = + wrapperColumnSchema.getParentColumnTableRelations(); + if (null != parentColumnTableRelations) { + thriftColumnSchema.setParentColumnTableRelations( + wrapperToThriftRelationList(parentColumnTableRelations)); + } return thriftColumnSchema; } @@ -280,11 +287,79 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { */ @Override public org.apache.carbondata.format.TableInfo fromWrapperToExternalTableInfo( TableInfo wrapperTableInfo, String dbName, String tableName) { - org.apache.carbondata.format.TableSchema thriftFactTable = fromWrapperToExternalTableSchema(wrapperTableInfo.getFactTable()); - return new org.apache.carbondata.format.TableInfo(thriftFactTable, new ArrayList<org.apache - .carbondata.format.TableSchema>()); + org.apache.carbondata.format.TableInfo tableInfo = + new org.apache.carbondata.format.TableInfo(thriftFactTable, + new ArrayList<org.apache.carbondata.format.TableSchema>()); + List<DataMapSchema> wrapperChildSchemaList = wrapperTableInfo.getDataMapSchemaList(); + if (null != wrapperChildSchemaList) { + List<org.apache.carbondata.format.DataMapSchema> thriftChildSchemas = + fromWrapperToExternalChildSchemaList(wrapperChildSchemaList); + tableInfo.setDataMapSchemas(thriftChildSchemas); + } + return tableInfo; + } + + private List<org.apache.carbondata.format.RelationIdentifier> fromWrapperToExternalRI( + List<RelationIdentifier> relationIdentifiersList) { + List<org.apache.carbondata.format.RelationIdentifier> thriftRelationIdentifierList = + new ArrayList<>(); + for (RelationIdentifier relationIdentifier : relationIdentifiersList) { + org.apache.carbondata.format.RelationIdentifier thriftRelationIdentifier = + new org.apache.carbondata.format.RelationIdentifier(); + thriftRelationIdentifier.setDatabaseName(relationIdentifier.getDatabaseName()); + thriftRelationIdentifier.setTableName(relationIdentifier.getTableName()); + thriftRelationIdentifier.setTableId(relationIdentifier.getTableId()); + thriftRelationIdentifierList.add(thriftRelationIdentifier); + } + return thriftRelationIdentifierList; + } + + private List<org.apache.carbondata.format.DataMapSchema> fromWrapperToExternalChildSchemaList( + List<DataMapSchema> wrapperChildSchemaList) { + List<org.apache.carbondata.format.DataMapSchema> thriftChildSchemas = new ArrayList<>(); + for (DataMapSchema wrapperChildSchema : wrapperChildSchemaList) { + org.apache.carbondata.format.DataMapSchema thriftChildSchema = + new org.apache.carbondata.format.DataMapSchema(); + org.apache.carbondata.format.RelationIdentifier relationIdentifier = + new org.apache.carbondata.format.RelationIdentifier(); + relationIdentifier + .setDatabaseName(wrapperChildSchema.getRelationIdentifier().getDatabaseName()); + relationIdentifier.setTableName(wrapperChildSchema.getRelationIdentifier().getTableName()); + relationIdentifier.setTableId(wrapperChildSchema.getRelationIdentifier().getTableId()); + thriftChildSchema.setRelationIdentifire(relationIdentifier); + thriftChildSchema.setProperties(wrapperChildSchema.getProperties()); + thriftChildSchema.setClassName(wrapperChildSchema.getClassName()); + thriftChildSchema.setChildTableSchema( + fromWrapperToExternalTableSchema(wrapperChildSchema.getChildSchema())); + thriftChildSchemas.add(thriftChildSchema); + } + return thriftChildSchemas; + } + + private List<org.apache.carbondata.format.ParentColumnTableRelation> wrapperToThriftRelationList( + List<ParentColumnTableRelation> wrapperColumnRelations) { + List<org.apache.carbondata.format.ParentColumnTableRelation> thriftColumnRelationList = + new ArrayList<>(); + + for (ParentColumnTableRelation wrapperColumnRealtion : wrapperColumnRelations) { + org.apache.carbondata.format.ParentColumnTableRelation thriftColumnTableRelation = + new org.apache.carbondata.format.ParentColumnTableRelation(); + thriftColumnTableRelation.setColumnId(wrapperColumnRealtion.getColumnId()); + thriftColumnTableRelation.setColumnName(wrapperColumnRealtion.getColumnName()); + org.apache.carbondata.format.RelationIdentifier thriftRelationIdentifier = + new org.apache.carbondata.format.RelationIdentifier(); + thriftRelationIdentifier + .setDatabaseName(wrapperColumnRealtion.getRelationIdentifier().getDatabaseName()); + thriftRelationIdentifier + .setTableName(wrapperColumnRealtion.getRelationIdentifier().getTableName()); + thriftRelationIdentifier + .setTableId(wrapperColumnRealtion.getRelationIdentifier().getTableId()); + thriftColumnTableRelation.setRelationIdentifier(thriftRelationIdentifier); + thriftColumnRelationList.add(thriftColumnTableRelation); + } + return thriftColumnRelationList; } /* (non-Javadoc) @@ -436,9 +511,17 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { wrapperColumnSchema.setSortColumn(true); } } + wrapperColumnSchema.setAggFunction(externalColumnSchema.getAggregate_function()); + List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation = + externalColumnSchema.getParentColumnTableRelations(); + if (null != parentColumnTableRelation) { + wrapperColumnSchema.setParentColumnTableRelations( + fromExtrenalToWrapperParentTableColumnRelations(parentColumnTableRelation)); + } return wrapperColumnSchema; } + private PartitionType fromExternalToWrapperPartitionType( org.apache.carbondata.format.PartitionType externalPartitionType) { if (null == externalPartitionType) { @@ -508,7 +591,7 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { org.apache.carbondata.format.BucketingInfo externalBucketInfo) { List<ColumnSchema> listOfColumns = new ArrayList<ColumnSchema>(); for (org.apache.carbondata.format.ColumnSchema externalColumnSchema : - externalBucketInfo.table_columns) { + externalBucketInfo.table_columns) { listOfColumns.add(fromExternalToWrapperColumnSchema(externalColumnSchema)); } return new BucketingInfo(listOfColumns, externalBucketInfo.number_of_buckets); @@ -531,7 +614,51 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter { wrapperTableInfo.setStorePath(storePath); wrapperTableInfo.setFactTable( fromExternalToWrapperTableSchema(externalTableInfo.getFact_table(), tableName)); + if (null != externalTableInfo.getDataMapSchemas()) { + wrapperTableInfo.setDataMapSchemaList( + fromExternalToWrapperChildSchemaList(externalTableInfo.getDataMapSchemas())); + } return wrapperTableInfo; } + @Override public DataMapSchema fromExternalToWrapperDataMapSchema( + org.apache.carbondata.format.DataMapSchema thriftDataMapSchema) { + RelationIdentifier relationIdentifier = + new RelationIdentifier(thriftDataMapSchema.getRelationIdentifire().getDatabaseName(), + thriftDataMapSchema.getRelationIdentifire().getTableName(), + thriftDataMapSchema.getRelationIdentifire().getTableId()); + DataMapSchema childSchema = new DataMapSchema(thriftDataMapSchema.getClassName()); + childSchema.setProperties(thriftDataMapSchema.getProperties()); + childSchema.setChildSchema( + fromExternalToWrapperTableSchema(thriftDataMapSchema.getChildTableSchema(), + relationIdentifier.getTableName())); + childSchema.setRelationIdentifier(relationIdentifier); + return childSchema; + } + + private List<ParentColumnTableRelation> fromExtrenalToWrapperParentTableColumnRelations( + List<org.apache.carbondata.format.ParentColumnTableRelation> thirftParentColumnRelation) { + List<ParentColumnTableRelation> parentColumnTableRelationList = new ArrayList<>(); + for (org.apache.carbondata.format.ParentColumnTableRelation carbonTableRelation : + thirftParentColumnRelation) { + RelationIdentifier relationIdentifier = + new RelationIdentifier(carbonTableRelation.getRelationIdentifier().getDatabaseName(), + carbonTableRelation.getRelationIdentifier().getTableName(), + carbonTableRelation.getRelationIdentifier().getTableId()); + ParentColumnTableRelation parentColumnTableRelation = + new ParentColumnTableRelation(relationIdentifier, carbonTableRelation.getColumnId(), + carbonTableRelation.getColumnName()); + parentColumnTableRelationList.add(parentColumnTableRelation); + } + return parentColumnTableRelationList; + } + + public List<DataMapSchema> fromExternalToWrapperChildSchemaList( + List<org.apache.carbondata.format.DataMapSchema> childSchemaList) { + List<DataMapSchema> childSchemas = new ArrayList<>(); + for (org.apache.carbondata.format.DataMapSchema childSchemaThrift : childSchemaList) { + childSchemas.add(fromExternalToWrapperDataMapSchema(childSchemaThrift)); + } + return childSchemas; + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java index d4aaa29..e63f4e3 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java @@ -214,7 +214,7 @@ public class CarbonTable implements Serializable { if (columnSchema.getNumberOfChild() > 0) { CarbonDimension complexDimension = new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal); + columnSchema.getSchemaOrdinal(), -1, -1, ++complexTypeOrdinal); complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild()); allDimensions.add(complexDimension); dimensionOrdinal = @@ -228,8 +228,8 @@ public class CarbonTable implements Serializable { } if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) { CarbonDimension dimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), -1, -1, -1); + new CarbonDimension(columnSchema, dimensionOrdinal++, + columnSchema.getSchemaOrdinal(), -1, -1, -1); if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) { this.numberOfNoDictSortColumns++; } @@ -238,8 +238,8 @@ public class CarbonTable implements Serializable { } else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY) && columnSchema.getColumnGroupId() == -1) { CarbonDimension dimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1); + new CarbonDimension(columnSchema, dimensionOrdinal++, + columnSchema.getSchemaOrdinal(), keyOrdinal++, -1, -1); allDimensions.add(dimension); primitiveDimensions.add(dimension); } else { @@ -247,15 +247,15 @@ public class CarbonTable implements Serializable { previousColumnGroupId == columnSchema.getColumnGroupId() ? ++columnGroupOrdinal : 0; previousColumnGroupId = columnSchema.getColumnGroupId(); CarbonDimension dimension = new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), keyOrdinal++, - columnGroupOrdinal, -1); + columnSchema.getSchemaOrdinal(), keyOrdinal++, + columnGroupOrdinal, -1); allDimensions.add(dimension); primitiveDimensions.add(dimension); } } } else { allMeasures.add(new CarbonMeasure(columnSchema, measureOrdinal++, - columnSchema.getSchemaOrdinal())); + columnSchema.getSchemaOrdinal())); } } fillVisibleDimensions(tableSchema.getTableName()); @@ -290,14 +290,14 @@ public class CarbonTable implements Serializable { */ private int readAllComplexTypeChildrens(int dimensionOrdinal, int childCount, List<ColumnSchema> listOfColumns, CarbonDimension parentDimension, - List<CarbonDimension> primitiveDimensions) { + List<CarbonDimension> primitiveDimensions) { for (int i = 0; i < childCount; i++) { ColumnSchema columnSchema = listOfColumns.get(dimensionOrdinal); if (columnSchema.isDimensionColumn()) { if (columnSchema.getNumberOfChild() > 0) { CarbonDimension complexDimension = new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), -1, -1, -1); + columnSchema.getSchemaOrdinal(), -1, -1, -1); complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild()); parentDimension.getListOfChildDimensions().add(complexDimension); dimensionOrdinal = @@ -305,8 +305,8 @@ public class CarbonTable implements Serializable { listOfColumns, complexDimension, primitiveDimensions); } else { CarbonDimension carbonDimension = - new CarbonDimension(columnSchema, dimensionOrdinal++, - columnSchema.getSchemaOrdinal(), -1, -1, -1); + new CarbonDimension(columnSchema, dimensionOrdinal++, + columnSchema.getSchemaOrdinal(), -1, -1, -1); parentDimension.getListOfChildDimensions().add(carbonDimension); primitiveDimensions.add(carbonDimension); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java new file mode 100644 index 0000000..5f92ec8 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DataMapSchema.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.metadata.schema.table; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +/** + * Child schema class to maintain the child table details inside parent table + */ +public class DataMapSchema implements Serializable, Writable { + + private String className; + + private RelationIdentifier relationIdentifier; + /** + * child table schema + */ + private TableSchema childSchema; + + /** + * relation properties + */ + private Map<String, String> properties; + + public DataMapSchema(String className) { + this.className = className; + } + + public String getClassName() { + return className; + } + + public TableSchema getChildSchema() { + return childSchema; + } + + public RelationIdentifier getRelationIdentifier() { + return relationIdentifier; + } + + public Map<String, String> getProperties() { + return properties; + } + + public void setRelationIdentifier(RelationIdentifier relationIdentifier) { + this.relationIdentifier = relationIdentifier; + } + + public void setChildSchema(TableSchema childSchema) { + this.childSchema = childSchema; + } + + public void setProperties(Map<String, String> properties) { + this.properties = properties; + } + + @Override public void write(DataOutput out) throws IOException { + out.writeUTF(className); + boolean isRelationIdentifierExists = null != relationIdentifier; + out.writeBoolean(isRelationIdentifierExists); + if (isRelationIdentifierExists) { + this.relationIdentifier.write(out); + } + boolean isChildSchemaExists = null != this.childSchema; + out.writeBoolean(isChildSchemaExists); + if (isChildSchemaExists) { + this.childSchema.write(out); + } + if (properties == null) { + out.writeShort(0); + } else { + out.writeShort(properties.size()); + for (Map.Entry<String, String> entry : properties.entrySet()) { + out.writeUTF(entry.getKey()); + out.writeUTF(entry.getValue()); + } + } + } + + @Override public void readFields(DataInput in) throws IOException { + this.className = in.readUTF(); + boolean isRelationIdnentifierExists = in.readBoolean(); + if (isRelationIdnentifierExists) { + this.relationIdentifier = new RelationIdentifier(null, null, null); + this.relationIdentifier.readFields(in); + } + boolean isChildSchemaExists = in.readBoolean(); + if (isChildSchemaExists) { + this.childSchema = new TableSchema(); + this.childSchema.readFields(in); + } + + int mapSize = in.readShort(); + this.properties = new HashMap<>(mapSize); + for (int i = 0; i < mapSize; i++) { + String key = in.readUTF(); + String value = in.readUTF(); + this.properties.put(key, value); + } + + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java new file mode 100644 index 0000000..9a70b8b --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.metadata.schema.table; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.io.Serializable; + +/** + * class to maintain the relation between parent and child + */ +public class RelationIdentifier implements Serializable, Writable { + + private String databaseName; + + private String tableName; + + private String tableId; + + public RelationIdentifier(String databaseName, String tableName, String tableId) { + this.databaseName = databaseName; + this.tableName = tableName; + this.tableId = tableId; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getTableName() { + return tableName; + } + + public String getTableId() { + return tableId; + } + + @Override public void write(DataOutput out) throws IOException { + out.writeUTF(databaseName); + out.writeUTF(tableName); + out.writeUTF(tableId); + } + + @Override public void readFields(DataInput in) throws IOException { + this.databaseName = in.readUTF(); + this.tableName = in.readUTF(); + this.tableId = in.readUTF(); + } + + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + RelationIdentifier that = (RelationIdentifier) o; + + if (databaseName != null ? + !databaseName.equals(that.databaseName) : + that.databaseName != null) { + return false; + } + if (tableName != null ? !tableName.equals(that.tableName) : that.tableName != null) { + return false; + } + return tableId != null ? tableId.equals(that.tableId) : that.tableId == null; + } + + @Override public int hashCode() { + int result = databaseName != null ? databaseName.hashCode() : 0; + result = 31 * result + (tableName != null ? tableName.hashCode() : 0); + result = 31 * result + (tableId != null ? tableId.hashCode() : 0); + return result; + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java index d1a7e5b..3acd6d6 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java @@ -24,13 +24,15 @@ import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; import java.io.Serializable; -import java.util.Map; +import java.util.*; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; +import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation; /** * Store the information about the table. @@ -79,6 +81,14 @@ public class TableInfo implements Serializable, Writable { // this idenifier is a lazy field which will be created when it is used first time private AbsoluteTableIdentifier identifier; + private List<DataMapSchema> dataMapSchemaList; + + private List<RelationIdentifier> parentRelationIdentifiers; + + public TableInfo() { + dataMapSchemaList = new ArrayList<>(); + } + /** * @return the factTable */ @@ -91,6 +101,23 @@ public class TableInfo implements Serializable, Writable { */ public void setFactTable(TableSchema factTable) { this.factTable = factTable; + updateParentRelationIdentifier(); + } + + private void updateParentRelationIdentifier() { + Set<RelationIdentifier> parentRelationIdentifiers = new HashSet<>(); + this.parentRelationIdentifiers = new ArrayList<>(); + List<ColumnSchema> listOfColumns = this.factTable.getListOfColumns(); + for (ColumnSchema columnSchema : listOfColumns) { + List<ParentColumnTableRelation> parentColumnTableRelations = + columnSchema.getParentColumnTableRelations(); + if (null != parentColumnTableRelations) { + for (int i = 0; i < parentColumnTableRelations.size(); i++) { + parentRelationIdentifiers.add(parentColumnTableRelations.get(i).getRelationIdentifier()); + } + } + } + this.parentRelationIdentifiers.addAll(parentRelationIdentifiers); } /** @@ -157,6 +184,14 @@ public class TableInfo implements Serializable, Writable { this.storePath = storePath; } + public List<DataMapSchema> getDataMapSchemaList() { + return dataMapSchemaList; + } + + public void setDataMapSchemaList(List<DataMapSchema> dataMapSchemaList) { + this.dataMapSchemaList = dataMapSchemaList; + } + /** * to generate the hash code */ @@ -225,10 +260,27 @@ public class TableInfo implements Serializable, Writable { out.writeLong(lastUpdatedTime); out.writeUTF(metaDataFilepath); out.writeUTF(storePath); + boolean isChildSchemaExists = + null != dataMapSchemaList && dataMapSchemaList.size() > 0 ? true : false; + out.writeBoolean(isChildSchemaExists); + if (isChildSchemaExists) { + out.writeShort(dataMapSchemaList.size()); + for (int i = 0; i < dataMapSchemaList.size(); i++) { + dataMapSchemaList.get(i).write(out); + } + } + boolean isParentTableRelationIndentifierExists = + null != parentRelationIdentifiers && parentRelationIdentifiers.size() > 0; + out.writeBoolean(isParentTableRelationIndentifierExists); + if (isParentTableRelationIndentifierExists) { + out.writeShort(parentRelationIdentifiers.size()); + for (int i = 0; i < parentRelationIdentifiers.size(); i++) { + parentRelationIdentifiers.get(i).write(out); + } + } } - @Override - public void readFields(DataInput in) throws IOException { + @Override public void readFields(DataInput in) throws IOException { this.databaseName = in.readUTF(); this.tableUniqueName = in.readUTF(); this.factTable = new TableSchema(); @@ -236,6 +288,26 @@ public class TableInfo implements Serializable, Writable { this.lastUpdatedTime = in.readLong(); this.metaDataFilepath = in.readUTF(); this.storePath = in.readUTF(); + boolean isChildSchemaExists = in.readBoolean(); + this.dataMapSchemaList = new ArrayList<>(); + if (isChildSchemaExists) { + short numberOfChildTable = in.readShort(); + for (int i = 0; i < numberOfChildTable; i++) { + DataMapSchema childSchema = new DataMapSchema(null); + childSchema.readFields(in); + dataMapSchemaList.add(childSchema); + } + } + boolean isParentTableRelationIndentifierExists = in.readBoolean(); + if (isParentTableRelationIndentifierExists) { + short parentTableIndentifiersListSize = in.readShort(); + this.parentRelationIdentifiers = new ArrayList<>(); + for (int i = 0; i < parentTableIndentifiersListSize; i++) { + RelationIdentifier relationIdentifier = new RelationIdentifier(null, null, null); + relationIdentifier.readFields(in); + this.parentRelationIdentifiers.add(relationIdentifier); + } + } } public AbsoluteTableIdentifier getOrCreateAbsoluteTableIdentifier() { @@ -259,4 +331,9 @@ public class TableInfo implements Serializable, Writable { tableInfo.readFields(in); return tableInfo; } + + public List<RelationIdentifier> getParentRelationIdentifiers() { + return parentRelationIdentifiers; + } + } http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java index 7dc41a4..5b013d1 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java @@ -21,6 +21,7 @@ import java.io.DataOutput; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -250,4 +251,29 @@ public class TableSchema implements Serializable, Writable { } } + /** + * Below method will be used to build child schema object which will be stored in + * parent table + * + * @param className + * @param databaseName + * @param queryString + * @param queryType + * + * @return datamap schema + */ + public DataMapSchema buildChildSchema(String className, String databaseName, String queryString, + String queryType) { + RelationIdentifier relationIdentifier = + new RelationIdentifier(databaseName, tableName, tableId); + Map<String, String> properties = new HashMap<>(); + properties.put("CHILD_SELECT QUERY", queryString); + properties.put("QUERYTYPE", queryType); + DataMapSchema dataMapSchema = new DataMapSchema(className); + dataMapSchema.setChildSchema(this); + dataMapSchema.setProperties(properties); + dataMapSchema.setRelationIdentifier(relationIdentifier); + return dataMapSchema; + } + } http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java index cad3dd6..ea7005f 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java @@ -122,6 +122,13 @@ public class ColumnSchema implements Serializable, Writable { private boolean isSortColumn = false; /** + * aggregate function used in pre aggregate table + */ + private String aggFunction = ""; + + private List<ParentColumnTableRelation> parentColumnTableRelations; + + /** * @return the columnName */ public String getColumnName() { @@ -274,6 +281,15 @@ public class ColumnSchema implements Serializable, Writable { this.defaultValue = defaultValue; } + public List<ParentColumnTableRelation> getParentColumnTableRelations() { + return parentColumnTableRelations; + } + + public void setParentColumnTableRelations( + List<ParentColumnTableRelation> parentColumnTableRelations) { + this.parentColumnTableRelations = parentColumnTableRelations; + } + /** * hash code method to check get the hashcode based. * for generating the hash code only column name and column unique id will considered @@ -282,7 +298,7 @@ public class ColumnSchema implements Serializable, Writable { final int prime = 31; int result = 1; result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()) + - ((dataType == null) ? 0 : dataType.hashCode()); + ((dataType == null) ? 0 : dataType.hashCode()); return result; } @@ -415,6 +431,14 @@ public class ColumnSchema implements Serializable, Writable { isSortColumn = sortColumn; } + public String getAggFunction() { + return aggFunction; + } + + public void setAggFunction(String aggFunction) { + this.aggFunction = aggFunction; + } + @Override public void write(DataOutput out) throws IOException { out.writeShort(dataType.getId()); @@ -451,6 +475,16 @@ public class ColumnSchema implements Serializable, Writable { } out.writeBoolean(invisible); out.writeBoolean(isSortColumn); + out.writeUTF(null != aggFunction ? aggFunction : ""); + boolean isParentTableColumnRelationExists = + null != parentColumnTableRelations && parentColumnTableRelations.size() > 0; + out.writeBoolean(isParentTableColumnRelationExists); + if (isParentTableColumnRelationExists) { + out.writeShort(parentColumnTableRelations.size()); + for (int i = 0; i < parentColumnTableRelations.size(); i++) { + parentColumnTableRelations.get(i).write(out); + } + } } @Override @@ -486,5 +520,17 @@ public class ColumnSchema implements Serializable, Writable { } this.invisible = in.readBoolean(); this.isSortColumn = in.readBoolean(); + this.aggFunction = in.readUTF(); + boolean isParentTableColumnRelationExists = in.readBoolean(); + if (isParentTableColumnRelationExists) { + short parentColumnTableRelationSize = in.readShort(); + this.parentColumnTableRelations = new ArrayList<>(parentColumnTableRelationSize); + for (int i = 0; i < parentColumnTableRelationSize; i++) { + ParentColumnTableRelation parentColumnTableRelation = + new ParentColumnTableRelation(null, null, null); + parentColumnTableRelation.readFields(in); + parentColumnTableRelations.add(parentColumnTableRelation); + } + } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java index ea8bcb2..b9ec3f1 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java @@ -39,7 +39,9 @@ import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.datatype.DecimalType; import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; +import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTableRelation; import org.apache.carbondata.core.reader.CarbonIndexFileReader; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.format.BlockIndex; @@ -287,9 +289,35 @@ public abstract class AbstractDataFileFooterConverter { wrapperColumnSchema.setSortColumn(true); } } + wrapperColumnSchema.setAggFunction(externalColumnSchema.getAggregate_function()); + List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation = + externalColumnSchema.getParentColumnTableRelations(); + if (null != parentColumnTableRelation) { + wrapperColumnSchema.setParentColumnTableRelations( + fromThriftToWrapperParentTableColumnRelations(parentColumnTableRelation)); + } return wrapperColumnSchema; } + private List<ParentColumnTableRelation> fromThriftToWrapperParentTableColumnRelations( + List<org.apache.carbondata.format.ParentColumnTableRelation> thirftParentColumnRelation) { + List<ParentColumnTableRelation> parentColumnTableRelationList = new ArrayList<>(); + for (org.apache.carbondata.format.ParentColumnTableRelation carbonTableRelation : + thirftParentColumnRelation) { + RelationIdentifier relationIdentifier = + new RelationIdentifier(carbonTableRelation.getRelationIdentifier().getDatabaseName(), + carbonTableRelation.getRelationIdentifier().getTableName(), + carbonTableRelation.getRelationIdentifier().getTableId()); + ParentColumnTableRelation parentColumnTableRelation = + new ParentColumnTableRelation(relationIdentifier, carbonTableRelation.getColumnId(), + carbonTableRelation.getColumnName()); + parentColumnTableRelationList.add(parentColumnTableRelation); + } + return parentColumnTableRelationList; + } + + + /** * Below method is convert the thrift encoding to wrapper encoding * http://git-wip-us.apache.org/repos/asf/carbondata/blob/037f630a/format/src/main/thrift/schema.thrift ---------------------------------------------------------------------- diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift index 216d91f..4e9bb21 100644 --- a/format/src/main/thrift/schema.thrift +++ b/format/src/main/thrift/schema.thrift @@ -116,6 +116,12 @@ struct ColumnSchema{ * It will have column order which user has provided */ 16: optional i32 schemaOrdinal + + /** + * to maintain the column relation with parent table. + * will be usefull in case of pre-aggregate + **/ + 17: optional list<ParentColumnTableRelation> parentColumnTableRelations; } /** @@ -168,7 +174,32 @@ struct TableSchema{ 6: optional PartitionInfo partitionInfo; // Partition information } +struct RelationIdentifier { + 1: optional string databaseName; + 2: required string tableName; + 3: required string tableId; +} + +struct ParentColumnTableRelation { + 1: required RelationIdentifier relationIdentifier; + 2: required string columnId; + 3: required string columnName +} + +struct DataMapSchema { + // class name + 1: required string className; + // relation indentifier + 2: optional RelationIdentifier relationIdentifire; + // in case of preaggregate it will be used to maintain the child schema + // which will be usefull in case of query and data load + 3: optional TableSchema childTableSchema; + // to maintain properties like select query, query type like groupby, join + 4: optional map<string, string> properties; +} + struct TableInfo{ 1: required TableSchema fact_table; 2: required list<TableSchema> aggregate_table_list; + 3: optional list<DataMapSchema> dataMapSchemas; // childSchema information }