Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableGroupConverter.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableGroupConverter.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableGroupConverter.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableGroupConverter.java Fri Aug 22 21:36:47 2014 @@ -16,6 +16,7 @@ package org.apache.hadoop.hive.ql.io.par import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; @@ -36,19 +37,21 @@ public class DataWritableGroupConverter private final Object[] currentArr; private Writable[] rootMap; - public DataWritableGroupConverter(final GroupType requestedSchema, final GroupType tableSchema) { - this(requestedSchema, null, 0, tableSchema); + public DataWritableGroupConverter(final GroupType requestedSchema, final GroupType tableSchema, + final List<TypeInfo> hiveSchemaTypeInfos) { + this(requestedSchema, null, 0, tableSchema, hiveSchemaTypeInfos); final int fieldCount = tableSchema.getFieldCount(); this.rootMap = new Writable[fieldCount]; } public DataWritableGroupConverter(final GroupType groupType, final HiveGroupConverter parent, - final int index) { - this(groupType, parent, index, groupType); + final int index, final List<TypeInfo> hiveSchemaTypeInfos) { + this(groupType, parent, index, groupType, hiveSchemaTypeInfos); } public DataWritableGroupConverter(final GroupType selectedGroupType, - final HiveGroupConverter parent, final int index, final GroupType containingGroupType) { + final HiveGroupConverter parent, final int index, final GroupType containingGroupType, + final List<TypeInfo> hiveSchemaTypeInfos) { this.parent = parent; this.index = index; final int totalFieldCount = containingGroupType.getFieldCount(); @@ -62,7 +65,8 @@ public class DataWritableGroupConverter Type subtype = selectedFields.get(i); if (containingGroupType.getFields().contains(subtype)) { converters[i] = getConverterFromDescription(subtype, - containingGroupType.getFieldIndex(subtype.getName()), this); + containingGroupType.getFieldIndex(subtype.getName()), this, + hiveSchemaTypeInfos); } else { throw new IllegalStateException("Group type [" + containingGroupType + "] does not contain requested field: " + subtype);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java Fri Aug 22 21:36:47 2014 @@ -13,6 +13,9 @@ */ package org.apache.hadoop.hive.ql.io.parquet.convert; +import java.util.List; + +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.ArrayWritable; import parquet.io.api.GroupConverter; @@ -28,8 +31,10 @@ public class DataWritableRecordConverter private final DataWritableGroupConverter root; - public DataWritableRecordConverter(final GroupType requestedSchema, final GroupType tableSchema) { - this.root = new DataWritableGroupConverter(requestedSchema, tableSchema); + public DataWritableRecordConverter(final GroupType requestedSchema, final GroupType tableSchema, + final List<TypeInfo> hiveColumnTypeInfos) { + this.root = new DataWritableGroupConverter(requestedSchema, tableSchema, + hiveColumnTypeInfos); } @Override @@ -41,4 +46,4 @@ public class DataWritableRecordConverter public GroupConverter getRootConverter() { return root; } -} +} \ No newline at end of file Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java Fri Aug 22 21:36:47 2014 @@ -16,12 +16,19 @@ package org.apache.hadoop.hive.ql.io.par import java.math.BigDecimal; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -145,6 +152,32 @@ public enum ETypeConverter { } }; } + }, + ECHAR_CONVERTER(HiveCharWritable.class) { + @Override + Converter getConverter(final PrimitiveType type, final int index, final HiveGroupConverter parent) { + return new BinaryConverter<HiveCharWritable>(type, parent, index) { + @Override + protected HiveCharWritable convert(Binary binary) { + HiveChar hiveChar = new HiveChar(); + hiveChar.setValue(binary.toStringUsingUTF8()); + return new HiveCharWritable(hiveChar); + } + }; + } + }, + EVARCHAR_CONVERTER(HiveVarcharWritable.class) { + @Override + Converter getConverter(final PrimitiveType type, final int index, final HiveGroupConverter parent) { + return new BinaryConverter<HiveVarcharWritable>(type, parent, index) { + @Override + protected HiveVarcharWritable convert(Binary binary) { + HiveVarchar hiveVarchar = new HiveVarchar(); + hiveVarchar.setValue(binary.toStringUsingUTF8()); + return new HiveVarcharWritable(hiveVarchar); + } + }; + } }; final Class<?> _type; @@ -159,7 +192,8 @@ public enum ETypeConverter { abstract Converter getConverter(final PrimitiveType type, final int index, final HiveGroupConverter parent); - public static Converter getNewConverter(final PrimitiveType type, final int index, final HiveGroupConverter parent) { + public static Converter getNewConverter(final PrimitiveType type, final int index, + final HiveGroupConverter parent, List<TypeInfo> hiveSchemaTypeInfos) { if (type.isPrimitive() && (type.asPrimitiveType().getPrimitiveTypeName().equals(PrimitiveType.PrimitiveTypeName.INT96))) { //TODO- cleanup once parquet support Timestamp type annotation. return ETypeConverter.ETIMESTAMP_CONVERTER.getConverter(type, index, parent); @@ -167,7 +201,15 @@ public enum ETypeConverter { if (OriginalType.DECIMAL == type.getOriginalType()) { return EDECIMAL_CONVERTER.getConverter(type, index, parent); } else if (OriginalType.UTF8 == type.getOriginalType()) { - return ESTRING_CONVERTER.getConverter(type, index, parent); + if (hiveSchemaTypeInfos.get(index).getTypeName() + .startsWith(serdeConstants.CHAR_TYPE_NAME)) { + return ECHAR_CONVERTER.getConverter(type, index, parent); + } else if (hiveSchemaTypeInfos.get(index).getTypeName() + .startsWith(serdeConstants.VARCHAR_TYPE_NAME)) { + return EVARCHAR_CONVERTER.getConverter(type, index, parent); + } else if (type.isPrimitive()) { + return ESTRING_CONVERTER.getConverter(type, index, parent); + } } Class<?> javaType = type.getPrimitiveTypeName().javaType; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java Fri Aug 22 21:36:47 2014 @@ -13,6 +13,9 @@ */ package org.apache.hadoop.hive.ql.io.parquet.convert; +import java.util.List; + +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.Writable; import parquet.io.api.Converter; @@ -23,17 +26,20 @@ import parquet.schema.Type.Repetition; public abstract class HiveGroupConverter extends GroupConverter { protected static Converter getConverterFromDescription(final Type type, final int index, - final HiveGroupConverter parent) { + final HiveGroupConverter parent, List<TypeInfo> hiveSchemaTypeInfos) { if (type == null) { return null; } if (type.isPrimitive()) { - return ETypeConverter.getNewConverter(type.asPrimitiveType(), index, parent); + return ETypeConverter.getNewConverter(type.asPrimitiveType(), index, parent, + hiveSchemaTypeInfos); } else { if (type.asGroupType().getRepetition() == Repetition.REPEATED) { - return new ArrayWritableGroupConverter(type.asGroupType(), parent, index); + return new ArrayWritableGroupConverter(type.asGroupType(), parent, index, + hiveSchemaTypeInfos); } else { - return new DataWritableGroupConverter(type.asGroupType(), parent, index); + return new DataWritableGroupConverter(type.asGroupType(), parent, index, + hiveSchemaTypeInfos); } } } @@ -42,4 +48,4 @@ public abstract class HiveGroupConverter protected abstract void add(int index, Writable value); -} +} \ No newline at end of file Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java Fri Aug 22 21:36:47 2014 @@ -16,6 +16,7 @@ package org.apache.hadoop.hive.ql.io.par import java.util.List; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; @@ -25,7 +26,6 @@ import org.apache.hadoop.hive.serde2.typ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import parquet.schema.ConversionPatterns; -import parquet.schema.DecimalMetadata; import parquet.schema.GroupType; import parquet.schema.MessageType; import parquet.schema.OriginalType; @@ -81,6 +81,14 @@ public class HiveSchemaConverter { return new PrimitiveType(repetition, PrimitiveTypeName.INT96, name); } else if (typeInfo.equals(TypeInfoFactory.voidTypeInfo)) { throw new UnsupportedOperationException("Void type not implemented"); + } else if (typeInfo.getTypeName().toLowerCase().startsWith( + serdeConstants.CHAR_TYPE_NAME)) { + return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8) + .named(name); + } else if (typeInfo.getTypeName().toLowerCase().startsWith( + serdeConstants.VARCHAR_TYPE_NAME)) { + return Types.optional(PrimitiveTypeName.BINARY).as(OriginalType.UTF8) + .named(name); } else if (typeInfo instanceof DecimalTypeInfo) { DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; int prec = decimalTypeInfo.precision(); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java Fri Aug 22 21:36:47 2014 @@ -14,6 +14,7 @@ package org.apache.hadoop.hive.ql.io.parquet.read; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -23,6 +24,8 @@ import org.apache.hadoop.hive.ql.io.IOCo import org.apache.hadoop.hive.ql.io.parquet.convert.DataWritableRecordConverter; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.util.StringUtils; @@ -60,6 +63,28 @@ public class DataWritableReadSupport ext return (List<String>) VirtualColumn. removeVirtualColumns(StringUtils.getStringCollection(columns)); } + + private static List<TypeInfo> getColumnTypes(Configuration configuration) { + + List<String> columnNames; + String columnNamesProperty = configuration.get(IOConstants.COLUMNS); + if (columnNamesProperty.length() == 0) { + columnNames = new ArrayList<String>(); + } else { + columnNames = Arrays.asList(columnNamesProperty.split(",")); + } + List<TypeInfo> columnTypes; + String columnTypesProperty = configuration.get(IOConstants.COLUMNS_TYPES); + if (columnTypesProperty.length() == 0) { + columnTypes = new ArrayList<TypeInfo>(); + } else { + columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypesProperty); + } + + columnTypes = VirtualColumn.removeVirtualColumnTypes(columnNames, columnTypes); + return columnTypes; + } + /** * * It creates the readContext for Parquet side with the requested schema during the init phase. @@ -100,20 +125,22 @@ public class DataWritableReadSupport ext final List<Type> typeListWanted = new ArrayList<Type>(); final boolean indexAccess = configuration.getBoolean(PARQUET_COLUMN_INDEX_ACCESS, false); for (final Integer idx : indexColumnsWanted) { - String col = listColumns.get(idx); - if (indexAccess) { - typeListWanted.add(tableSchema.getType(col)); - } else { - col = col.toLowerCase(); - if (lowerCaseFileSchemaColumns.containsKey(col)) { - typeListWanted.add(tableSchema.getType(lowerCaseFileSchemaColumns.get(col))); + if (idx < listColumns.size()) { + String col = listColumns.get(idx); + if (indexAccess) { + typeListWanted.add(tableSchema.getType(col)); } else { - // should never occur? - String msg = "Column " + col + " at index " + idx + " does not exist in " + + col = col.toLowerCase(); + if (lowerCaseFileSchemaColumns.containsKey(col)) { + typeListWanted.add(tableSchema.getType(lowerCaseFileSchemaColumns.get(col))); + } else { + // should never occur? + String msg = "Column " + col + " at index " + idx + " does not exist in " + lowerCaseFileSchemaColumns; - throw new IllegalStateException(msg); + throw new IllegalStateException(msg); + } } - } + } } requestedSchemaByUser = resolveSchemaAccess(new MessageType(fileSchema.getName(), typeListWanted), fileSchema, configuration); @@ -146,7 +173,8 @@ public class DataWritableReadSupport ext } final MessageType tableSchema = resolveSchemaAccess(MessageTypeParser. parseMessageType(metadata.get(HIVE_SCHEMA_KEY)), fileSchema, configuration); - return new DataWritableRecordConverter(readContext.getRequestedSchema(), tableSchema); + return new DataWritableRecordConverter(readContext.getRequestedSchema(), tableSchema, + getColumnTypes(configuration)); } /** Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java Fri Aug 22 21:36:47 2014 @@ -25,12 +25,14 @@ import org.apache.hadoop.hive.serde2.obj import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.ArrayWritable; /** @@ -102,12 +104,10 @@ public class ArrayWritableObjectInspecto return PrimitiveObjectInspectorFactory.writableTimestampObjectInspector; } else if (typeInfo.equals(TypeInfoFactory.dateTypeInfo)) { throw new UnsupportedOperationException("Parquet does not support date. See HIVE-6384"); - } else if (typeInfo.getTypeName().toLowerCase().startsWith(serdeConstants.DECIMAL_TYPE_NAME)) { - throw new UnsupportedOperationException("Parquet does not support decimal. See HIVE-6384"); } else if (typeInfo.getTypeName().toLowerCase().startsWith(serdeConstants.CHAR_TYPE_NAME)) { - throw new UnsupportedOperationException("Parquet does not support char. See HIVE-6384"); + return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector((CharTypeInfo) typeInfo); } else if (typeInfo.getTypeName().toLowerCase().startsWith(serdeConstants.VARCHAR_TYPE_NAME)) { - throw new UnsupportedOperationException("Parquet does not support varchar. See HIVE-6384"); + return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector((VarcharTypeInfo) typeInfo); } else { throw new UnsupportedOperationException("Unknown field type: " + typeInfo); } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java Fri Aug 22 21:36:47 2014 @@ -42,6 +42,8 @@ import org.apache.hadoop.hive.serde2.obj import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; @@ -60,6 +62,7 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +import parquet.io.api.Binary; /** * @@ -280,6 +283,12 @@ public class ParquetHiveSerDe extends Ab return new BytesWritable(tgt); case TIMESTAMP: return new TimestampWritable(((TimestampObjectInspector) inspector).getPrimitiveJavaObject(obj)); + case CHAR: + String strippedValue = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(obj).getStrippedValue(); + return new BytesWritable(Binary.fromString(strippedValue).getBytes()); + case VARCHAR: + String value = ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(obj).getValue(); + return new BytesWritable(Binary.fromString(value).getBytes()); default: throw new SerDeException("Unknown primitive : " + inspector.getPrimitiveCategory()); } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java Fri Aug 22 21:36:47 2014 @@ -18,14 +18,6 @@ package org.apache.hadoop.hive.ql.io.sarg; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; @@ -57,6 +49,15 @@ import org.apache.hadoop.hive.serde2.obj import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import java.math.BigDecimal; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * The implementation of SearchArguments. */ @@ -947,7 +948,8 @@ final class SearchArgumentImpl implement literal instanceof Long || literal instanceof Double || literal instanceof DateWritable || - literal instanceof HiveDecimal) { + literal instanceof HiveDecimal || + literal instanceof BigDecimal) { return literal; } else if (literal instanceof HiveChar || literal instanceof HiveVarchar) { @@ -979,7 +981,8 @@ final class SearchArgumentImpl implement return PredicateLeaf.Type.FLOAT; } else if (literal instanceof DateWritable) { return PredicateLeaf.Type.DATE; - } else if (literal instanceof HiveDecimal) { + } else if (literal instanceof HiveDecimal || + literal instanceof BigDecimal) { return PredicateLeaf.Type.DECIMAL; } throw new IllegalArgumentException("Unknown type for literal " + literal); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java Fri Aug 22 21:36:47 2014 @@ -165,13 +165,13 @@ public class DbTxnManager extends HiveTx break; case TABLE: + case DUMMYPARTITION: // in case of dynamic partitioning lock the table t = output.getTable(); compBuilder.setDbName(t.getDbName()); compBuilder.setTableName(t.getTableName()); break; case PARTITION: - case DUMMYPARTITION: compBuilder.setPartitionName(output.getPartition().getName()); t = output.getPartition().getTable(); compBuilder.setDbName(t.getDbName()); @@ -301,7 +301,10 @@ public class DbTxnManager extends HiveTx try { if (txnId > 0) rollbackTxn(); if (lockMgr != null) lockMgr.close(); + if (client != null) client.close(); } catch (Exception e) { + LOG.error("Caught exception " + e.getClass().getName() + " with message <" + e.getMessage() + + ">, swallowing as there is nothing we can do with it."); // Not much we can do about it here. } } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Aug 22 21:36:47 2014 @@ -89,6 +89,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -979,7 +980,7 @@ public class Hive { tTable = getMSC().getTable(dbName, tableName); } catch (NoSuchObjectException e) { if (throwException) { - LOG.error(StringUtils.stringifyException(e)); + LOG.error("Table " + tableName + " not found: " + e.getMessage()); throw new InvalidTableException(tableName); } return null; @@ -2553,6 +2554,15 @@ private void constructOneLBLocationMap(F throw new HiveException(e); } } + + public boolean setPartitionColumnStatistics(SetPartitionsStatsRequest request) throws HiveException { + try { + return getMSC().setPartitionColumnStatistics(request); + } catch (Exception e) { + LOG.debug(StringUtils.stringifyException(e)); + throw new HiveException(e); + } + } public List<ColumnStatisticsObj> getTableColumnStatistics( String dbName, String tableName, List<String> colNames) throws HiveException { @@ -2716,4 +2726,19 @@ private void constructOneLBLocationMap(F } } + public void setMetaConf(String propName, String propValue) throws HiveException { + try { + getMSC().setMetaConf(propName, propValue); + } catch (TException te) { + throw new HiveException(te); + } + } + + public String getMetaConf(String propName) throws HiveException { + try { + return getMSC().getMetaConf(propName); + } catch (TException te) { + throw new HiveException(te); + } + } }; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java Fri Aug 22 21:36:47 2014 @@ -249,6 +249,8 @@ public class SessionHiveMetaStoreClient + " is not a directory or unable to create one"); } } + // Make sure location string is in proper format + tbl.getSd().setLocation(tblPath.toString()); } // Add temp table info to current session Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java Fri Aug 22 21:36:47 2014 @@ -22,25 +22,36 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.ListIterator; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +@InterfaceAudience.Private public class VirtualColumn implements Serializable { private static final long serialVersionUID = 1L; - public static VirtualColumn FILENAME = new VirtualColumn("INPUT__FILE__NAME", (PrimitiveTypeInfo)TypeInfoFactory.stringTypeInfo); - public static VirtualColumn BLOCKOFFSET = new VirtualColumn("BLOCK__OFFSET__INSIDE__FILE", (PrimitiveTypeInfo)TypeInfoFactory.longTypeInfo); - public static VirtualColumn ROWOFFSET = new VirtualColumn("ROW__OFFSET__INSIDE__BLOCK", (PrimitiveTypeInfo)TypeInfoFactory.longTypeInfo); + public static final VirtualColumn FILENAME = new VirtualColumn("INPUT__FILE__NAME", (PrimitiveTypeInfo)TypeInfoFactory.stringTypeInfo); + public static final VirtualColumn BLOCKOFFSET = new VirtualColumn("BLOCK__OFFSET__INSIDE__FILE", (PrimitiveTypeInfo)TypeInfoFactory.longTypeInfo); + public static final VirtualColumn ROWOFFSET = new VirtualColumn("ROW__OFFSET__INSIDE__BLOCK", (PrimitiveTypeInfo)TypeInfoFactory.longTypeInfo); - public static VirtualColumn RAWDATASIZE = new VirtualColumn("RAW__DATA__SIZE", (PrimitiveTypeInfo)TypeInfoFactory.longTypeInfo); + public static final VirtualColumn RAWDATASIZE = new VirtualColumn("RAW__DATA__SIZE", (PrimitiveTypeInfo)TypeInfoFactory.longTypeInfo); + /** + * {@link org.apache.hadoop.hive.ql.io.RecordIdentifier} + */ + public static final VirtualColumn ROWID = new VirtualColumn("ROW__ID", RecordIdentifier.StructInfo.typeInfo, true, RecordIdentifier.StructInfo.oi); /** * GROUPINGID is used with GROUP BY GROUPINGS SETS, ROLLUP and CUBE. @@ -49,27 +60,28 @@ public class VirtualColumn implements Se * set if that column has been aggregated in that row. Otherwise the * value is "0". Returns the decimal representation of the bit vector. */ - public static VirtualColumn GROUPINGID = + public static final VirtualColumn GROUPINGID = new VirtualColumn("GROUPING__ID", (PrimitiveTypeInfo) TypeInfoFactory.intTypeInfo); - public static VirtualColumn[] VIRTUAL_COLUMNS = - new VirtualColumn[] {FILENAME, BLOCKOFFSET, ROWOFFSET, RAWDATASIZE, GROUPINGID}; - - private String name; - private PrimitiveTypeInfo typeInfo; - private boolean isHidden = true; + public static ImmutableSet<String> VIRTUAL_COLUMN_NAMES = + ImmutableSet.of(FILENAME.getName(), BLOCKOFFSET.getName(), ROWOFFSET.getName(), + RAWDATASIZE.getName(), GROUPINGID.getName(), ROWID.getName()); - public VirtualColumn() { - } + private final String name; + private final TypeInfo typeInfo; + private final boolean isHidden; + private final ObjectInspector oi; - public VirtualColumn(String name, PrimitiveTypeInfo typeInfo) { - this(name, typeInfo, true); + private VirtualColumn(String name, PrimitiveTypeInfo typeInfo) { + this(name, typeInfo, true, + PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(typeInfo)); } - VirtualColumn(String name, PrimitiveTypeInfo typeInfo, boolean isHidden) { + private VirtualColumn(String name, TypeInfo typeInfo, boolean isHidden, ObjectInspector oi) { this.name = name; this.typeInfo = typeInfo; this.isHidden = isHidden; + this.oi = oi; } public static List<VirtualColumn> getStatsRegistry(Configuration conf) { @@ -87,26 +99,19 @@ public class VirtualColumn implements Se if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEROWOFFSET)) { l.add(ROWOFFSET); } + l.add(ROWID); return l; } - public PrimitiveTypeInfo getTypeInfo() { + public TypeInfo getTypeInfo() { return typeInfo; } - public void setTypeInfo(PrimitiveTypeInfo typeInfo) { - this.typeInfo = typeInfo; - } - public String getName() { return this.name; } - public void setName(String name) { - this.name = name; - } - public boolean isHidden() { return isHidden; } @@ -115,37 +120,58 @@ public class VirtualColumn implements Se return isHidden; } - public void setIsHidden(boolean isHidden) { - this.isHidden = isHidden; + public ObjectInspector getObjectInspector() { + return oi; } @Override public boolean equals(Object o) { - if (o == null) { - return false; - } if (this == o) { return true; } + if(!(o instanceof VirtualColumn)) { + return false; + } VirtualColumn c = (VirtualColumn) o; return this.name.equals(c.name) && this.typeInfo.getTypeName().equals(c.getTypeInfo().getTypeName()); } - + @Override + public int hashCode() { + int c = 19; + c = 31 * name.hashCode() + c; + return 31 * typeInfo.getTypeName().hashCode() + c; + } public static Collection<String> removeVirtualColumns(final Collection<String> columns) { - for(VirtualColumn vcol : VIRTUAL_COLUMNS) { - columns.remove(vcol.getName()); - } + Iterables.removeAll(columns, VIRTUAL_COLUMN_NAMES); return columns; } + public static List<TypeInfo> removeVirtualColumnTypes(final List<String> columnNames, + final List<TypeInfo> columnTypes) { + if (columnNames.size() != columnTypes.size()) { + throw new IllegalArgumentException("Number of column names in configuration " + + columnNames.size() + " differs from column types " + columnTypes.size()); + } + + int i = 0; + ListIterator<TypeInfo> it = columnTypes.listIterator(); + while(it.hasNext()) { + it.next(); + if (VIRTUAL_COLUMN_NAMES.contains(columnNames.get(i))) { + it.remove(); + } + ++i; + } + return columnTypes; + } + public static StructObjectInspector getVCSObjectInspector(List<VirtualColumn> vcs) { List<String> names = new ArrayList<String>(vcs.size()); List<ObjectInspector> inspectors = new ArrayList<ObjectInspector>(vcs.size()); for (VirtualColumn vc : vcs) { names.add(vc.getName()); - inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - vc.getTypeInfo())); + inspectors.add(vc.oi); } return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Fri Aug 22 21:36:47 2014 @@ -927,11 +927,9 @@ public class Vectorizer implements Physi if (desc instanceof ExprNodeColumnDesc) { ExprNodeColumnDesc c = (ExprNodeColumnDesc) desc; // Currently, we do not support vectorized virtual columns (see HIVE-5570). - for (VirtualColumn vc : VirtualColumn.VIRTUAL_COLUMNS) { - if (c.getColumn().equals(vc.getName())) { - LOG.info("Cannot vectorize virtual column " + c.getColumn()); - return false; - } + if (VirtualColumn.VIRTUAL_COLUMN_NAMES.contains(c.getColumn())) { + LOG.info("Cannot vectorize virtual column " + c.getColumn()); + return false; } } String typeName = desc.getTypeInfo().getTypeName(); @@ -1076,10 +1074,8 @@ public class Vectorizer implements Physi // Not using method column.getIsVirtualCol() because partitioning columns are also // treated as virtual columns in ColumnInfo. - for (VirtualColumn vc : VirtualColumn.VIRTUAL_COLUMNS) { - if (column.getInternalName().equals(vc.getName())) { + if (VirtualColumn.VIRTUAL_COLUMN_NAMES.contains(column.getInternalName())) { return true; - } } return false; } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java Fri Aug 22 21:36:47 2014 @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.optimi import com.google.common.collect.Lists; import com.google.common.collect.Maps; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; @@ -67,8 +68,10 @@ import org.apache.hadoop.hive.ql.udf.gen import org.apache.hadoop.hive.serde.serdeConstants; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Stack; public class StatsRulesProcFactory { @@ -803,12 +806,13 @@ public class StatsRulesProcFactory { // statistics object that is combination of statistics from all // relations involved in JOIN Statistics stats = new Statistics(); - List<Long> rowCountParents = Lists.newArrayList(); + Map<String, Long> rowCountParents = new HashMap<String, Long>(); List<Long> distinctVals = Lists.newArrayList(); // 2 relations, multiple attributes boolean multiAttr = false; int numAttr = 1; + int numParent = parents.size(); Map<String, ColStatistics> joinedColStats = Maps.newHashMap(); Map<Integer, List<String>> joinKeys = Maps.newHashMap(); @@ -818,9 +822,20 @@ public class StatsRulesProcFactory { ReduceSinkOperator parent = (ReduceSinkOperator) jop.getParentOperators().get(pos); Statistics parentStats = parent.getStatistics(); - rowCountParents.add(parentStats.getNumRows()); List<ExprNodeDesc> keyExprs = parent.getConf().getKeyCols(); + // Parent RS may have column statistics from multiple parents. + // Populate table alias to row count map, this will be used later to + // scale down/up column statistics based on new row count + // NOTE: JOIN with UNION as parent of RS will not have table alias + // propagated properly. UNION operator does not propagate the table + // alias of subqueries properly to expression nodes. Hence union20.q + // will have wrong number of rows. + Set<String> tableAliases = StatsUtils.getAllTableAlias(parent.getColumnExprMap()); + for (String tabAlias : tableAliases) { + rowCountParents.put(tabAlias, parentStats.getNumRows()); + } + // multi-attribute join key if (keyExprs.size() > 1) { multiAttr = true; @@ -860,12 +875,19 @@ public class StatsRulesProcFactory { perAttrDVs.add(cs.getCountDistint()); } } + distinctVals.add(getDenominator(perAttrDVs)); perAttrDVs.clear(); } - for (Long l : distinctVals) { - denom *= l; + if (numAttr > numParent) { + // To avoid denominator getting larger and aggressively reducing + // number of rows, we will ease out denominator. + denom = getEasedOutDenominator(distinctVals); + } else { + for (Long l : distinctVals) { + denom *= l; + } } } else { for (List<String> jkeys : joinKeys.values()) { @@ -890,6 +912,7 @@ public class StatsRulesProcFactory { Map<String, ExprNodeDesc> colExprMap = jop.getColumnExprMap(); RowSchema rs = jop.getSchema(); List<ColStatistics> outColStats = Lists.newArrayList(); + Map<String, String> outInTabAlias = new HashMap<String, String>(); for (ColumnInfo ci : rs.getSignature()) { String key = ci.getInternalName(); ExprNodeDesc end = colExprMap.get(key); @@ -901,6 +924,7 @@ public class StatsRulesProcFactory { ColStatistics cs = joinedColStats.get(fqColName); String outColName = key; String outTabAlias = ci.getTabAlias(); + outInTabAlias.put(outTabAlias, tabAlias); if (cs != null) { cs.setColumnName(outColName); cs.setTableAlias(outTabAlias); @@ -911,7 +935,8 @@ public class StatsRulesProcFactory { // update join statistics stats.setColumnStats(outColStats); - long newRowCount = computeNewRowCount(rowCountParents, denom); + long newRowCount = computeNewRowCount( + Lists.newArrayList(rowCountParents.values()), denom); if (newRowCount <= 0 && LOG.isDebugEnabled()) { newRowCount = 0; @@ -920,7 +945,8 @@ public class StatsRulesProcFactory { + " #Rows of parents: " + rowCountParents.toString() + ". Denominator: " + denom); } - updateStatsForJoinType(stats, newRowCount, true, jop.getConf()); + updateStatsForJoinType(stats, newRowCount, jop.getConf(), + rowCountParents, outInTabAlias); jop.setStatistics(stats); if (LOG.isDebugEnabled()) { @@ -966,37 +992,54 @@ public class StatsRulesProcFactory { return null; } + private Long getEasedOutDenominator(List<Long> distinctVals) { + // Exponential back-off for NDVs. + // 1) Descending order sort of NDVs + // 2) denominator = NDV1 * (NDV2 ^ (1/2)) * (NDV3 ^ (1/4))) * .... + Collections.sort(distinctVals, Collections.reverseOrder()); + + long denom = distinctVals.get(0); + for (int i = 1; i < distinctVals.size(); i++) { + denom = (long) (denom * Math.pow(distinctVals.get(i), 1.0 / (1 << i))); + } + + return denom; + } + private void updateStatsForJoinType(Statistics stats, long newNumRows, - boolean useColStats, JoinDesc conf) { - long oldRowCount = stats.getNumRows(); - double ratio = (double) newNumRows / (double) oldRowCount; + JoinDesc conf, Map<String, Long> rowCountParents, + Map<String, String> outInTabAlias) { stats.setNumRows(newNumRows); - if (useColStats) { - List<ColStatistics> colStats = stats.getColumnStats(); - for (ColStatistics cs : colStats) { - long oldDV = cs.getCountDistint(); - long newDV = oldDV; - - // if ratio is greater than 1, then number of rows increases. This can happen - // when some operators like GROUPBY duplicates the input rows in which case - // number of distincts should not change. Update the distinct count only when - // the output number of rows is less than input number of rows. - if (ratio <= 1.0) { - newDV = (long) Math.ceil(ratio * oldDV); - } - // Assumes inner join - // TODO: HIVE-5579 will handle different join types - cs.setNumNulls(0); - cs.setCountDistint(newDV); - } - stats.setColumnStats(colStats); - long newDataSize = StatsUtils.getDataSizeFromColumnStats(newNumRows, colStats); - stats.setDataSize(newDataSize); - } else { - long newDataSize = (long) (ratio * stats.getDataSize()); - stats.setDataSize(newDataSize); + // scale down/up the column statistics based on the changes in number of + // rows from each parent. For ex: If there are 2 parents for JOIN operator + // with 1st parent having 200 rows and 2nd parent having 2000 rows. Now if + // the new number of rows after applying join rule is 10, then the column + // stats for columns from 1st parent should be scaled down by 200/10 = 20x + // and stats for columns from 2nd parent should be scaled down by 200x + List<ColStatistics> colStats = stats.getColumnStats(); + for (ColStatistics cs : colStats) { + long oldRowCount = rowCountParents.get(outInTabAlias.get(cs.getTableAlias())); + double ratio = (double) newNumRows / (double) oldRowCount; + long oldDV = cs.getCountDistint(); + long newDV = oldDV; + + // if ratio is greater than 1, then number of rows increases. This can happen + // when some operators like GROUPBY duplicates the input rows in which case + // number of distincts should not change. Update the distinct count only when + // the output number of rows is less than input number of rows. + if (ratio <= 1.0) { + newDV = (long) Math.ceil(ratio * oldDV); + } + // Assumes inner join + // TODO: HIVE-5579 will handle different join types + cs.setNumNulls(0); + cs.setCountDistint(newDV); } + stats.setColumnStats(colStats); + long newDataSize = StatsUtils + .getDataSizeFromColumnStats(newNumRows, colStats); + stats.setDataSize(newDataSize); } private long computeNewRowCount(List<Long> rowCountParents, long denom) { Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Fri Aug 22 21:36:47 2014 @@ -25,6 +25,7 @@ import java.io.Serializable; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -57,6 +58,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ArchiveUtils; +import org.apache.hadoop.hive.ql.exec.ColumnStatsUpdateTask; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Task; @@ -91,6 +93,8 @@ import org.apache.hadoop.hive.ql.plan.Al import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.AlterTableExchangePartition; import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc; +import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; +import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc; import org.apache.hadoop.hive.ql.plan.CreateIndexDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -276,6 +280,8 @@ public class DDLSemanticAnalyzer extends analyzeAlterTableClusterSort(ast, tableName, partSpec); } else if (ast.getToken().getType() == HiveParser.TOK_COMPACT) { analyzeAlterTableCompact(ast, tableName, partSpec); + } else if(ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS){ + analyzeAlterTableUpdateStats(ast,tblPart); } break; } @@ -378,6 +384,9 @@ public class DDLSemanticAnalyzer extends case HiveParser.TOK_ALTERTABLE_RENAME: analyzeAlterTableRename(ast, false); break; + case HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS: + analyzeAlterTableUpdateStats(ast, null); + break; case HiveParser.TOK_ALTERTABLE_TOUCH: analyzeAlterTableTouch(ast); break; @@ -507,6 +516,57 @@ public class DDLSemanticAnalyzer extends } } + private void analyzeAlterTableUpdateStats(ASTNode ast, TablePartition tblPart) + throws SemanticException { + String tblName = null; + String colName = null; + Map<String, String> mapProp = null; + Map<String, String> partSpec = null; + String partName = null; + if (tblPart == null) { + tblName = getUnescapedName((ASTNode) ast.getChild(0)); + colName = getUnescapedName((ASTNode) ast.getChild(1)); + mapProp = getProps((ASTNode) (ast.getChild(2)).getChild(0)); + } else { + tblName = tblPart.tableName; + partSpec = tblPart.partSpec; + try { + partName = Warehouse.makePartName(partSpec, false); + } catch (MetaException e) { + // TODO Auto-generated catch block + throw new SemanticException("partition " + partSpec.toString() + + " not found"); + } + colName = getUnescapedName((ASTNode) ast.getChild(0)); + mapProp = getProps((ASTNode) (ast.getChild(1)).getChild(0)); + } + + Table tbl = null; + try { + tbl = db.getTable(tblName); + } catch (HiveException e) { + throw new SemanticException("table " + tbl + " not found"); + } + + String colType = null; + List<FieldSchema> cols = tbl.getCols(); + for (FieldSchema col : cols) { + if (colName.equalsIgnoreCase(col.getName())) { + colType = col.getType(); + break; + } + } + + if (colType == null) + throw new SemanticException("column type not found"); + + ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tbl.getTableName(), + Arrays.asList(colName), Arrays.asList(colType), partSpec == null); + ColumnStatsUpdateTask cStatsUpdateTask = (ColumnStatsUpdateTask) TaskFactory + .get(new ColumnStatsUpdateWork(cStatsDesc, partName, mapProp), conf); + rootTasks.add(cStatsUpdateTask); + } + private void analyzeSetShowRole(ASTNode ast) throws SemanticException { switch (ast.getChildCount()) { case 0: @@ -745,6 +805,8 @@ public class DDLSemanticAnalyzer extends if (dbProps != null) { createDatabaseDesc.setDatabaseProperties(dbProps); } + Database database = new Database(dbName, dbComment, dbLocation, dbProps); + outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_NO_LOCK)); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), createDatabaseDesc), conf)); @@ -795,8 +857,12 @@ public class DDLSemanticAnalyzer extends rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropDatabaseDesc), conf)); } - private void analyzeSwitchDatabase(ASTNode ast) { + private void analyzeSwitchDatabase(ASTNode ast) throws SemanticException { String dbName = unescapeIdentifier(ast.getChild(0).getText()); + Database database = getDatabase(dbName, true); + ReadEntity dbReadEntity = new ReadEntity(database); + dbReadEntity.noLockNeeded(); + inputs.add(dbReadEntity); SwitchDatabaseDesc switchDatabaseDesc = new SwitchDatabaseDesc(dbName); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), switchDatabaseDesc), conf)); @@ -1015,7 +1081,7 @@ public class DDLSemanticAnalyzer extends private void analyzeCreateIndex(ASTNode ast) throws SemanticException { String indexName = unescapeIdentifier(ast.getChild(0).getText()); String typeName = unescapeSQLString(ast.getChild(1).getText()); - String[] qualified = getQualifiedTableName((ASTNode) ast.getChild(2)); + String[] qTabName = getQualifiedTableName((ASTNode) ast.getChild(2)); List<String> indexedCols = getColumnNames((ASTNode) ast.getChild(3)); IndexType indexType = HiveIndex.getIndexType(typeName); @@ -1080,15 +1146,15 @@ public class DDLSemanticAnalyzer extends } storageFormat.fillDefaultStorageFormat(); - if (indexTableName == null) { - indexTableName = MetaStoreUtils.getIndexTableName(qualified[0], qualified[1], indexName); - indexTableName = qualified[0] + "." + indexTableName; // on same database with base table + indexTableName = MetaStoreUtils.getIndexTableName(qTabName[0], qTabName[1], indexName); + indexTableName = qTabName[0] + "." + indexTableName; // on same database with base table } else { indexTableName = getDotName(Utilities.getDbTableName(indexTableName)); } + inputs.add(new ReadEntity(getTable(qTabName))); - CreateIndexDesc crtIndexDesc = new CreateIndexDesc(getDotName(qualified), indexName, + CreateIndexDesc crtIndexDesc = new CreateIndexDesc(getDotName(qTabName), indexName, indexedCols, indexTableName, deferredRebuild, storageFormat.getInputFormat(), storageFormat.getOutputFormat(), storageFormat.getStorageHandler(), typeName, location, idxProps, tblProps, @@ -1116,6 +1182,8 @@ public class DDLSemanticAnalyzer extends } } + inputs.add(new ReadEntity(getTable(tableName))); + DropIndexDesc dropIdxDesc = new DropIndexDesc(indexName, tableName); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropIdxDesc), conf)); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g Fri Aug 22 21:36:47 2014 @@ -144,7 +144,7 @@ fromSource @init { gParent.pushMsg("from source", state); } @after { gParent.popMsg(state); } : - ((Identifier LPAREN)=> partitionedTableFunction | tableSource | subQuerySource) (lateralView^)* + ((Identifier LPAREN)=> partitionedTableFunction | tableSource | subQuerySource | virtualTableSource) (lateralView^)* ; tableBucketSample @@ -256,3 +256,46 @@ searchCondition ; //----------------------------------------------------------------------------------- + +//-------- Row Constructor ---------------------------------------------------------- +//in support of SELECT * FROM (VALUES(1,2,3),(4,5,6),...) as FOO(a,b,c) and +// INSERT INTO <table> (col1,col2,...) VALUES(...),(...),... +// INSERT INTO <table> (col1,col2,...) SELECT * FROM (VALUES(1,2,3),(4,5,6),...) as Foo(a,b,c) +valueRowConstructor + : + LPAREN atomExpression (COMMA atomExpression)* RPAREN -> ^(TOK_VALUE_ROW atomExpression+) + ; + +valuesTableConstructor + : + valueRowConstructor (COMMA valueRowConstructor)* -> ^(TOK_VALUES_TABLE valueRowConstructor+) + ; + +/* +VALUES(1),(2) means 2 rows, 1 column each. +VALUES(1,2),(3,4) means 2 rows, 2 columns each. +VALUES(1,2,3) means 1 row, 3 columns +*/ +valuesClause + : + KW_VALUES valuesTableConstructor -> valuesTableConstructor + ; + +/* +This represents a clause like this: +(VALUES(1,2),(2,3)) as VirtTable(col1,col2) +*/ +virtualTableSource + : + LPAREN valuesClause RPAREN tableNameColList -> ^(TOK_VIRTUAL_TABLE tableNameColList valuesClause) + ; +/* +e.g. as VirtTable(col1,col2) +Note that we only want literals as column names +*/ +tableNameColList + : + KW_AS? identifier LPAREN identifier (COMMA identifier)* RPAREN -> ^(TOK_VIRTUAL_TABREF ^(TOK_TABNAME identifier) ^(TOK_COL_NAME identifier+)) + ; + +//----------------------------------------------------------------------------------- \ No newline at end of file Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java Fri Aug 22 21:36:47 2014 @@ -169,6 +169,7 @@ public class FunctionSemanticAnalyzer ex try { String[] qualifiedNameParts = FunctionUtils.getQualifiedFunctionNameParts(functionName); String dbName = qualifiedNameParts[0]; + functionName = qualifiedNameParts[1]; database = getDatabase(dbName); } catch (HiveException e) { LOG.error(e); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Fri Aug 22 21:36:47 2014 @@ -292,6 +292,7 @@ KW_TRANSACTIONS: 'TRANSACTIONS'; KW_REWRITE : 'REWRITE'; KW_AUTHORIZATION: 'AUTHORIZATION'; KW_CONF: 'CONF'; +KW_VALUES: 'VALUES'; // Operators // NOTE: if you add a new function/operator, add it to sysFuncNames so that describe function _FUNC_ will work. Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Fri Aug 22 21:36:47 2014 @@ -146,6 +146,7 @@ TOK_ALTERTABLE_ARCHIVE; TOK_ALTERTABLE_UNARCHIVE; TOK_ALTERTABLE_SERDEPROPERTIES; TOK_ALTERTABLE_SERIALIZER; +TOK_ALTERTABLE_UPDATECOLSTATS; TOK_TABLE_PARTITION; TOK_ALTERTABLE_FILEFORMAT; TOK_ALTERTABLE_LOCATION; @@ -330,6 +331,15 @@ TOK_RESOURCE_LIST; TOK_COMPACT; TOK_SHOW_COMPACTIONS; TOK_SHOW_TRANSACTIONS; +TOK_DELETE_FROM; +TOK_UPDATE_TABLE; +TOK_SET_COLUMNS_CLAUSE; +TOK_VALUE_ROW; +TOK_VALUES_TABLE; +TOK_VIRTUAL_TABLE; +TOK_VIRTUAL_TABREF; +TOK_ANONYMOUS; +TOK_COL_NAME; } @@ -468,6 +478,9 @@ import java.util.HashMap; xlateMap.put("KW_DEFINED", "DEFINED"); xlateMap.put("KW_SUBQUERY", "SUBQUERY"); xlateMap.put("KW_REWRITE", "REWRITE"); + xlateMap.put("KW_UPDATE", "UPDATE"); + + xlateMap.put("KW_VALUES", "VALUES"); // Operators xlateMap.put("DOT", "."); @@ -637,6 +650,8 @@ execStatement | exportStatement | importStatement | ddlStatement + | deleteStatement + | updateStatement ; loadStatement @@ -938,6 +953,7 @@ alterTableStatementSuffix : alterStatementSuffixRename | alterStatementSuffixAddCol | alterStatementSuffixRenameCol + | alterStatementSuffixUpdateStatsCol | alterStatementSuffixDropPartitions | alterStatementSuffixAddPartitions | alterStatementSuffixTouch @@ -1028,6 +1044,13 @@ alterStatementSuffixRenameCol ->^(TOK_ALTERTABLE_RENAMECOL tableName $oldName $newName colType $comment? alterStatementChangeColPosition?) ; +alterStatementSuffixUpdateStatsCol +@init { pushMsg("update column statistics", state); } +@after { popMsg(state); } + : identifier KW_UPDATE KW_STATISTICS KW_FOR KW_COLUMN? colName=identifier KW_SET tableProperties (KW_COMMENT comment=StringLiteral)? + ->^(TOK_ALTERTABLE_UPDATECOLSTATS identifier $colName tableProperties $comment?) + ; + alterStatementChangeColPosition : first=KW_FIRST|KW_AFTER afterCol=identifier ->{$first != null}? ^(TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION ) @@ -1130,6 +1153,7 @@ alterTblPartitionStatementSuffix | alterStatementSuffixMergeFiles | alterStatementSuffixSerdeProperties | alterStatementSuffixRenamePart + | alterStatementSuffixStatsPart | alterStatementSuffixBucketNum | alterTblPartitionStatementSuffixSkewedLocation | alterStatementSuffixClusterbySortby @@ -1221,6 +1245,13 @@ alterStatementSuffixRenamePart ->^(TOK_ALTERTABLE_RENAMEPART partitionSpec) ; +alterStatementSuffixStatsPart +@init { pushMsg("alter table stats partition statement", state); } +@after { popMsg(state); } + : KW_UPDATE KW_STATISTICS KW_FOR KW_COLUMN? colName=identifier KW_SET tableProperties (KW_COMMENT comment=StringLiteral)? + ->^(TOK_ALTERTABLE_UPDATECOLSTATS $colName tableProperties $comment?) + ; + alterStatementSuffixMergeFiles @init { pushMsg("", state); } @after { popMsg(state); } @@ -1300,6 +1331,7 @@ descStatement | (KW_DESCRIBE|KW_DESC) (KW_DATABASE|KW_SCHEMA) KW_EXTENDED? (dbName=identifier) -> ^(TOK_DESCDATABASE $dbName KW_EXTENDED?) ; + analyzeStatement @init { pushMsg("analyze statement", state); } @after { popMsg(state); } @@ -2077,11 +2109,28 @@ singleFromStatement ( b+=body )+ -> ^(TOK_QUERY fromClause body+) ; +/* +The valuesClause rule below ensures that the parse tree for +"insert into table FOO values (1,2),(3,4)" looks the same as +"insert into table FOO select a,b from (values(1,2),(3,4)) as BAR(a,b)" which itself is made to look +very similar to the tree for "insert into table FOO select a,b from BAR". Since virtual table name +is implicit, it's represented as TOK_ANONYMOUS. +*/ regularBody[boolean topLevel] : i=insertClause + ( s=selectStatement[topLevel] {$s.tree.getChild(1).replaceChildren(0, 0, $i.tree);} -> {$s.tree} + | + valuesClause + -> ^(TOK_QUERY + ^(TOK_FROM + ^(TOK_VIRTUAL_TABLE ^(TOK_VIRTUAL_TABREF ^(TOK_ANONYMOUS)) valuesClause) + ) + ^(TOK_INSERT {$i.tree} ^(TOK_SELECT ^(TOK_SELEXPR TOK_ALLCOLREF))) + ) + ) | selectStatement[topLevel] ; @@ -2190,3 +2239,34 @@ limitClause : KW_LIMIT num=Number -> ^(TOK_LIMIT $num) ; + +//DELETE FROM <tableName> WHERE ...; +deleteStatement +@init { pushMsg("delete statement", state); } +@after { popMsg(state); } + : + KW_DELETE KW_FROM tableName (whereClause)? -> ^(TOK_DELETE_FROM tableName whereClause?) + ; + +/*SET <columName> = (3 + col2)*/ +columnAssignmentClause + : + tableOrColumn EQUAL^ atomExpression + ; + +/*SET col1 = 5, col2 = (4 + col4), ...*/ +setColumnsClause + : + KW_SET columnAssignmentClause (COMMA columnAssignmentClause)* -> ^(TOK_SET_COLUMNS_CLAUSE columnAssignmentClause* ) + ; + +/* + UPDATE <table> + SET col1 = val1, col2 = val2... WHERE ... +*/ +updateStatement +@init { pushMsg("update statement", state); } +@after { popMsg(state); } + : + KW_UPDATE tableName setColumnsClause whereClause? -> ^(TOK_UPDATE_TABLE tableName setColumnsClause whereClause?) + ; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Fri Aug 22 21:36:47 2014 @@ -538,5 +538,5 @@ functionIdentifier nonReserved : - KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION + KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION | KW_VALUES ; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Fri Aug 22 21:36:47 2014 @@ -1091,6 +1091,8 @@ public class SemanticAnalyzer extends Ba ASTNode frm = (ASTNode) ast.getChild(0); if (frm.getToken().getType() == HiveParser.TOK_TABREF) { processTable(qb, frm); + } else if (frm.getToken().getType() == HiveParser.TOK_VIRTUAL_TABLE) { + throw new RuntimeException("VALUES() clause is not fully supported yet..."); } else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) { processSubQuery(qb, frm); } else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW || @@ -1282,6 +1284,10 @@ public class SemanticAnalyzer extends Ba case HiveParser.TOK_CTE: processCTE(qb, ast); break; + case HiveParser.TOK_DELETE_FROM: + throw new RuntimeException("DELETE is not (yet) implemented..."); + case HiveParser.TOK_UPDATE_TABLE: + throw new RuntimeException("UPDATE is not (yet) implemented..."); default: skipRecursion = false; break; @@ -10353,6 +10359,7 @@ public class SemanticAnalyzer extends Ba String dbName = qualified.length == 1 ? SessionState.get().getCurrentDatabase() : qualified[0]; Database database = getDatabase(dbName); outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_SHARED)); + outputs.add(new WriteEntity(new Table(dbName, tableName), WriteEntity.WriteType.DDL_NO_LOCK)); if (isTemporary) { if (partCols.size() > 0) { @@ -10526,6 +10533,19 @@ public class SemanticAnalyzer extends Ba try { Table oldView = getTable(createVwDesc.getViewName(), false); + // Do not allow view to be defined on temp table + Set<String> tableAliases = qb.getTabAliases(); + for (String alias : tableAliases) { + try { + Table table = db.getTable(qb.getTabNameForAlias(alias)); + if (table.isTemporary()) { + throw new SemanticException("View definition references temporary table " + alias); + } + } catch (HiveException ex) { + throw new SemanticException(ex); + } + } + // ALTER VIEW AS SELECT requires the view must exist if (createVwDesc.getIsAlterViewAs() && oldView == null) { String viewNotExistErrorMsg = Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Fri Aug 22 21:36:47 2014 @@ -109,6 +109,7 @@ public final class SemanticAnalyzerFacto commandType.put(HiveParser.TOK_ALTERTABLE_PARTCOLTYPE, HiveOperation.ALTERTABLE_PARTCOLTYPE); commandType.put(HiveParser.TOK_SHOW_COMPACTIONS, HiveOperation.SHOW_COMPACTIONS); commandType.put(HiveParser.TOK_SHOW_TRANSACTIONS, HiveOperation.SHOW_TRANSACTIONS); + commandType.put(HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS, HiveOperation.ALTERTABLE_UPDATETABLESTATS); } static { @@ -231,12 +232,14 @@ public final class SemanticAnalyzerFacto case HiveParser.TOK_TRUNCATETABLE: case HiveParser.TOK_EXCHANGEPARTITION: case HiveParser.TOK_SHOW_SET_ROLE: - + case HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS: return new DDLSemanticAnalyzer(conf); case HiveParser.TOK_ALTERTABLE_PARTITION: HiveOperation commandType = null; Integer type = ((ASTNode) tree.getChild(1)).getToken().getType(); - if (tree.getChild(0).getChildCount() > 1) { + if (type == HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS) { + commandType = HiveOperation.ALTERTABLE_UPDATEPARTSTATS; + } else if (tree.getChild(0).getChildCount() > 1) { commandType = tablePartitionCommandType.get(type)[1]; } else { commandType = tablePartitionCommandType.get(type)[0]; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java Fri Aug 22 21:36:47 2014 @@ -27,7 +27,7 @@ public enum HiveOperation { IMPORT("IMPORT", null, new Privilege[]{Privilege.ALTER_METADATA, Privilege.ALTER_DATA}), CREATEDATABASE("CREATEDATABASE", null, new Privilege[]{Privilege.CREATE}), DROPDATABASE("DROPDATABASE", null, new Privilege[]{Privilege.DROP}), - SWITCHDATABASE("SWITCHDATABASE", new Privilege[]{Privilege.SELECT}, null), + SWITCHDATABASE("SWITCHDATABASE", null, null), LOCKDB("LOCKDATABASE", new Privilege[]{Privilege.LOCK}, null), UNLOCKDB("UNLOCKDATABASE", new Privilege[]{Privilege.LOCK}, null), DROPTABLE ("DROPTABLE", null, new Privilege[]{Privilege.DROP}), @@ -38,6 +38,8 @@ public enum HiveOperation { ALTERTABLE_REPLACECOLS("ALTERTABLE_REPLACECOLS", new Privilege[]{Privilege.ALTER_METADATA}, null), ALTERTABLE_RENAMECOL("ALTERTABLE_RENAMECOL", new Privilege[]{Privilege.ALTER_METADATA}, null), ALTERTABLE_RENAMEPART("ALTERTABLE_RENAMEPART", new Privilege[]{Privilege.DROP}, new Privilege[]{Privilege.CREATE}), + ALTERTABLE_UPDATEPARTSTATS("ALTERTABLE_UPDATEPARTSTATS", new Privilege[]{Privilege.ALTER_METADATA}, null), + ALTERTABLE_UPDATETABLESTATS("ALTERTABLE_UPDATETABLESTATS", new Privilege[]{Privilege.ALTER_METADATA}, null), ALTERTABLE_RENAME("ALTERTABLE_RENAME", new Privilege[]{Privilege.ALTER_METADATA}, null), ALTERTABLE_DROPPARTS("ALTERTABLE_DROPPARTS", new Privilege[]{Privilege.DROP}, null), // The location is input and table is output for alter-table add partitions Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Fri Aug 22 21:36:47 2014 @@ -157,7 +157,7 @@ public final class PlanUtils { } catch (ClassNotFoundException e) { // mimicking behaviour in CreateTableDesc tableDesc creation // returning null table description for output. - e.printStackTrace(); + LOG.warn("Unable to find class in getDefaultTableDesc: " + e.getMessage(), e); return null; } return ret; @@ -364,8 +364,7 @@ public final class PlanUtils { ret.setInputFileFormatClass(in_class); ret.setOutputFileFormatClass(out_class); } catch (ClassNotFoundException e) { - e.printStackTrace(); - return null; + throw new RuntimeException("Unable to find class in getTableDesc: " + e.getMessage(), e); } return ret; } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandUtil.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandUtil.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandUtil.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandUtil.java Fri Aug 22 21:36:47 2014 @@ -22,11 +22,14 @@ import java.util.Arrays; import java.util.List; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzContext; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; import org.apache.hadoop.hive.ql.session.SessionState; +import com.google.common.base.Joiner; + class CommandUtil { /** @@ -68,7 +71,10 @@ class CommandUtil { static void authorizeCommandThrowEx(SessionState ss, HiveOperationType type, List<String> command) throws HiveAuthzPluginException, HiveAccessControlException { HivePrivilegeObject commandObj = HivePrivilegeObject.createHivePrivilegeObject(command); - ss.getAuthorizerV2().checkPrivileges(type, Arrays.asList(commandObj), null, null); + HiveAuthzContext.Builder ctxBuilder = new HiveAuthzContext.Builder(); + ctxBuilder.setCommandString(Joiner.on(' ').join(command)); + ctxBuilder.setUserIpAddress(ss.getUserIpAddress()); + ss.getAuthorizerV2().checkPrivileges(type, Arrays.asList(commandObj), null, ctxBuilder.build()); } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java Fri Aug 22 21:36:47 2014 @@ -49,6 +49,9 @@ public enum HiveCommand { if (command.length > 1 && "role".equalsIgnoreCase(command[1])) { // special handling for set role r1 statement return null; + } else if(command.length > 1 && "from".equalsIgnoreCase(command[1])) { + //special handling for SQL "delete from <table> where..." + return null; } else if (COMMANDS.contains(cmd)) { return HiveCommand.valueOf(cmd); } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java Fri Aug 22 21:36:47 2014 @@ -32,6 +32,7 @@ import java.util.TreeMap; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; +import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.parse.VariableSubstitution; import org.apache.hadoop.hive.ql.session.SessionState; @@ -104,11 +105,12 @@ public class SetProcessor implements Com try { return new CommandProcessorResponse(setVariable(varname, varvalue)); } catch (Exception e) { - return new CommandProcessorResponse(1, e.getMessage(), "42000"); + return new CommandProcessorResponse(1, e.getMessage(), "42000", + e instanceof IllegalArgumentException ? null : e); } } - public static int setVariable(String varname, String varvalue) throws IllegalArgumentException { + public static int setVariable(String varname, String varvalue) throws Exception { SessionState ss = SessionState.get(); if (varvalue.contains("\n")){ ss.err.println("Warning: Value had a \\n character in it."); @@ -126,6 +128,10 @@ public class SetProcessor implements Com } else if (varname.startsWith(HIVEVAR_PREFIX)) { String propName = varname.substring(HIVEVAR_PREFIX.length()); ss.getHiveVariables().put(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); + } else if (varname.startsWith(METACONF_PREFIX)) { + String propName = varname.substring(METACONF_PREFIX.length()); + Hive hive = Hive.get(ss.getConf()); + hive.setMetaConf(propName, new VariableSubstitution().substitute(ss.getConf(), varvalue)); } else { setConf(varname, varname, varvalue, true); } @@ -178,8 +184,7 @@ public class SetProcessor implements Com return sortedEnvMap; } - - private CommandProcessorResponse getVariable(String varname) { + private CommandProcessorResponse getVariable(String varname) throws Exception { SessionState ss = SessionState.get(); if (varname.equals("silent")){ ss.out.println("silent" + "=" + ss.getIsSilent()); @@ -222,6 +227,17 @@ public class SetProcessor implements Com ss.out.println(varname + " is undefined as a hive variable"); return new CommandProcessorResponse(1); } + } else if (varname.indexOf(METACONF_PREFIX) == 0) { + String var = varname.substring(METACONF_PREFIX.length()); + Hive hive = Hive.get(ss.getConf()); + String value = hive.getMetaConf(var); + if (value != null) { + ss.out.println(METACONF_PREFIX + var + "=" + value); + return createProcessorSuccessResponse(); + } else { + ss.out.println(varname + " is undefined as a hive meta variable"); + return new CommandProcessorResponse(1); + } } else { dumpOption(varname); return createProcessorSuccessResponse(); @@ -263,10 +279,12 @@ public class SetProcessor implements Com return new CommandProcessorResponse(0); } return executeSetVariable(part[0],part[1]); - } else { + } + try { return getVariable(nwcmd); + } catch (Exception e) { + return new CommandProcessorResponse(1, e.getMessage(), "42000", e); } - } // create a Schema object containing the give column Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java?rev=1619936&r1=1619935&r2=1619936&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java Fri Aug 22 21:36:47 2014 @@ -148,22 +148,19 @@ public class StorageBasedAuthorizationPr public void authorize(Table table, Privilege[] readRequiredPriv, Privilege[] writeRequiredPriv) throws HiveException, AuthorizationException { - // Table path can be null in the case of a new create table - in this case, - // we try to determine what the path would be after the create table is issued. - Path path = null; + // To create/drop/alter a table, the owner should have WRITE permission on the database directory + authorize(hive_db.getDatabase(table.getDbName()), readRequiredPriv, writeRequiredPriv); + + // If the user has specified a location - external or not, check if the user has the try { initWh(); String location = table.getTTable().getSd().getLocation(); - if (location == null || location.isEmpty()) { - path = wh.getTablePath(hive_db.getDatabase(table.getDbName()), table.getTableName()); - } else { - path = new Path(location); + if (location != null && !location.isEmpty()) { + authorize(new Path(location), readRequiredPriv, writeRequiredPriv); } } catch (MetaException ex) { throw hiveException(ex); } - - authorize(path, readRequiredPriv, writeRequiredPriv); } @Override
