http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java index 2a6b484..8b83bf7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java @@ -21,7 +21,7 @@ import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PArrayDataType; -import org.apache.phoenix.schema.types.PArrayDataType.PArrayDataTypeBytesArrayBuilder; +import org.apache.phoenix.schema.types.PArrayDataTypeEncoder; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.TrustedByteArrayOutputStream; @@ -35,38 +35,26 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { private Object[] elements; private final ImmutableBytesWritable valuePtr = new ImmutableBytesWritable(); private int estimatedSize = 0; - // store the offset postion in this. Later based on the total size move this to a byte[] - // and serialize into byte stream - private int[] offsetPos; private boolean rowKeyOrderOptimizable; - private byte serializationVersion; public ArrayConstructorExpression() { } public ArrayConstructorExpression(List<Expression> children, PDataType baseType, boolean rowKeyOrderOptimizable) { - this(children, baseType, rowKeyOrderOptimizable, PArrayDataType.SORTABLE_SERIALIZATION_VERSION); - } - - public ArrayConstructorExpression(List<Expression> children, PDataType baseType, boolean rowKeyOrderOptimizable, byte serializationVersion) { super(children); - init(baseType, rowKeyOrderOptimizable, serializationVersion); + init(baseType, rowKeyOrderOptimizable); } public ArrayConstructorExpression clone(List<Expression> children) { - return new ArrayConstructorExpression(children, this.baseType, this.rowKeyOrderOptimizable, PArrayDataType.SORTABLE_SERIALIZATION_VERSION); + return new ArrayConstructorExpression(children, this.baseType, this.rowKeyOrderOptimizable); } - private void init(PDataType baseType, boolean rowKeyOrderOptimizable, byte serializationVersion) { + private void init(PDataType baseType, boolean rowKeyOrderOptimizable) { this.baseType = baseType; this.rowKeyOrderOptimizable = rowKeyOrderOptimizable; elements = new Object[getChildren().size()]; valuePtr.set(ByteUtil.EMPTY_BYTE_ARRAY); estimatedSize = PArrayDataType.estimateSize(this.children.size(), this.baseType); - if (!this.baseType.isFixedWidth()) { - offsetPos = new int[children.size()]; - } - this.serializationVersion = serializationVersion; } @Override @@ -90,37 +78,24 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { } TrustedByteArrayOutputStream byteStream = new TrustedByteArrayOutputStream(estimatedSize); DataOutputStream oStream = new DataOutputStream(byteStream); - PArrayDataTypeBytesArrayBuilder builder = - new PArrayDataTypeBytesArrayBuilder(byteStream, oStream, children.size(), baseType, getSortOrder(), rowKeyOrderOptimizable, serializationVersion); - try { - for (int i = position >= 0 ? position : 0; i < elements.length; i++) { - Expression child = children.get(i); - if (!child.evaluate(tuple, ptr)) { - if (tuple != null && !tuple.isImmutable()) { - if (position >= 0) position = i; - return false; - } - else { - // its possible for the expression to evaluate to null if the serialization format is immutable and the data type is variable length - builder.appendMissingElement(); - } - } else { - builder.appendElem(ptr.get(), ptr.getOffset(), ptr.getLength()); + PArrayDataTypeEncoder builder = + new PArrayDataTypeEncoder(byteStream, oStream, children.size(), baseType, getSortOrder(), rowKeyOrderOptimizable, PArrayDataType.SORTABLE_SERIALIZATION_VERSION); + for (int i = position >= 0 ? position : 0; i < elements.length; i++) { + Expression child = children.get(i); + if (!child.evaluate(tuple, ptr)) { + if (tuple != null && !tuple.isImmutable()) { + if (position >= 0) position = i; + return false; } - } - if (position >= 0) position = elements.length; - byte[] bytes = builder.getBytesAndClose(); - ptr.set(bytes, 0, bytes.length); - valuePtr.set(ptr.get(), ptr.getOffset(), ptr.getLength()); - return true; - } finally { - try { - byteStream.close(); - oStream.close(); - } catch (IOException e) { - // Should not happen + } else { + builder.appendValue(ptr.get(), ptr.getOffset(), ptr.getLength()); } } + if (position >= 0) position = elements.length; + byte[] bytes = builder.encode(); + ptr.set(bytes, 0, bytes.length); + valuePtr.set(ptr.get(), ptr.getOffset(), ptr.getLength()); + return true; } @@ -133,8 +108,7 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { rowKeyOrderOptimizable = true; baseTypeOrdinal = -(baseTypeOrdinal+1); } - byte serializationVersion = input.readByte(); - init(PDataType.values()[baseTypeOrdinal], rowKeyOrderOptimizable, serializationVersion); + init(PDataType.values()[baseTypeOrdinal], rowKeyOrderOptimizable); } @Override @@ -145,7 +119,6 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { } else { WritableUtils.writeVInt(output, baseType.ordinal()); } - output.write(serializationVersion); } @Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java index 006777b..7f95da6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java @@ -179,7 +179,7 @@ public enum ExpressionType { DayOfWeekFunction(DayOfWeekFunction.class), DayOfYearFunction(DayOfYearFunction.class), DefaultValueExpression(DefaultValueExpression.class), - ArrayColumnExpression(ArrayColumnExpression.class); + ArrayColumnExpression(SingleCellColumnExpression.class); ExpressionType(Class<? extends Expression> clazz) { this.clazz = clazz; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java new file mode 100644 index 0000000..e27f9d0 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression; + +import static org.apache.phoenix.query.QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.compile.CreateTableCompiler.ViewWhereExpressionVisitor; +import org.apache.phoenix.expression.visitor.ExpressionVisitor; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.ColumnValueDecoder; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PDatum; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.util.SchemaUtil; + +import com.google.common.base.Preconditions; + +/** + * + * Class to access a column that is stored in a Cell that contains all + * columns for a given column family (stored in a serialized array). + * + */ +public class SingleCellColumnExpression extends KeyValueColumnExpression { + + private int decodedColumnQualifier; + private String arrayColDisplayName; + private KeyValueColumnExpression keyValueColumnExpression; + private QualifierEncodingScheme encodingScheme; + + public SingleCellColumnExpression() { + } + + public SingleCellColumnExpression(PDatum column, byte[] cf, byte[] cq, QualifierEncodingScheme encodingScheme) { + super(column, cf, SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + Preconditions.checkNotNull(encodingScheme); + Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); + this.decodedColumnQualifier = encodingScheme.decode(cq); + this.encodingScheme = encodingScheme; + setKeyValueExpression(); + } + + public SingleCellColumnExpression(PColumn column, String displayName, QualifierEncodingScheme encodingScheme) { + super(column, column.getFamilyName().getBytes(), SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + Preconditions.checkNotNull(encodingScheme); + Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); + this.arrayColDisplayName = displayName; + this.decodedColumnQualifier = encodingScheme.decode(column.getColumnQualifierBytes()); + this.encodingScheme = encodingScheme; + setKeyValueExpression(); + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + if (!super.evaluate(tuple, ptr)) { + return false; + } else if (ptr.getLength() == 0) { + return true; + } + // the first position is reserved and we offset maxEncodedColumnQualifier by ENCODED_CQ_COUNTER_INITIAL_VALUE (which is the minimum encoded column qualifier) + int index = decodedColumnQualifier-QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE+1; + byte serializedImmutableStorageScheme = ptr.get()[ptr.getOffset() + ptr.getLength() - Bytes.SIZEOF_BYTE]; + ImmutableStorageScheme immutableStorageScheme = ImmutableStorageScheme.fromSerializedValue(serializedImmutableStorageScheme); + // Given a ptr to the entire array, set ptr to point to a particular element within that array + ColumnValueDecoder encoderDecoder = immutableStorageScheme.getDecoder(); + return encoderDecoder.decode(ptr, index); + } + + @Override + public void readFields(DataInput input) throws IOException { + super.readFields(input); + this.decodedColumnQualifier = WritableUtils.readVInt(input); + this.encodingScheme = QualifierEncodingScheme.values()[WritableUtils.readVInt(input)]; + setKeyValueExpression(); + } + + @Override + public void write(DataOutput output) throws IOException { + super.write(output); + WritableUtils.writeVInt(output, decodedColumnQualifier); + WritableUtils.writeVInt(output, encodingScheme.ordinal()); + } + + public KeyValueColumnExpression getKeyValueExpression() { + return keyValueColumnExpression; + } + + private void setKeyValueExpression() { + final boolean isNullable = isNullable(); + final SortOrder sortOrder = getSortOrder(); + final Integer scale = getScale(); + final Integer maxLength = getMaxLength(); + final PDataType datatype = getDataType(); + this.keyValueColumnExpression = new KeyValueColumnExpression(new PDatum() { + @Override + public boolean isNullable() { + return isNullable; + } + + @Override + public SortOrder getSortOrder() { + return sortOrder; + } + + @Override + public Integer getScale() { + return scale; + } + + @Override + public Integer getMaxLength() { + return maxLength; + } + + @Override + public PDataType getDataType() { + return datatype; + } + }, getColumnFamily(), getPositionInArray()); + } + + @Override + public String toString() { + if (arrayColDisplayName == null) { + arrayColDisplayName = SchemaUtil.getColumnDisplayName(getColumnFamily(), getColumnQualifier()); + } + return arrayColDisplayName; + } + + public byte[] getPositionInArray() { + return encodingScheme.encode(decodedColumnQualifier); + } + + @Override + public <T> T accept(ExpressionVisitor<T> visitor) { + //FIXME: this is ugly but can't think of a good solution. + if (visitor instanceof ViewWhereExpressionVisitor) { + return visitor.visit(this); + } else { + return super.accept(visitor); + } + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java new file mode 100644 index 0000000..48485be --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by + * applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.apache.phoenix.expression; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.expression.visitor.ExpressionVisitor; +import org.apache.phoenix.schema.ColumnValueEncoder; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarbinary; + +/** + * Expression used to create a single cell containing all the column values for a column family + */ +public class SingleCellConstructorExpression extends BaseCompoundExpression { + + private ImmutableStorageScheme immutableStorageScheme; + + public SingleCellConstructorExpression(ImmutableStorageScheme immutableStorageScheme, List<Expression> children) { + super(children); + this.immutableStorageScheme = immutableStorageScheme; + } + + @Override + public PDataType getDataType() { + return PVarbinary.INSTANCE; + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + ColumnValueEncoder encoderDecoder = immutableStorageScheme.getEncoder(children.size()); + for (int i=0; i < children.size(); i++) { + Expression child = children.get(i); + if (!child.evaluate(tuple, ptr)) { + encoderDecoder.appendAbsentValue(); + } else { + encoderDecoder.appendValue(ptr.get(), ptr.getOffset(), ptr.getLength()); + } + } + byte[] bytes = encoderDecoder.encode(); + ptr.set(bytes, 0, bytes.length); + return true; + } + + + @Override + public void readFields(DataInput input) throws IOException { + super.readFields(input); + this.immutableStorageScheme = WritableUtils.readEnum(input, ImmutableStorageScheme.class); + } + + @Override + public void write(DataOutput output) throws IOException { + super.write(output); + WritableUtils.writeEnum(output, immutableStorageScheme); + } + + @Override + public boolean requiresFinalEvaluation() { + return true; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder("["); + if (children.size()==0) + return buf.append("]").toString(); + for (int i = 0; i < children.size() - 1; i++) { + buf.append(children.get(i) + ","); + } + buf.append(children.get(children.size()-1) + "]"); + return buf.toString(); + } + + @Override + public final <T> T accept(ExpressionVisitor<T> visitor) { + List<T> l = acceptChildren(visitor, visitor.visitEnter(this)); + T t = visitor.visitLeave(this, l); + if (t == null) { + t = visitor.defaultReturn(this, l); + } + return t; + } + + public SingleCellConstructorExpression clone(List<Expression> children) { + return new SingleCellConstructorExpression(immutableStorageScheme, children); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java index 6631e70..06bbced 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java @@ -27,7 +27,7 @@ import org.apache.phoenix.expression.BaseCompoundExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.tuple.Tuple; -import org.apache.phoenix.schema.types.PArrayDataType; +import org.apache.phoenix.schema.types.PArrayDataTypeDecoder; import org.apache.phoenix.schema.types.PDataType; public class ArrayElemRefExpression extends BaseCompoundExpression { @@ -48,7 +48,7 @@ public class ArrayElemRefExpression extends BaseCompoundExpression { @Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { Expression arrayExpr = children.get(0); - return PArrayDataType.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), getMaxLength()); + return PArrayDataTypeDecoder.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), getMaxLength()); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java index 7a23ef5..0f3c40c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java @@ -24,9 +24,9 @@ import org.apache.phoenix.expression.Expression; import org.apache.phoenix.parse.FunctionParseNode.Argument; import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction; import org.apache.phoenix.parse.ParseException; +import org.apache.phoenix.schema.types.PArrayDataTypeDecoder; import org.apache.phoenix.schema.types.PBinaryArray; import org.apache.phoenix.schema.types.PInteger; -import org.apache.phoenix.schema.types.PArrayDataType; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PVarbinaryArray; import org.apache.phoenix.schema.SortOrder; @@ -61,7 +61,7 @@ public class ArrayIndexFunction extends ScalarFunction { throw new ParseException("Index cannot be negative :" + index); } Expression arrayExpr = children.get(0); - return PArrayDataType.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), + return PArrayDataTypeDecoder.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), getMaxLength()); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java index 522a4e7..f2ed97b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.schema.SortOrder; -import org.apache.phoenix.schema.types.PArrayDataType.PArrayDataTypeBytesArrayBuilder; +import org.apache.phoenix.schema.types.PArrayDataTypeEncoder; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; @@ -159,8 +159,8 @@ public class JONIPattern extends AbstractBasePattern implements AbstractBaseSpli private boolean split(byte[] srcBytes, int srcOffset, int srcLen, ImmutableBytesWritable outPtr) { SortOrder sortOrder = SortOrder.ASC; - PArrayDataTypeBytesArrayBuilder builder = - new PArrayDataTypeBytesArrayBuilder(PVarchar.INSTANCE, sortOrder); + PArrayDataTypeEncoder builder = + new PArrayDataTypeEncoder(PVarchar.INSTANCE, sortOrder); int srcRange = srcOffset + srcLen; Matcher matcher = pattern.matcher(srcBytes, 0, srcRange); int cur = srcOffset; @@ -168,31 +168,29 @@ public class JONIPattern extends AbstractBasePattern implements AbstractBaseSpli while (true) { int nextCur = matcher.search(cur, srcRange, Option.DEFAULT); if (nextCur < 0) { - append = builder.appendElem(srcBytes, cur, srcRange - cur); - if (!append) return false; + builder.appendValue(srcBytes, cur, srcRange - cur); break; } // To handle the following case, which adds null at first. // REGEXP_SPLIT("12ONE34TWO56THREE78","[0-9]+")={null, "ONE", "TWO", "THREE", null} if (cur == matcher.getBegin()) { - builder.appendElem(srcBytes, cur, 0); + builder.appendValue(srcBytes, cur, 0); } if (cur < matcher.getBegin()) { - append = builder.appendElem(srcBytes, cur, matcher.getBegin() - cur); - if (!append) return false; + builder.appendValue(srcBytes, cur, matcher.getBegin() - cur); } cur = matcher.getEnd(); // To handle the following case, which adds null at last. // REGEXP_SPLIT("12ONE34TWO56THREE78","[0-9]+")={null, "ONE", "TWO", "THREE", null} if (cur == srcRange) { - builder.appendElem(srcBytes, cur, 0); + builder.appendValue(srcBytes, cur, 0); break; } } - byte[] bytes = builder.getBytesAndClose(); + byte[] bytes = builder.encode(); if (bytes == null) return false; outPtr.set(bytes); return true; http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java index 8e8b32d..d79b546 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java @@ -36,6 +36,7 @@ import org.apache.phoenix.expression.MultiplyExpression; import org.apache.phoenix.expression.NotExpression; import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -142,6 +143,11 @@ public abstract class BaseExpressionVisitor<E> implements ExpressionVisitor<E> { } @Override + public Iterator<Expression> visitEnter(SingleCellConstructorExpression node) { + return null; + } + + @Override public Iterator<Expression> visitEnter(ModulusExpression modulusExpression) { return null; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java index 15a9f74..e47fb64 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java @@ -26,7 +26,6 @@ import org.apache.phoenix.expression.ArrayConstructorExpression; import org.apache.phoenix.expression.CaseExpression; import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.ComparisonExpression; -import org.apache.phoenix.expression.ArrayColumnExpression; import org.apache.phoenix.expression.CorrelateVariableFieldAccessExpression; import org.apache.phoenix.expression.DivideExpression; import org.apache.phoenix.expression.Expression; @@ -42,6 +41,8 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -83,7 +84,7 @@ public abstract class CloneExpressionVisitor extends TraverseAllExpressionVisito } @Override - public Expression visit(ArrayColumnExpression node) { + public Expression visit(SingleCellColumnExpression node) { return node; } @@ -188,6 +189,11 @@ public abstract class CloneExpressionVisitor extends TraverseAllExpressionVisito public Expression visitLeave(ArrayConstructorExpression node, List<Expression> l) { return isCloneNode(node, l) ? node.clone(l) : node; } + + @Override + public Expression visitLeave(SingleCellConstructorExpression node, List<Expression> l) { + return isCloneNode(node, l) ? node.clone(l) : node; + } @Override public Expression visitLeave(StringConcatExpression node, List<Expression> l) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java index 1aeb9a9..9a56e36 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java @@ -24,6 +24,7 @@ import org.apache.phoenix.expression.Expression; public class CloneNonDeterministicExpressionVisitor extends CloneExpressionVisitor { + @Override public boolean isCloneNode(Expression node, List<Expression> children) { return Determinism.PER_INVOCATION.compareTo(node.getDeterminism()) <= 0; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java index 100f099..5936dc7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java @@ -23,11 +23,11 @@ import java.util.List; import org.apache.phoenix.compile.SequenceValueExpression; import org.apache.phoenix.expression.AddExpression; import org.apache.phoenix.expression.AndExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.expression.ArrayConstructorExpression; import org.apache.phoenix.expression.CaseExpression; import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.ComparisonExpression; -import org.apache.phoenix.expression.ArrayColumnExpression; import org.apache.phoenix.expression.CorrelateVariableFieldAccessExpression; import org.apache.phoenix.expression.DivideExpression; import org.apache.phoenix.expression.Expression; @@ -43,6 +43,7 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -110,11 +111,14 @@ public interface ExpressionVisitor<E> { public Iterator<Expression> visitEnter(ArrayConstructorExpression node); public E visitLeave(ArrayConstructorExpression node, List<E> l); + public Iterator<Expression> visitEnter(SingleCellConstructorExpression node); + public E visitLeave(SingleCellConstructorExpression node, List<E> l); + public E visit(CorrelateVariableFieldAccessExpression node); public E visit(LiteralExpression node); public E visit(RowKeyColumnExpression node); public E visit(KeyValueColumnExpression node); - public E visit(ArrayColumnExpression node); + public E visit(SingleCellColumnExpression node); public E visit(ProjectedColumnExpression node); public E visit(SequenceValueExpression node); http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java index 9e50bc4..f5615be 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java @@ -26,7 +26,6 @@ import org.apache.phoenix.expression.ArrayConstructorExpression; import org.apache.phoenix.expression.CaseExpression; import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.ComparisonExpression; -import org.apache.phoenix.expression.ArrayColumnExpression; import org.apache.phoenix.expression.CorrelateVariableFieldAccessExpression; import org.apache.phoenix.expression.DivideExpression; import org.apache.phoenix.expression.InListExpression; @@ -41,6 +40,8 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -121,7 +122,7 @@ public class StatelessTraverseAllExpressionVisitor<E> extends TraverseAllExpress } @Override - public E visit(ArrayColumnExpression node) { + public E visit(SingleCellColumnExpression node) { return null; } @@ -169,6 +170,11 @@ public class StatelessTraverseAllExpressionVisitor<E> extends TraverseAllExpress public E visitLeave(ArrayConstructorExpression node, List<E> l) { return null; } + + @Override + public E visitLeave(SingleCellConstructorExpression node, List<E> l) { + return null; + } @Override public E visitLeave(ModulusExpression node, List<E> l) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java index 1a2f2cc..7f447b3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java @@ -26,7 +26,6 @@ import org.apache.phoenix.expression.ArrayConstructorExpression; import org.apache.phoenix.expression.CaseExpression; import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.ComparisonExpression; -import org.apache.phoenix.expression.ArrayColumnExpression; import org.apache.phoenix.expression.CorrelateVariableFieldAccessExpression; import org.apache.phoenix.expression.DivideExpression; import org.apache.phoenix.expression.InListExpression; @@ -41,6 +40,8 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -116,7 +117,7 @@ public class StatelessTraverseNoExpressionVisitor<E> extends TraverseNoExpressio } @Override - public E visit(ArrayColumnExpression node) { + public E visit(SingleCellColumnExpression node) { return null; } @@ -169,6 +170,11 @@ public class StatelessTraverseNoExpressionVisitor<E> extends TraverseNoExpressio public E visitLeave(ArrayConstructorExpression node, List<E> l) { return null; } + + @Override + public E visitLeave(SingleCellConstructorExpression node, List<E> l) { + return null; + } @Override public E visitLeave(ModulusExpression node, List<E> l) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java index a7146fc..88f707d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java @@ -26,7 +26,7 @@ import java.util.TreeSet; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.expression.ArrayColumnExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.visitor.ExpressionVisitor; http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java index b97c4e9..ae3557d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java @@ -22,7 +22,7 @@ import java.io.IOException; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.expression.ArrayColumnExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.visitor.StatelessTraverseAllExpressionVisitor; http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java index bd8236c..99b6bf8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java @@ -52,13 +52,13 @@ import org.apache.phoenix.compile.ColumnResolver; import org.apache.phoenix.compile.FromCompiler; import org.apache.phoenix.compile.IndexExpressionCompiler; import org.apache.phoenix.compile.StatementContext; -import org.apache.phoenix.expression.ArrayColumnExpression; -import org.apache.phoenix.expression.ArrayConstructorExpression; import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.ExpressionType; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.LiteralExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.visitor.KeyValueExpressionVisitor; import org.apache.phoenix.hbase.index.ValueGetter; import org.apache.phoenix.hbase.index.covered.update.ColumnReference; @@ -80,9 +80,9 @@ import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PDatum; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.schema.PTable.IndexType; import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; -import org.apache.phoenix.schema.PTable.StorageScheme; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SaltingUtil; @@ -93,7 +93,6 @@ import org.apache.phoenix.schema.ValueSchema.Field; import org.apache.phoenix.schema.tuple.BaseTuple; import org.apache.phoenix.schema.tuple.ValueGetterTuple; import org.apache.phoenix.schema.types.PDataType; -import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.BitSet; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.EncodedColumnsUtil; @@ -309,7 +308,6 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { private boolean indexWALDisabled; private boolean isLocalIndex; private boolean immutableRows; - private boolean storeColsInSingleCell; // Transient state private final boolean isDataTableSalted; @@ -323,6 +321,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { private boolean rowKeyOrderOptimizable; private ImmutableBytesPtr emptyKeyValueQualifierPtr; private QualifierEncodingScheme encodingScheme; + private ImmutableStorageScheme immutableStorageScheme; private IndexMaintainer(RowKeySchema dataRowKeySchema, boolean isDataTableSalted) { this.dataRowKeySchema = dataRowKeySchema; @@ -337,6 +336,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { this.viewIndexId = index.getViewIndexId() == null ? null : MetaDataUtil.getViewIndexIdDataType().toBytes(index.getViewIndexId()); this.isLocalIndex = index.getIndexType() == IndexType.LOCAL; this.encodingScheme = index.getEncodingScheme(); + this.immutableStorageScheme = index.getImmutableStorageScheme(); byte[] indexTableName = index.getPhysicalName().getBytes(); // Use this for the nDataSaltBuckets as we need this for local indexes // TODO: persist nDataSaltBuckets separately, but maintain b/w compat. @@ -395,7 +395,6 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { // TODO: check whether index is immutable or not. Currently it's always false so checking // data table is with immutable rows or not. this.immutableRows = dataTable.isImmutableRows(); - this.storeColsInSingleCell = index.getStorageScheme() == StorageScheme.ONE_CELL_PER_COLUMN_FAMILY; int indexColByteSize = 0; ColumnResolver resolver = null; List<ParseNode> parseNodes = new ArrayList<ParseNode>(1); @@ -464,18 +463,18 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { } @Override - public Void visit(ArrayColumnExpression expression) { + public Void visit(SingleCellColumnExpression expression) { return addDataColInfo(dataTable, expression); } private Void addDataColInfo(final PTable dataTable, Expression expression) { - Preconditions.checkArgument(expression instanceof ArrayColumnExpression + Preconditions.checkArgument(expression instanceof SingleCellColumnExpression || expression instanceof KeyValueColumnExpression); KeyValueColumnExpression colExpression = null; - if (expression instanceof ArrayColumnExpression) { + if (expression instanceof SingleCellColumnExpression) { colExpression = - ((ArrayColumnExpression) expression).getKeyValueExpression(); + ((SingleCellColumnExpression) expression).getKeyValueExpression(); } else { colExpression = ((KeyValueColumnExpression) expression); } @@ -934,7 +933,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { put.setDurability(!indexWALDisabled ? Durability.USE_DEFAULT : Durability.SKIP_WAL); } ImmutableBytesPtr rowKey = new ImmutableBytesPtr(indexRowKey); - if (storeColsInSingleCell) { + if (immutableStorageScheme != immutableStorageScheme.ONE_CELL_PER_COLUMN) { // map from index column family to list of pair of index column and data column (for covered columns) Map<ImmutableBytesPtr, List<Pair<ColumnReference, ColumnReference>>> familyToColListMap = Maps.newHashMap(); for (ColumnReference ref : this.getCoveredColumns()) { @@ -949,18 +948,17 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { for (Entry<ImmutableBytesPtr, List<Pair<ColumnReference, ColumnReference>>> entry : familyToColListMap.entrySet()) { byte[] columnFamily = entry.getKey().copyBytesIfNecessary(); List<Pair<ColumnReference, ColumnReference>> colRefPairs = entry.getValue(); - int maxIndex = Integer.MIN_VALUE; + int maxEncodedColumnQualifier = Integer.MIN_VALUE; // find the max col qualifier for (Pair<ColumnReference, ColumnReference> colRefPair : colRefPairs) { - int qualifier = encodingScheme.decode(colRefPair.getFirst().getQualifier()); - maxIndex = Math.max(maxIndex, qualifier); + maxEncodedColumnQualifier = Math.max(maxEncodedColumnQualifier, encodingScheme.decode(colRefPair.getFirst().getQualifier())); } - byte[][] colValues = new byte[maxIndex+1][]; + Expression[] colValues = EncodedColumnsUtil.createColumnExpressionArray(maxEncodedColumnQualifier); // set the values of the columns for (Pair<ColumnReference, ColumnReference> colRefPair : colRefPairs) { ColumnReference indexColRef = colRefPair.getFirst(); ColumnReference dataColRef = colRefPair.getSecond(); - Expression expression = new ArrayColumnExpression(new PDatum() { + Expression expression = new SingleCellColumnExpression(new PDatum() { @Override public boolean isNullable() { return false; @@ -990,21 +988,16 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { expression.evaluate(new ValueGetterTuple(valueGetter), ptr); byte[] value = ptr.copyBytesIfNecessary(); if (value != null) { - int indexArrayPos = encodingScheme.decode(indexColRef.getQualifier()); - colValues[indexArrayPos] = value; + int indexArrayPos = encodingScheme.decode(indexColRef.getQualifier())-QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE+1; + colValues[indexArrayPos] = new LiteralExpression(value); } } - List<Expression> children = Lists.newArrayListWithExpectedSize(colRefPairs.size()); - // create an expression list with all the columns - for (int j=0; j<colValues.length; ++j) { - children.add(new LiteralExpression(colValues[j]==null ? ByteUtil.EMPTY_BYTE_ARRAY : colValues[j] )); - } - // we use ArrayConstructorExpression to serialize multiple columns into a single byte[] - // construct the ArrayConstructorExpression with a variable length data type (PVarchar) since columns can be of fixed or variable length - ArrayConstructorExpression arrayExpression = new ArrayConstructorExpression(children, PVarchar.INSTANCE, rowKeyOrderOptimizable); + List<Expression> children = Arrays.asList(colValues); + // we use SingleCellConstructorExpression to serialize multiple columns into a single byte[] + SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - arrayExpression.evaluate(new BaseTuple() {}, ptr); + singleCellConstructorExpression.evaluate(new BaseTuple() {}, ptr); if (put == null) { put = new Put(indexRowKey); put.setDurability(!indexWALDisabled ? Durability.USE_DEFAULT : Durability.SKIP_WAL); @@ -1308,7 +1301,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { coveredColumnsInfo.add(new Pair<>(Bytes.toString(dataTableCf), Bytes.toString(dataTableCq))); } encodingScheme = WritableUtils.readEnum(input, QualifierEncodingScheme.class); - storeColsInSingleCell = WritableUtils.readVInt(input) > 0; + immutableStorageScheme = WritableUtils.readEnum(input, ImmutableStorageScheme.class); initCachedState(); } @@ -1370,7 +1363,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { Bytes.writeByteArray(output, colInfo.getSecond().getBytes()); } WritableUtils.writeEnum(output, encodingScheme); - WritableUtils.writeVInt(output, storeColsInSingleCell ? 1 : -1); + WritableUtils.writeEnum(output, immutableStorageScheme); } public int getEstimatedByteSize() { @@ -1705,7 +1698,7 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { return indexedColumnsInfo; } - public StorageScheme getIndexStorageScheme() { - return storeColsInSingleCell ? StorageScheme.ONE_CELL_PER_COLUMN_FAMILY : StorageScheme.ONE_CELL_PER_KEYVALUE_COLUMN; + public ImmutableStorageScheme getIndexStorageScheme() { + return immutableStorageScheme; } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 7ece6df..780d70f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -87,7 +87,6 @@ import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; -import org.apache.phoenix.schema.PTable.StorageScheme; import org.apache.phoenix.schema.PTable.ViewType; import org.apache.phoenix.schema.StaleRegionBoundaryCacheException; import org.apache.phoenix.schema.TableRef; @@ -249,6 +248,7 @@ public abstract class BaseResultIterators extends ExplainTable implements Result } } scan.setAttribute(BaseScannerRegionObserver.QUALIFIER_ENCODING_SCHEME, new byte[]{table.getEncodingScheme().getSerializedMetadataValue()}); + scan.setAttribute(BaseScannerRegionObserver.IMMUTABLE_STORAGE_ENCODING_SCHEME, new byte[]{table.getImmutableStorageScheme().getSerializedMetadataValue()}); // When analyzing the table, there is no look up for key values being done. // So there is no point setting the range. if (EncodedColumnsUtil.setQualifierRanges(table) && !ScanUtil.isAnalyzeTable(scan)) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index d345a07..f7c76f5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -323,8 +323,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { /** Version below which we fall back on the generic KeyValueBuilder */ public static final int CLIENT_KEY_VALUE_BUILDER_THRESHOLD = VersionUtil.encodeVersion("0", "94", "14"); - public static final String STORAGE_SCHEME = "STORAGE_SCHEME"; - public static final byte[] STORAGE_SCHEME_BYTES = Bytes.toBytes(STORAGE_SCHEME); + public static final String IMMUTABLE_STORAGE_SCHEME = "IMMUTABLE_STORAGE_SCHEME"; + public static final byte[] STORAGE_SCHEME_BYTES = Bytes.toBytes(IMMUTABLE_STORAGE_SCHEME); public static final String ENCODING_SCHEME = "ENCODING_SCHEME"; public static final byte[] ENCODING_SCHEME_BYTES = Bytes.toBytes(ENCODING_SCHEME); public static final String COLUMN_QUALIFIER = "COLUMN_QUALIFIER"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 076b788..2c36af4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -2722,7 +2722,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 2, - PhoenixDatabaseMetaData.STORAGE_SCHEME + " " + PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME + " " + PTinyint.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists( metaConnection, http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index 4085251..9f4a569 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -88,7 +88,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.START_WITH; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORAGE_SCHEME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; @@ -323,7 +323,7 @@ public interface QueryConstants { APPEND_ONLY_SCHEMA + " BOOLEAN," + GUIDE_POSTS_WIDTH + " BIGINT," + COLUMN_QUALIFIER + " VARBINARY," + - STORAGE_SCHEME + " TINYINT, " + + IMMUTABLE_STORAGE_SCHEME + " TINYINT, " + ENCODING_SCHEME + " TINYINT, " + COLUMN_QUALIFIER_COUNTER + " INTEGER, " + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index 3789138..6261efd 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -228,6 +228,7 @@ public interface QueryServices extends SQLCloseable { public static final String AUTO_UPGRADE_ENABLED = "phoenix.autoupgrade.enabled"; public static final String DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB = "phoenix.default.column.encoded.bytes.attrib"; + public static final String DEFAULT_IMMUTABLE_STORAGE_SCHEME_ATTRIB = "phoenix.default.immutable.storage.scheme"; /** * Get executor service used for parallel scans */ http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 60599f9..fea5ee4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -91,6 +91,8 @@ import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.ipc.controller.ClientRpcControllerFactory; import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.PTableRefFactory; import org.apache.phoenix.trace.util.Tracing; import org.apache.phoenix.util.DateUtil; @@ -257,7 +259,8 @@ public class QueryServicesOptions { public static final String DEFAULT_CLIENT_CACHE_ENCODING = PTableRefFactory.Encoding.OBJECT.toString(); public static final boolean DEFAULT_AUTO_UPGRADE_ENABLED = true; - public static final int DEFAULT_COLUMN_ENCODED_BYTES = 0; + public static final int DEFAULT_COLUMN_ENCODED_BYTES = QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS.getSerializedMetadataValue(); + public static final String DEFAULT_IMMUTABLE_STORAGE_SCHEME = ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS.toString(); @SuppressWarnings("serial") public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() { http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java index c997074..c73b860 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnRef.java @@ -25,17 +25,17 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.http.annotation.Immutable; import org.apache.phoenix.compile.ExpressionCompiler; import org.apache.phoenix.compile.StatementContext; -import org.apache.phoenix.expression.ArrayColumnExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.expression.function.DefaultValueExpression; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.ParseNode; import org.apache.phoenix.parse.SQLParser; -import org.apache.phoenix.schema.PTable.StorageScheme; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.util.ExpressionUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; @@ -125,8 +125,8 @@ public class ColumnRef { return new ProjectedColumnExpression(column, table, displayName); } - Expression expression = table.getStorageScheme() == StorageScheme.ONE_CELL_PER_COLUMN_FAMILY ? - new ArrayColumnExpression(column, displayName, table.getEncodingScheme()) : new KeyValueColumnExpression(column, displayName); + Expression expression = table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ? + new SingleCellColumnExpression(column, displayName, table.getEncodingScheme()) : new KeyValueColumnExpression(column, displayName); if (column.getExpressionStr() != null) { String url = PhoenixRuntime.JDBC_PROTOCOL http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnValueDecoder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnValueDecoder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnValueDecoder.java new file mode 100644 index 0000000..5ae72d1 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnValueDecoder.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; + +/** + * Interface to decode column values that are stored in a byte[] + */ +public interface ColumnValueDecoder { + /** + * sets the ptr to the column value at the given index + * @return false if the column value is absent (used to support DEFAULT expressions) or else true + */ + boolean decode(ImmutableBytesWritable ptr, int index); +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnValueEncoder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnValueEncoder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnValueEncoder.java new file mode 100644 index 0000000..5e930bd --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/ColumnValueEncoder.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema; + +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; + + +/** + * Interface to encode column values into a serialized byte[] that will be stored in a single cell + * The last byte of the serialized byte[] should be the serialized value of the {@link ImmutableStorageScheme} + * that was used. + */ +public interface ColumnValueEncoder { + + /** + * append a column value to the array + */ + void appendValue(byte[] bytes, int offset, int length); + + /** + * append a value that is not present to the array (used to support DEFAULT expressions) + */ + void appendAbsentValue(); + + /** + * @return the encoded byte[] that contains the serialized column values + */ + byte[] encode(); + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/1f3f7323/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java index 7168c5e..3168b95 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java @@ -292,8 +292,8 @@ public class DelegateTable implements PTable { } @Override - public StorageScheme getStorageScheme() { - return delegate.getStorageScheme(); + public ImmutableStorageScheme getImmutableStorageScheme() { + return delegate.getImmutableStorageScheme(); } @Override
