http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index fde403c..8595eda 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -368,6 +368,10 @@ public enum SQLExceptionCode { CANNOT_ALTER_TABLE_PROPERTY_ON_VIEW(1134, "XCL34", "Altering this table property on a view is not allowed"), IMMUTABLE_TABLE_PROPERTY_INVALID(1135, "XCL35", "IMMUTABLE table property cannot be used with CREATE IMMUTABLE TABLE statement "), + + MAX_COLUMNS_EXCEEDED(1136, "XCL36", "The number of columns exceed the maximum supported by the table's qualifier encoding scheme"), + INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES(1137, "XCL37", "If IMMUTABLE_STORAGE_SCHEME property is not set to ONE_CELL_PER_COLUMN COLUMN_ENCODED_BYTES cannot be 0"), + INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE(1138, "XCL38", "IMMUTABLE_STORAGE_SCHEME property cannot be changed from/to ONE_CELL_PER_COLUMN "), /** * Implementation defined class. Phoenix internal error. (errorcode 20, sqlstate INT).
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java index 352b533..7b0451a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java @@ -64,11 +64,13 @@ import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.trace.TracingIterator; import org.apache.phoenix.trace.util.Tracing; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.LogUtil; import org.apache.phoenix.util.SQLCloseable; @@ -307,10 +309,6 @@ public abstract class BaseQueryPlan implements QueryPlan { // project is not present in the index then we need to skip this plan. if (!dataColumns.isEmpty()) { // Set data columns to be join back from data table. - serializeDataTableColumnsToJoin(scan, dataColumns); - KeyValueSchema schema = ProjectedColumnExpression.buildSchema(dataColumns); - // Set key value schema of the data columns. - serializeSchemaIntoScan(scan, schema); PTable parentTable = context.getCurrentTable().getTable(); String parentSchemaName = parentTable.getParentSchemaName().getString(); String parentTableName = parentTable.getParentTableName().getString(); @@ -321,6 +319,12 @@ public abstract class BaseQueryPlan implements QueryPlan { FACTORY.namedTable(null, TableName.create(parentSchemaName, parentTableName)), context.getConnection()).resolveTable(parentSchemaName, parentTableName); PTable dataTable = dataTableRef.getTable(); + // Set data columns to be join back from data table. + serializeDataTableColumnsToJoin(scan, dataColumns, dataTable); + KeyValueSchema schema = ProjectedColumnExpression.buildSchema(dataColumns); + // Set key value schema of the data columns. + serializeSchemaIntoScan(scan, schema); + // Set index maintainer of the local index. serializeIndexMaintainerIntoScan(scan, dataTable); // Set view constants if exists. @@ -367,7 +371,7 @@ public abstract class BaseQueryPlan implements QueryPlan { } ImmutableBytesWritable ptr = new ImmutableBytesWritable(); IndexMaintainer.serialize(dataTable, ptr, indexes, context.getConnection()); - scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD, ByteUtil.copyKeyBytesIfNecessary(ptr)); + scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO, ByteUtil.copyKeyBytesIfNecessary(ptr)); if (dataTable.isTransactional()) { scan.setAttribute(BaseScannerRegionObserver.TX_STATE, context.getConnection().getMutationState().encodeTransaction()); } @@ -423,14 +427,21 @@ public abstract class BaseQueryPlan implements QueryPlan { } } - private void serializeDataTableColumnsToJoin(Scan scan, Set<PColumn> dataColumns) { + private void serializeDataTableColumnsToJoin(Scan scan, Set<PColumn> dataColumns, PTable dataTable) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); try { DataOutputStream output = new DataOutputStream(stream); + boolean storeColsInSingleCell = dataTable.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS; + if (storeColsInSingleCell) { + // if storeColsInSingleCell is true all columns of a given column family are stored in a single cell + scan.setAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL, QueryConstants.EMPTY_COLUMN_VALUE_BYTES); + } WritableUtils.writeVInt(output, dataColumns.size()); for (PColumn column : dataColumns) { - Bytes.writeByteArray(output, column.getFamilyName().getBytes()); - Bytes.writeByteArray(output, column.getName().getBytes()); + byte[] cf = column.getFamilyName().getBytes(); + byte[] cq = column.getColumnQualifierBytes(); + Bytes.writeByteArray(output, cf); + Bytes.writeByteArray(output, cq); } scan.setAttribute(BaseScannerRegionObserver.DATA_TABLE_COLUMNS_TO_JOIN, stream.toByteArray()); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index 6889ca4..a04725c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -588,7 +588,7 @@ public class MutationState implements SQLCloseable { List<Mutation> indexMutations; try { indexMutations = - IndexUtil.generateIndexData(table, index, mutationsPertainingToIndex, + IndexUtil.generateIndexData(table, index, values, mutationsPertainingToIndex, connection.getKeyValueBuilder(), connection); // we may also have to include delete mutations for immutable tables if we are not processing all the tables in the mutations map if (!sendAll) { @@ -622,6 +622,7 @@ public class MutationState implements SQLCloseable { Iterator<Map.Entry<ImmutableBytesPtr, RowMutationState>> iterator = values.entrySet().iterator(); long timestampToUse = timestamp; + Map<ImmutableBytesPtr, RowMutationState> modifiedValues = Maps.newHashMap(); while (iterator.hasNext()) { Map.Entry<ImmutableBytesPtr, RowMutationState> rowEntry = iterator.next(); byte[] onDupKeyBytes = rowEntry.getValue().getOnDupKeyBytes(); @@ -631,6 +632,10 @@ public class MutationState implements SQLCloseable { if (tableWithRowTimestampCol) { RowTimestampColInfo rowTsColInfo = state.getRowTimestampColInfo(); if (rowTsColInfo.useServerTimestamp()) { + // since we are about to modify the byte[] stored in key (which changes its hashcode) + // we need to remove the entry from the values map and add a new entry with the modified byte[] + modifiedValues.put(key, state); + iterator.remove(); // regenerate the key with this timestamp. key = getNewRowKeyWithRowTimestamp(key, timestampToUse, table); } else { @@ -671,6 +676,7 @@ public class MutationState implements SQLCloseable { if (mutationsPertainingToIndex != null) mutationsPertainingToIndex .addAll(rowMutationsPertainingToIndex); } + values.putAll(modifiedValues); } /** @@ -808,7 +814,7 @@ public class MutationState implements SQLCloseable { } for (PColumn column : columns) { if (column != null) { - resolvedTable.getColumnFamily(column.getFamilyName().getString()).getColumn(column.getName().getString()); + resolvedTable.getColumnFamily(column.getFamilyName().getString()).getPColumnForColumnName(column.getName().getString()); } } } @@ -1228,7 +1234,7 @@ public class MutationState implements SQLCloseable { } mutation.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue); if (attribValue != null) { - mutation.setAttribute(PhoenixIndexCodec.INDEX_MD, attribValue); + mutation.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, attribValue); if (txState.length > 0) { mutation.setAttribute(BaseScannerRegionObserver.TX_STATE, txState); } @@ -1523,8 +1529,8 @@ public class MutationState implements SQLCloseable { byte[] getOnDupKeyBytes() { return onDupKeyBytes; } - - Map<PColumn, byte[]> getColumnValues() { + + public Map<PColumn, byte[]> getColumnValues() { return columnValues; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java index 592b68e..127af96 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.execute; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_QUALIFIER; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -51,9 +54,6 @@ import org.apache.phoenix.util.SchemaUtil; import com.google.common.base.Preconditions; public class TupleProjector { - public static final byte[] VALUE_COLUMN_FAMILY = Bytes.toBytes("_v"); - public static final byte[] VALUE_COLUMN_QUALIFIER = new byte[0]; - private static final String SCAN_PROJECTOR = "scanProjector"; private final KeyValueSchema schema; http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java index bda1b96..51cb67e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java @@ -33,6 +33,7 @@ import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PArrayDataType; +import org.apache.phoenix.schema.types.PArrayDataTypeDecoder; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PInteger; @@ -138,7 +139,7 @@ public class UnnestArrayPlan extends DelegateQueryPlan { @Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { ptr.set(arrayPtr.get(), arrayPtr.getOffset(), arrayPtr.getLength()); - PArrayDataType.positionAtArrayElement(ptr, index++, getDataType(), getMaxLength()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, index++, getDataType(), getMaxLength()); return true; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java index c2f4dd2..8b83bf7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java @@ -21,6 +21,7 @@ import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PArrayDataType; +import org.apache.phoenix.schema.types.PArrayDataTypeEncoder; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.TrustedByteArrayOutputStream; @@ -31,13 +32,9 @@ import org.apache.phoenix.util.TrustedByteArrayOutputStream; public class ArrayConstructorExpression extends BaseCompoundExpression { private PDataType baseType; private int position = -1; - private int nNulls = 0; private Object[] elements; private final ImmutableBytesWritable valuePtr = new ImmutableBytesWritable(); private int estimatedSize = 0; - // store the offset postion in this. Later based on the total size move this to a byte[] - // and serialize into byte stream - private int[] offsetPos; private boolean rowKeyOrderOptimizable; public ArrayConstructorExpression() { @@ -58,9 +55,6 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { elements = new Object[getChildren().size()]; valuePtr.set(ByteUtil.EMPTY_BYTE_ARRAY); estimatedSize = PArrayDataType.estimateSize(this.children.size(), this.baseType); - if (!this.baseType.isFixedWidth()) { - offsetPos = new int[children.size()]; - } } @Override @@ -72,7 +66,6 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { public void reset() { super.reset(); position = 0; - nNulls = 0; Arrays.fill(elements, null); valuePtr.set(ByteUtil.EMPTY_BYTE_ARRAY); } @@ -85,66 +78,24 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { } TrustedByteArrayOutputStream byteStream = new TrustedByteArrayOutputStream(estimatedSize); DataOutputStream oStream = new DataOutputStream(byteStream); - try { - int noOfElements = children.size(); - nNulls = 0; - for (int i = position >= 0 ? position : 0; i < elements.length; i++) { - Expression child = children.get(i); - if (!child.evaluate(tuple, ptr)) { - if (tuple != null && !tuple.isImmutable()) { - if (position >= 0) position = i; - return false; - } - } else { - // track the offset position here from the size of the byteStream - if (!baseType.isFixedWidth()) { - // Any variable length array would follow the below order - // Every element would be seperated by a seperator byte '0' - // Null elements are counted and once a first non null element appears we - // write the count of the nulls prefixed with a seperator byte - // Trailing nulls are not taken into account - // The last non null element is followed by two seperator bytes - // For eg - // a, b, null, null, c, null would be - // 65 0 66 0 0 2 67 0 0 0 - // a null null null b c null d would be - // 65 0 0 3 66 0 67 0 0 1 68 0 0 0 - if (ptr.getLength() == 0) { - offsetPos[i] = byteStream.size(); - nNulls++; - } else { - PArrayDataType.serializeNulls(oStream, nNulls); - offsetPos[i] = byteStream.size(); - oStream.write(ptr.get(), ptr.getOffset(), ptr.getLength()); - oStream.write(PArrayDataType.getSeparatorByte(rowKeyOrderOptimizable, getSortOrder())); - nNulls=0; - } - } else { // No nulls for fixed length - oStream.write(ptr.get(), ptr.getOffset(), ptr.getLength()); - } + PArrayDataTypeEncoder builder = + new PArrayDataTypeEncoder(byteStream, oStream, children.size(), baseType, getSortOrder(), rowKeyOrderOptimizable, PArrayDataType.SORTABLE_SERIALIZATION_VERSION); + for (int i = position >= 0 ? position : 0; i < elements.length; i++) { + Expression child = children.get(i); + if (!child.evaluate(tuple, ptr)) { + if (tuple != null && !tuple.isImmutable()) { + if (position >= 0) position = i; + return false; } - } - if (position >= 0) position = elements.length; - if (!baseType.isFixedWidth()) { - // Double seperator byte to show end of the non null array - PArrayDataType.writeEndSeperatorForVarLengthArray(oStream, getSortOrder(), rowKeyOrderOptimizable); - noOfElements = PArrayDataType.serailizeOffsetArrayIntoStream(oStream, byteStream, noOfElements, - offsetPos[offsetPos.length - 1], offsetPos); - PArrayDataType.serializeHeaderInfoIntoStream(oStream, noOfElements); - } - ptr.set(byteStream.getBuffer(), 0, byteStream.size()); - valuePtr.set(ptr.get(), ptr.getOffset(), ptr.getLength()); - return true; - } catch (IOException e) { - throw new RuntimeException("Exception while serializing the byte array"); - } finally { - try { - byteStream.close(); - oStream.close(); - } catch (IOException e) { - // Should not happen + } else { + builder.appendValue(ptr.get(), ptr.getOffset(), ptr.getLength()); } } + if (position >= 0) position = elements.length; + byte[] bytes = builder.encode(); + ptr.set(bytes, 0, bytes.length); + valuePtr.set(ptr.get(), ptr.getOffset(), ptr.getLength()); + return true; } @@ -196,4 +147,5 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { buf.append(children.get(children.size()-1) + "]"); return buf.toString(); } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java index 658605e..7f95da6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java @@ -178,9 +178,8 @@ public enum ExpressionType { CeilYearExpression(CeilYearExpression.class), DayOfWeekFunction(DayOfWeekFunction.class), DayOfYearFunction(DayOfYearFunction.class), - DefaultValueExpression(DefaultValueExpression.class); - - + DefaultValueExpression(DefaultValueExpression.class), + ArrayColumnExpression(SingleCellColumnExpression.class); ExpressionType(Class<? extends Expression> clazz) { this.clazz = clazz; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java index 4b5fdbb..5c0be57 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java @@ -41,33 +41,38 @@ import org.apache.phoenix.util.SchemaUtil; public class KeyValueColumnExpression extends ColumnExpression { private byte[] cf; private byte[] cq; - private String displayName; // client-side only + private String displayName; // client-side only. public KeyValueColumnExpression() { } - + public KeyValueColumnExpression(PColumn column) { - this(column, null); - } - - public KeyValueColumnExpression(PDatum column, byte[] cf, byte[] cq) { super(column); - this.cf = cf; - this.cq = cq; + this.cf = column.getFamilyName().getBytes(); + // for backward compatibility since older tables won't have columnQualifierBytes in their metadata + this.cq = column.getColumnQualifierBytes() != null ? column.getColumnQualifierBytes() : column.getName().getBytes(); + this.displayName = column.getName().getString(); } - + public KeyValueColumnExpression(PColumn column, String displayName) { super(column); this.cf = column.getFamilyName().getBytes(); - this.cq = column.getName().getBytes(); + // for backward compatibility since older tables won't have columnQualifierBytes in their metadata + this.cq = column.getColumnQualifierBytes() != null ? column.getColumnQualifierBytes() : column.getName().getBytes(); this.displayName = displayName; } + public KeyValueColumnExpression(PDatum column, byte[] cf, byte[] cq) { + super(column); + this.cf = cf; + this.cq = cq; + } + public byte[] getColumnFamily() { return cf; } - - public byte[] getColumnName() { + + public byte[] getColumnQualifier() { return cq; } @@ -120,7 +125,11 @@ public class KeyValueColumnExpression extends ColumnExpression { } @Override - public final <T> T accept(ExpressionVisitor<T> visitor) { + public <T> T accept(ExpressionVisitor<T> visitor) { return visitor.visit(this); } + +// public void setDisplayName(String displayName) { +// this.displayName = displayName; +// } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java index 90882a2..f20d7e2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.sql.SQLException; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.IllegalDataException; @@ -214,6 +215,11 @@ public class LiteralExpression extends BaseTerminalExpression { public LiteralExpression() { } + + public LiteralExpression(byte[] byteValue) { + this.byteValue = byteValue!=null ? byteValue : ByteUtil.EMPTY_BYTE_ARRAY; + this.determinism = Determinism.ALWAYS; + } private LiteralExpression(PDataType type, Determinism determinism) { this(null, type, ByteUtil.EMPTY_BYTE_ARRAY, determinism); @@ -242,7 +248,10 @@ public class LiteralExpression extends BaseTerminalExpression { @Override public String toString() { - if (value == null) { + if (value == null && byteValue!=null) { + return Bytes.toStringBinary(byteValue); + } + else if (value == null) { return "null"; } // TODO: move into PDataType? http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java index 3a38dee..2744f35 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java @@ -154,6 +154,7 @@ public class ProjectedColumnExpression extends ColumnExpression { return Determinism.PER_INVOCATION; } + @Override public ProjectedColumnExpression clone() { return new ProjectedColumnExpression(this.column, this.columns, this.position, this.displayName); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java new file mode 100644 index 0000000..8c1e0b6 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression; + +import static org.apache.phoenix.query.QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.compile.CreateTableCompiler.ViewWhereExpressionVisitor; +import org.apache.phoenix.expression.visitor.ExpressionVisitor; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.ColumnValueDecoder; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PDatum; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.util.SchemaUtil; + +import com.google.common.base.Preconditions; + +/** + * + * Class to access a column that is stored in a Cell that contains all + * columns for a given column family (stored in a serialized array). + * + */ +public class SingleCellColumnExpression extends KeyValueColumnExpression { + + private int decodedColumnQualifier; + private String arrayColDisplayName; + private KeyValueColumnExpression keyValueColumnExpression; + private QualifierEncodingScheme encodingScheme; + + public SingleCellColumnExpression() { + } + + public SingleCellColumnExpression(PDatum column, byte[] cf, byte[] cq, QualifierEncodingScheme encodingScheme) { + super(column, cf, SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + Preconditions.checkNotNull(encodingScheme); + Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); + this.decodedColumnQualifier = encodingScheme.decode(cq); + this.encodingScheme = encodingScheme; + setKeyValueExpression(); + } + + public SingleCellColumnExpression(PColumn column, String displayName, QualifierEncodingScheme encodingScheme) { + super(column, column.getFamilyName().getBytes(), SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + Preconditions.checkNotNull(encodingScheme); + Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); + this.arrayColDisplayName = displayName; + this.decodedColumnQualifier = encodingScheme.decode(column.getColumnQualifierBytes()); + this.encodingScheme = encodingScheme; + setKeyValueExpression(); + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + if (!super.evaluate(tuple, ptr)) { + return false; + } else if (ptr.getLength() == 0) { + return true; + } + // the first position is reserved and we offset maxEncodedColumnQualifier by ENCODED_CQ_COUNTER_INITIAL_VALUE (which is the minimum encoded column qualifier) + int index = decodedColumnQualifier-QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE+1; + byte serializedImmutableStorageScheme = ptr.get()[ptr.getOffset() + ptr.getLength() - Bytes.SIZEOF_BYTE]; + ImmutableStorageScheme immutableStorageScheme = ImmutableStorageScheme.fromSerializedValue(serializedImmutableStorageScheme); + // Given a ptr to the entire array, set ptr to point to a particular element within that array + ColumnValueDecoder encoderDecoder = immutableStorageScheme.getDecoder(); + return encoderDecoder.decode(ptr, index); + } + + @Override + public void readFields(DataInput input) throws IOException { + super.readFields(input); + this.decodedColumnQualifier = WritableUtils.readVInt(input); + this.encodingScheme = QualifierEncodingScheme.values()[WritableUtils.readVInt(input)]; + setKeyValueExpression(); + } + + @Override + public void write(DataOutput output) throws IOException { + super.write(output); + WritableUtils.writeVInt(output, decodedColumnQualifier); + WritableUtils.writeVInt(output, encodingScheme.ordinal()); + } + + public KeyValueColumnExpression getKeyValueExpression() { + return keyValueColumnExpression; + } + + private void setKeyValueExpression() { + final boolean isNullable = isNullable(); + final SortOrder sortOrder = getSortOrder(); + final Integer scale = getScale(); + final Integer maxLength = getMaxLength(); + final PDataType datatype = getDataType(); + this.keyValueColumnExpression = new KeyValueColumnExpression(new PDatum() { + @Override + public boolean isNullable() { + return isNullable; + } + + @Override + public SortOrder getSortOrder() { + return sortOrder; + } + + @Override + public Integer getScale() { + return scale; + } + + @Override + public Integer getMaxLength() { + return maxLength; + } + + @Override + public PDataType getDataType() { + return datatype; + } + }, getColumnFamily(), getPositionInArray()); + } + + @Override + public String toString() { + if (arrayColDisplayName == null) { + arrayColDisplayName = SchemaUtil.getColumnDisplayName(getColumnFamily(), getColumnQualifier()); + } + return arrayColDisplayName; + } + + public byte[] getPositionInArray() { + return encodingScheme.encode(decodedColumnQualifier); + } + + @Override + public <T> T accept(ExpressionVisitor<T> visitor) { + //FIXME: this is ugly but can't think of a good solution. + if (visitor instanceof ViewWhereExpressionVisitor) { + return visitor.visit(this); + } else { + return super.accept(visitor); + } + } + + @Override + public boolean equals(Object obj) { + if (obj.getClass() != SingleCellColumnExpression.class) return false; + return keyValueColumnExpression.equals(((SingleCellColumnExpression)obj).getKeyValueExpression()); + } + + @Override + public int hashCode() { + return keyValueColumnExpression.hashCode(); + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java new file mode 100644 index 0000000..48485be --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by + * applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.apache.phoenix.expression; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.expression.visitor.ExpressionVisitor; +import org.apache.phoenix.schema.ColumnValueEncoder; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarbinary; + +/** + * Expression used to create a single cell containing all the column values for a column family + */ +public class SingleCellConstructorExpression extends BaseCompoundExpression { + + private ImmutableStorageScheme immutableStorageScheme; + + public SingleCellConstructorExpression(ImmutableStorageScheme immutableStorageScheme, List<Expression> children) { + super(children); + this.immutableStorageScheme = immutableStorageScheme; + } + + @Override + public PDataType getDataType() { + return PVarbinary.INSTANCE; + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + ColumnValueEncoder encoderDecoder = immutableStorageScheme.getEncoder(children.size()); + for (int i=0; i < children.size(); i++) { + Expression child = children.get(i); + if (!child.evaluate(tuple, ptr)) { + encoderDecoder.appendAbsentValue(); + } else { + encoderDecoder.appendValue(ptr.get(), ptr.getOffset(), ptr.getLength()); + } + } + byte[] bytes = encoderDecoder.encode(); + ptr.set(bytes, 0, bytes.length); + return true; + } + + + @Override + public void readFields(DataInput input) throws IOException { + super.readFields(input); + this.immutableStorageScheme = WritableUtils.readEnum(input, ImmutableStorageScheme.class); + } + + @Override + public void write(DataOutput output) throws IOException { + super.write(output); + WritableUtils.writeEnum(output, immutableStorageScheme); + } + + @Override + public boolean requiresFinalEvaluation() { + return true; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder("["); + if (children.size()==0) + return buf.append("]").toString(); + for (int i = 0; i < children.size() - 1; i++) { + buf.append(children.get(i) + ","); + } + buf.append(children.get(children.size()-1) + "]"); + return buf.toString(); + } + + @Override + public final <T> T accept(ExpressionVisitor<T> visitor) { + List<T> l = acceptChildren(visitor, visitor.visitEnter(this)); + T t = visitor.visitLeave(this, l); + if (t == null) { + t = visitor.defaultReturn(this, l); + } + return t; + } + + public SingleCellConstructorExpression clone(List<Expression> children) { + return new SingleCellConstructorExpression(immutableStorageScheme, children); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java index 6631e70..06bbced 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java @@ -27,7 +27,7 @@ import org.apache.phoenix.expression.BaseCompoundExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.tuple.Tuple; -import org.apache.phoenix.schema.types.PArrayDataType; +import org.apache.phoenix.schema.types.PArrayDataTypeDecoder; import org.apache.phoenix.schema.types.PDataType; public class ArrayElemRefExpression extends BaseCompoundExpression { @@ -48,7 +48,7 @@ public class ArrayElemRefExpression extends BaseCompoundExpression { @Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { Expression arrayExpr = children.get(0); - return PArrayDataType.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), getMaxLength()); + return PArrayDataTypeDecoder.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), getMaxLength()); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java index 7a23ef5..0f3c40c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java @@ -24,9 +24,9 @@ import org.apache.phoenix.expression.Expression; import org.apache.phoenix.parse.FunctionParseNode.Argument; import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction; import org.apache.phoenix.parse.ParseException; +import org.apache.phoenix.schema.types.PArrayDataTypeDecoder; import org.apache.phoenix.schema.types.PBinaryArray; import org.apache.phoenix.schema.types.PInteger; -import org.apache.phoenix.schema.types.PArrayDataType; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PVarbinaryArray; import org.apache.phoenix.schema.SortOrder; @@ -61,7 +61,7 @@ public class ArrayIndexFunction extends ScalarFunction { throw new ParseException("Index cannot be negative :" + index); } Expression arrayExpr = children.get(0); - return PArrayDataType.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), + return PArrayDataTypeDecoder.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), getMaxLength()); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java index af5bc2b..f2ed97b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.schema.SortOrder; -import org.apache.phoenix.schema.types.PArrayDataType.PArrayDataTypeBytesArrayBuilder; +import org.apache.phoenix.schema.types.PArrayDataTypeEncoder; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; @@ -159,8 +159,8 @@ public class JONIPattern extends AbstractBasePattern implements AbstractBaseSpli private boolean split(byte[] srcBytes, int srcOffset, int srcLen, ImmutableBytesWritable outPtr) { SortOrder sortOrder = SortOrder.ASC; - PArrayDataTypeBytesArrayBuilder builder = - new PArrayDataTypeBytesArrayBuilder(PVarchar.INSTANCE, sortOrder); + PArrayDataTypeEncoder builder = + new PArrayDataTypeEncoder(PVarchar.INSTANCE, sortOrder); int srcRange = srcOffset + srcLen; Matcher matcher = pattern.matcher(srcBytes, 0, srcRange); int cur = srcOffset; @@ -168,31 +168,29 @@ public class JONIPattern extends AbstractBasePattern implements AbstractBaseSpli while (true) { int nextCur = matcher.search(cur, srcRange, Option.DEFAULT); if (nextCur < 0) { - append = builder.appendElem(srcBytes, cur, srcRange - cur); - if (!append) return false; + builder.appendValue(srcBytes, cur, srcRange - cur); break; } // To handle the following case, which adds null at first. // REGEXP_SPLIT("12ONE34TWO56THREE78","[0-9]+")={null, "ONE", "TWO", "THREE", null} if (cur == matcher.getBegin()) { - builder.appendElem(srcBytes, cur, 0); + builder.appendValue(srcBytes, cur, 0); } if (cur < matcher.getBegin()) { - append = builder.appendElem(srcBytes, cur, matcher.getBegin() - cur); - if (!append) return false; + builder.appendValue(srcBytes, cur, matcher.getBegin() - cur); } cur = matcher.getEnd(); // To handle the following case, which adds null at last. // REGEXP_SPLIT("12ONE34TWO56THREE78","[0-9]+")={null, "ONE", "TWO", "THREE", null} if (cur == srcRange) { - builder.appendElem(srcBytes, cur, 0); + builder.appendValue(srcBytes, cur, 0); break; } } - byte[] bytes = builder.getBytesAndClose(SortOrder.ASC); + byte[] bytes = builder.encode(); if (bytes == null) return false; outPtr.set(bytes); return true; http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java index 8e8b32d..d79b546 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java @@ -36,6 +36,7 @@ import org.apache.phoenix.expression.MultiplyExpression; import org.apache.phoenix.expression.NotExpression; import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -142,6 +143,11 @@ public abstract class BaseExpressionVisitor<E> implements ExpressionVisitor<E> { } @Override + public Iterator<Expression> visitEnter(SingleCellConstructorExpression node) { + return null; + } + + @Override public Iterator<Expression> visitEnter(ModulusExpression modulusExpression) { return null; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java index 00ece40..e47fb64 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java @@ -41,6 +41,8 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -80,6 +82,11 @@ public abstract class CloneExpressionVisitor extends TraverseAllExpressionVisito public Expression visit(KeyValueColumnExpression node) { return node; } + + @Override + public Expression visit(SingleCellColumnExpression node) { + return node; + } @Override public Expression visit(ProjectedColumnExpression node) { @@ -182,6 +189,11 @@ public abstract class CloneExpressionVisitor extends TraverseAllExpressionVisito public Expression visitLeave(ArrayConstructorExpression node, List<Expression> l) { return isCloneNode(node, l) ? node.clone(l) : node; } + + @Override + public Expression visitLeave(SingleCellConstructorExpression node, List<Expression> l) { + return isCloneNode(node, l) ? node.clone(l) : node; + } @Override public Expression visitLeave(StringConcatExpression node, List<Expression> l) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java index 1aeb9a9..9a56e36 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java @@ -24,6 +24,7 @@ import org.apache.phoenix.expression.Expression; public class CloneNonDeterministicExpressionVisitor extends CloneExpressionVisitor { + @Override public boolean isCloneNode(Expression node, List<Expression> children) { return Determinism.PER_INVOCATION.compareTo(node.getDeterminism()) <= 0; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java index 31f340d..5936dc7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.phoenix.compile.SequenceValueExpression; import org.apache.phoenix.expression.AddExpression; import org.apache.phoenix.expression.AndExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.expression.ArrayConstructorExpression; import org.apache.phoenix.expression.CaseExpression; import org.apache.phoenix.expression.CoerceExpression; @@ -42,6 +43,7 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -109,10 +111,14 @@ public interface ExpressionVisitor<E> { public Iterator<Expression> visitEnter(ArrayConstructorExpression node); public E visitLeave(ArrayConstructorExpression node, List<E> l); + public Iterator<Expression> visitEnter(SingleCellConstructorExpression node); + public E visitLeave(SingleCellConstructorExpression node, List<E> l); + public E visit(CorrelateVariableFieldAccessExpression node); public E visit(LiteralExpression node); public E visit(RowKeyColumnExpression node); public E visit(KeyValueColumnExpression node); + public E visit(SingleCellColumnExpression node); public E visit(ProjectedColumnExpression node); public E visit(SequenceValueExpression node); http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java index 3b7067a..f5615be 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java @@ -28,7 +28,6 @@ import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.ComparisonExpression; import org.apache.phoenix.expression.CorrelateVariableFieldAccessExpression; import org.apache.phoenix.expression.DivideExpression; -import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.InListExpression; import org.apache.phoenix.expression.IsNullExpression; import org.apache.phoenix.expression.KeyValueColumnExpression; @@ -41,6 +40,8 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -121,6 +122,11 @@ public class StatelessTraverseAllExpressionVisitor<E> extends TraverseAllExpress } @Override + public E visit(SingleCellColumnExpression node) { + return null; + } + + @Override public E visit(ProjectedColumnExpression node) { return null; } @@ -164,6 +170,11 @@ public class StatelessTraverseAllExpressionVisitor<E> extends TraverseAllExpress public E visitLeave(ArrayConstructorExpression node, List<E> l) { return null; } + + @Override + public E visitLeave(SingleCellConstructorExpression node, List<E> l) { + return null; + } @Override public E visitLeave(ModulusExpression node, List<E> l) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java index 83b28bd..7f447b3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java @@ -28,7 +28,6 @@ import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.ComparisonExpression; import org.apache.phoenix.expression.CorrelateVariableFieldAccessExpression; import org.apache.phoenix.expression.DivideExpression; -import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.InListExpression; import org.apache.phoenix.expression.IsNullExpression; import org.apache.phoenix.expression.KeyValueColumnExpression; @@ -41,6 +40,8 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -114,6 +115,11 @@ public class StatelessTraverseNoExpressionVisitor<E> extends TraverseNoExpressio public E visit(RowKeyColumnExpression node) { return null; } + + @Override + public E visit(SingleCellColumnExpression node) { + return null; + } @Override public E visit(KeyValueColumnExpression node) { @@ -164,6 +170,11 @@ public class StatelessTraverseNoExpressionVisitor<E> extends TraverseNoExpressio public E visitLeave(ArrayConstructorExpression node, List<E> l) { return null; } + + @Override + public E visitLeave(SingleCellConstructorExpression node, List<E> l) { + return null; + } @Override public E visitLeave(ModulusExpression node, List<E> l) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java index 92e5c20..3d6843d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; -import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.util.EncodedColumnsUtil; /** * When selecting specific columns in a SELECT query, this filter passes only selected columns @@ -54,6 +54,8 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { private byte[] emptyCFName; private Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker; private Set<byte[]> conditionOnlyCfs; + private boolean usesEncodedColumnNames; + private byte[] emptyKVQualifier; public ColumnProjectionFilter() { @@ -61,10 +63,12 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { public ColumnProjectionFilter(byte[] emptyCFName, Map<ImmutableBytesPtr, NavigableSet<ImmutableBytesPtr>> columnsTracker, - Set<byte[]> conditionOnlyCfs) { + Set<byte[]> conditionOnlyCfs, boolean usesEncodedColumnNames) { this.emptyCFName = emptyCFName; this.columnsTracker = columnsTracker; this.conditionOnlyCfs = conditionOnlyCfs; + this.usesEncodedColumnNames = usesEncodedColumnNames; + this.emptyKVQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); } @Override @@ -88,6 +92,9 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { familyMapSize--; } int conditionOnlyCfsSize = WritableUtils.readVInt(input); + usesEncodedColumnNames = conditionOnlyCfsSize > 0; + emptyKVQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); + conditionOnlyCfsSize = Math.abs(conditionOnlyCfsSize) - 1; // restore to the actual value. this.conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); while (conditionOnlyCfsSize > 0) { this.conditionOnlyCfs.add(WritableUtils.readCompressedByteArray(input)); @@ -111,12 +118,13 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { } } } - // Write conditionOnlyCfs - WritableUtils.writeVInt(output, this.conditionOnlyCfs.size()); + // Encode usesEncodedColumnNames in conditionOnlyCfs size. + WritableUtils.writeVInt(output, (this.conditionOnlyCfs.size() + 1) * (usesEncodedColumnNames ? 1 : -1)); for (byte[] f : this.conditionOnlyCfs) { WritableUtils.writeCompressedByteArray(output, f); } - } + +} @Override public byte[] toByteArray() throws IOException { @@ -156,9 +164,9 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { // make sure we're not holding to any of the byte[]'s ptr.set(HConstants.EMPTY_BYTE_ARRAY); if (kvs.isEmpty()) { - kvs.add(new KeyValue(firstKV.getRowArray(), firstKV.getRowOffset(),firstKV.getRowLength(), this.emptyCFName, - 0, this.emptyCFName.length, QueryConstants.EMPTY_COLUMN_BYTES, 0, - QueryConstants.EMPTY_COLUMN_BYTES.length, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0)); + kvs.add(new KeyValue(firstKV.getRowArray(), firstKV.getRowOffset(), firstKV.getRowLength(), + this.emptyCFName, 0, this.emptyCFName.length, emptyKVQualifier, 0, + emptyKVQualifier.length, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0)); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java index dba700b..88f707d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiKeyValueComparisonFilter.java @@ -26,6 +26,7 @@ import java.util.TreeSet; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.visitor.ExpressionVisitor; @@ -94,7 +95,7 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil refCount = foundColumns.size(); } - public ReturnCode resolveColumn(Cell value) { + private ReturnCode resolveColumn(Cell value) { // Always set key, in case we never find a key value column of interest, // and our expression uses row key columns. setKey(value); @@ -184,7 +185,7 @@ public abstract class MultiKeyValueComparisonFilter extends BooleanExpressionFil ExpressionVisitor<Void> visitor = new StatelessTraverseAllExpressionVisitor<Void>() { @Override public Void visit(KeyValueColumnExpression expression) { - inputTuple.addColumn(expression.getColumnFamily(), expression.getColumnName()); + inputTuple.addColumn(expression.getColumnFamily(), expression.getColumnQualifier()); return null; } }; http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java index 0d904bc..195c89c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleCQKeyValueComparisonFilter.java @@ -47,7 +47,8 @@ public class SingleCQKeyValueComparisonFilter extends SingleKeyValueComparisonFi public static SingleCQKeyValueComparisonFilter parseFrom(final byte [] pbBytes) throws DeserializationException { try { - return (SingleCQKeyValueComparisonFilter)Writables.getWritable(pbBytes, new SingleCQKeyValueComparisonFilter()); + SingleCQKeyValueComparisonFilter writable = (SingleCQKeyValueComparisonFilter)Writables.getWritable(pbBytes, new SingleCQKeyValueComparisonFilter()); + return writable; } catch (IOException e) { throw new DeserializationException(e); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java index eaf8d35..ae3557d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/SingleKeyValueComparisonFilter.java @@ -22,11 +22,13 @@ import java.io.IOException; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.visitor.StatelessTraverseAllExpressionVisitor; import org.apache.phoenix.expression.visitor.TraverseAllExpressionVisitor; import org.apache.phoenix.schema.tuple.SingleKeyValueTuple; +import org.apache.phoenix.schema.tuple.Tuple; @@ -58,7 +60,7 @@ public abstract class SingleKeyValueComparisonFilter extends BooleanExpressionFi @Override public Void visit(KeyValueColumnExpression expression) { cf = expression.getColumnFamily(); - cq = expression.getColumnName(); + cq = expression.getColumnQualifier(); return null; } }; http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ValueGetter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ValueGetter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ValueGetter.java index bcadc2b..19797cf 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ValueGetter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/ValueGetter.java @@ -35,4 +35,5 @@ public interface ValueGetter { public ImmutableBytesWritable getLatestValue(ColumnReference ref) throws IOException; public byte[] getRowKey(); + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java index 6f9caa6..0f960e4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/example/CoveredColumnIndexCodec.java @@ -32,7 +32,6 @@ import org.apache.phoenix.hbase.index.covered.TableState; import org.apache.phoenix.hbase.index.scanner.Scanner; import com.google.common.collect.Lists; -import com.google.common.collect.Lists; /** * http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/util/KeyValueBuilder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/util/KeyValueBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/util/KeyValueBuilder.java index 741bf87..56b60e9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/util/KeyValueBuilder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/util/KeyValueBuilder.java @@ -125,4 +125,5 @@ public abstract class KeyValueBuilder { public abstract KVComparator getKeyValueComparator(); public abstract List<Mutation> cloneIfNecessary(List<Mutation> mutations); + }