This is an automated email from the ASF dual-hosted git repository. richardantal pushed a commit to branch 4.x in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push: new 34f9938 PHOENIX-5865 Column that has default value can not be correctly indexed 34f9938 is described below commit 34f9938027b2f37370c87999a01f77ceda2eaf62 Author: Richard Antal <antal97rich...@gmail.com> AuthorDate: Wed Dec 1 12:28:11 2021 +0100 PHOENIX-5865 Column that has default value can not be correctly indexed --- .../phoenix/end2end/IndexWithDefaultValueIT.java | 280 +++++++++++++++++++++ .../expression/KeyValueColumnExpression.java | 9 + .../expression/SingleCellColumnExpression.java | 24 +- .../function/DefaultValueExpression.java | 12 +- .../phoenix/hbase/index/AbstractValueGetter.java | 4 +- .../org/apache/phoenix/schema/MetaDataClient.java | 7 +- .../phoenix/schema/tuple/ValueGetterTuple.java | 29 ++- 7 files changed, 348 insertions(+), 17 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexWithDefaultValueIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexWithDefaultValueIT.java new file mode 100644 index 0000000..a93f4ab --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexWithDefaultValueIT.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + + +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Properties; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertEquals; + + +public class IndexWithDefaultValueIT extends ParallelStatsDisabledIT { + + @Test + public void testQueryTableWithIndex() throws Exception { + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + + Properties props = new Properties(); + String schema = generateUniqueName(); + Connection conn = DriverManager.getConnection(getUrl(), props); + + conn.setSchema(schema); + conn.createStatement().execute("\n" + + "create table " + tableName + "(\n" + + "pk VARCHAR,\n" + + "b VARCHAR,\n" + + "c VARCHAR default '0',\n" + + "CONSTRAINT my_pk PRIMARY KEY (pk)\n" + + ")"); + + conn.commit(); + + conn.createStatement().execute("upsert into " + tableName + " values('1','1','1')"); + conn.commit(); + + conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + "(pk, b, c)"); + conn.commit(); + + + final PreparedStatement select = conn.prepareStatement( + "select * from " + tableName); + + ResultSet rs = select.executeQuery(); + + assertTrue(rs.next()); + assertEquals("1", rs.getString(3)); + assertFalse(rs.next()); + rs.close(); + conn.close(); + } + + + + @Test + public void testQueryTableWithIndexBigintDefault() throws Exception { + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + + Properties props = new Properties(); + String schema = generateUniqueName(); + Connection conn = DriverManager.getConnection(getUrl(), props); + + + conn.setSchema(schema); + conn.createStatement().execute("\n" + + "create table " + tableName + "(\n" + + "id CHAR(32) NOT NULL,\n" + + "no CHAR(32) default 'AB'," + + "total BIGINT default 0,\n" + + "score INTEGER default 0," + + "CONSTRAINT my_pk PRIMARY KEY (id)\n" + + ")"); + + conn.commit(); + + conn.createStatement().execute("upsert into " + tableName + "(id, no, total, score) values ('1111','1112', 1113, 1114)"); + conn.createStatement().execute("upsert into " + tableName + "(id, total) values ('1121', 1123)"); + conn.commit(); + + conn.createStatement().execute("CREATE INDEX " + indexName + " on " + tableName + " (no, total, score)"); + conn.commit(); + + + final PreparedStatement select = conn.prepareStatement( + "select * from " + tableName); + + ResultSet rs = select.executeQuery(); + + assertTrue(rs.next()); + assertEquals(1113L, rs.getObject(3)); + assertEquals(1114, rs.getObject(4)); + assertTrue(rs.next()); + assertEquals("AB", rs.getObject(2)); + assertEquals(1123L, rs.getObject(3)); + assertEquals(0, rs.getObject(4)); + assertFalse(rs.next()); + + rs.close(); + conn.close(); + } + + @Test + public void testQueryTableWithIndexDefaultValue() throws Exception { + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + + Properties props = new Properties(); + String schema = generateUniqueName(); + Connection conn = DriverManager.getConnection(getUrl(), props); + + + conn.setSchema(schema); + conn.createStatement().execute("\n" + + "create table " + tableName + "(\n" + + "pk1 INTEGER NOT NULL, " + + "pk2 INTEGER DEFAULT 10, " + + "CONSTRAINT my_pk PRIMARY KEY (pk1)\n" + + ")"); + + conn.commit(); + + conn.createStatement().execute("upsert into " + tableName + "(pk1, pk2) values (1,1)"); + conn.createStatement().execute("upsert into " + tableName + "(pk1, pk2) values (2, null)"); + conn.createStatement().execute("upsert into " + tableName + "(pk1) values (3)"); + conn.commit(); + + conn.createStatement().execute("CREATE INDEX " + indexName + " on " + tableName + " (pk1, pk2)"); + conn.commit(); + + + final PreparedStatement select = conn.prepareStatement( + "select * from " + tableName); + + ResultSet rs = select.executeQuery(); + + assertTrue(rs.next()); + assertEquals(1, rs.getObject(1)); + assertEquals(1, rs.getObject(2)); + assertTrue(rs.next()); + assertEquals(2, rs.getObject(1)); + assertEquals(null, rs.getObject(2)); + assertTrue(rs.next()); + assertEquals(3, rs.getObject(1)); + assertEquals(10, rs.getObject(2)); + assertFalse(rs.next()); + + rs.close(); + conn.close(); + } + + @Test + public void testDefaultLocalIndexed() throws Exception { + String table = generateUniqueName(); + String ddl = "CREATE TABLE IF NOT EXISTS " + table + " (" + + "pk INTEGER PRIMARY KEY," + + "c1 INTEGER," + + "c2 INTEGER DEFAULT 100)"; + + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute(ddl); + conn.commit(); + + String idx = generateUniqueName(); + ddl = "CREATE LOCAL INDEX " + idx + " on " + table + " (c2)"; + conn.createStatement().execute(ddl); + conn.commit(); + + String dml = "UPSERT INTO " + table + " (pk, c1) VALUES (1, 2)"; + conn.createStatement().execute(dml); + conn.commit(); + + ResultSet rs = + conn.createStatement().executeQuery("SELECT c2 FROM " + table + " WHERE c2 = 100"); + assertTrue(rs.next()); + assertEquals(100, rs.getInt(1)); + assertFalse(rs.next()); + + rs = conn.createStatement().executeQuery("SELECT c2 FROM " + table + " WHERE c2 = 5"); + assertFalse(rs.next()); + } + + @Test + public void testDefaultIndexed() throws Exception { + String table = generateUniqueName(); + String ddl = "CREATE TABLE IF NOT EXISTS " + table + " (" + + "pk INTEGER PRIMARY KEY," + + "c1 INTEGER," + + "c2 INTEGER DEFAULT 100)"; + + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute(ddl); + conn.commit(); + + String idx = generateUniqueName(); + ddl = "CREATE INDEX " + idx + " on " + table + " (c2)"; + conn.createStatement().execute(ddl); + conn.commit(); + + String dml = "UPSERT INTO " + table + " (pk, c1) VALUES (1, 2)"; + conn.createStatement().execute(dml); + conn.commit(); + + ResultSet rs = + conn.createStatement().executeQuery("SELECT c2 FROM " + table + " WHERE c2 = 100"); + assertTrue(rs.next()); + assertEquals(100, rs.getInt(1)); + assertFalse(rs.next()); + + rs = conn.createStatement().executeQuery("SELECT c2 FROM " + table + " WHERE c2 = 5"); + assertFalse(rs.next()); + } + + @Test + public void testDefaultColumnValue() throws Exception { + String sharedTable1 = generateUniqueName(); + String ddl = "CREATE TABLE IF NOT EXISTS " + sharedTable1 + " (" + + "pk1 INTEGER NOT NULL, " + + "pk2 INTEGER DEFAULT 10, " + + "CONSTRAINT NAME_PK PRIMARY KEY (pk1))"; + + Connection conn = DriverManager.getConnection(getUrl()); + conn.createStatement().execute(ddl); + + String dml = "UPSERT INTO " + sharedTable1 + " VALUES (1, 1)"; + conn.createStatement().execute(dml); + dml = "UPSERT INTO " + sharedTable1 + " VALUES (2, null)"; + conn.createStatement().execute(dml); + dml = "UPSERT INTO " + sharedTable1 + " VALUES (3)"; + conn.createStatement().execute(dml); + conn.commit(); + + + String projection = "*"; + + ResultSet rs = conn.createStatement() + .executeQuery("SELECT " + projection + " FROM " + sharedTable1 + " WHERE pk1 = 1"); + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertEquals(1, rs.getInt(2)); + assertFalse(rs.next()); + + rs = conn.createStatement() + .executeQuery("SELECT " + projection + " FROM " + sharedTable1 + " WHERE pk1 = 2"); + assertTrue(rs.next()); + assertEquals(2, rs.getInt(1)); + assertEquals(null, rs.getString(2)); + assertFalse(rs.next()); + + rs = conn.createStatement() + .executeQuery("SELECT " + projection + " FROM " + sharedTable1 + " WHERE pk1 = 3"); + assertTrue(rs.next()); + assertEquals(3, rs.getInt(1)); + assertEquals(10, rs.getInt(2)); + assertFalse(rs.next()); + } + +} \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java index 62665e4..f6a9e04 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java @@ -28,6 +28,7 @@ import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PDatum; import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.tuple.ValueGetterTuple; import org.apache.phoenix.util.SchemaUtil; @@ -115,6 +116,14 @@ public class KeyValueColumnExpression extends ColumnExpression { return tuple.getValue(cf, cq, ptr); } + public boolean evaluateUnsafe(Tuple tuple, ImmutableBytesWritable ptr) { + if (tuple instanceof ValueGetterTuple) { + return ((ValueGetterTuple) tuple).getValueUnsafe(cf, cq, ptr); + } else { + return tuple.getValue(cf, cq, ptr); + } + } + @Override public void readFields(DataInput input) throws IOException { super.readFields(input); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java index 16f08d8..2c15297 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java @@ -91,14 +91,32 @@ public class SingleCellColumnExpression extends KeyValueColumnExpression { } else if (ptr.getLength() == 0) { return true; } - // the first position is reserved and we offset maxEncodedColumnQualifier by ENCODED_CQ_COUNTER_INITIAL_VALUE (which is the minimum encoded column qualifier) - int index = decodedColumnQualifier-QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE+1; - // Given a ptr to the entire array, set ptr to point to a particular element within that array + // the first position is reserved and we offset maxEncodedColumnQualifier by + // ENCODED_CQ_COUNTER_INITIAL_VALUE (which is the minimum encoded column qualifier) + int index = decodedColumnQualifier - QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE + 1; + // Given a ptr to the entire array, set ptr to point to a particular element + // within that array ColumnValueDecoder encoderDecoder = immutableStorageScheme.getDecoder(); return encoderDecoder.decode(ptr, index); } @Override + public boolean evaluateUnsafe(Tuple tuple, ImmutableBytesWritable ptr) { + if (!super.evaluateUnsafe(tuple, ptr)) { + return false; + } else if (ptr.getLength() == 0) { + return true; + } + // the first position is reserved and we offset maxEncodedColumnQualifier by + // ENCODED_CQ_COUNTER_INITIAL_VALUE (which is the minimum encoded column qualifier) + int index = decodedColumnQualifier - QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE + 1; + // Given a ptr to the entire array, set ptr to point to a particular element + // within that array + ColumnValueDecoder encoderDecoder = immutableStorageScheme.getDecoder(); + return encoderDecoder.decode(ptr, index); + } + + @Override public void readFields(DataInput input) throws IOException { super.readFields(input); this.decodedColumnQualifier = WritableUtils.readVInt(input); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/DefaultValueExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/DefaultValueExpression.java index fceb442..bf27df4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/DefaultValueExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/DefaultValueExpression.java @@ -22,6 +22,8 @@ import java.util.List; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.KeyValueColumnExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PDataType; @@ -44,7 +46,15 @@ public class DefaultValueExpression extends ScalarFunction { @Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { - boolean evaluated = children.get(0).evaluate(tuple, ptr); + Expression firstChild = children.get(0); + boolean evaluated; + if (firstChild instanceof SingleCellColumnExpression) { + evaluated = ((SingleCellColumnExpression) firstChild).evaluateUnsafe(tuple, ptr); + } else if (firstChild instanceof KeyValueColumnExpression) { + evaluated = ((KeyValueColumnExpression) firstChild).evaluateUnsafe(tuple, ptr); + } else { + evaluated = children.get(0).evaluate(tuple, ptr); + } if (evaluated) { // Will potentially evaluate to null without evaluating the second expression return true; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/AbstractValueGetter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/AbstractValueGetter.java index 08d19d0..90f9094 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/AbstractValueGetter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/AbstractValueGetter.java @@ -33,7 +33,9 @@ public abstract class AbstractValueGetter implements ValueGetter{ int valueOffset = 0; int valueLength = 0; byte[] valueBytes = HConstants.EMPTY_BYTE_ARRAY; - if (value != null) { + if (value == null) { + return null; + } else { valueBytes = value.get(); valueOffset = value.getOffset(); valueLength = value.getLength(); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index fb2952a..7d72557 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -1530,9 +1530,6 @@ public class MetaDataClient { String columnFamilyName = column.getFamilyName()!=null ? column.getFamilyName().getString() : null; colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(columnFamilyName, column.getName().getString())); isRowTimestamp = column.isRowTimestamp(); - if (colRef.getColumn().getExpressionStr() != null) { - expressionStr = colRef.getColumn().getExpressionStr(); - } } else { // if this is an expression @@ -3766,7 +3763,7 @@ public class MetaDataClient { // if cascade keyword is passed and indexes are provided either implicitly or explicitly if (cascade && (indexes == null || !indexes.isEmpty())) { indexesPTable = getIndexesPTableForCascade(indexes, table); - if(indexesPTable.size() == 0) { + if (indexesPTable.size() == 0) { // go back to regular behavior of altering the table/view cascade = false; } else { @@ -4768,7 +4765,7 @@ public class MetaDataClient { try { if (newIndexState == PIndexState.ACTIVE){ tableUpsert = connection.prepareStatement(UPDATE_INDEX_STATE_TO_ACTIVE); - }else{ + } else { tableUpsert = connection.prepareStatement(UPDATE_INDEX_STATE); } tableUpsert.setString(1, connection.getTenantId() == null ? null : connection.getTenantId().getString()); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/ValueGetterTuple.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/ValueGetterTuple.java index 833e9f9..e25be80 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/ValueGetterTuple.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/ValueGetterTuple.java @@ -55,16 +55,20 @@ public class ValueGetterTuple extends BaseTuple { return true; } - @Override - public KeyValue getValue(byte[] family, byte[] qualifier) { + public KeyValue getValueUnsafe(byte[] family, byte[] qualifier) { try { - KeyValue kv = valueGetter.getLatestKeyValue(new ColumnReference(family, qualifier), ts); - if (kv != null) { - return kv; - } + return valueGetter.getLatestKeyValue(new ColumnReference(family, qualifier), ts); } catch (IOException e) { throw new RuntimeException(e); } + } + + @Override + public KeyValue getValue(byte[] family, byte[] qualifier) { + KeyValue kv = getValueUnsafe(family, qualifier); + if (kv != null) { + return kv; + } byte[] rowKey = valueGetter.getRowKey(); byte[] valueBytes = HConstants.EMPTY_BYTE_ARRAY; return new KeyValue(rowKey, 0, rowKey.length, family, 0, family.length, qualifier, 0, qualifier.length, ts, Type.Put, valueBytes, 0, 0); @@ -89,8 +93,19 @@ public class ValueGetterTuple extends BaseTuple { public boolean getValue(byte[] family, byte[] qualifier, ImmutableBytesWritable ptr) { KeyValue kv = getValue(family, qualifier); - if (kv == null) + if (kv == null) { return false; + } + ptr.set(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()); + return true; + } + + public boolean getValueUnsafe(byte[] family, byte[] qualifier, + ImmutableBytesWritable ptr) { + KeyValue kv = getValueUnsafe(family, qualifier); + if (kv == null) { + return false; + } ptr.set(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()); return true; }