PHOENIX-4059 Index maintenance incorrect when indexed column updated from null to null with STORE_NULLS=true
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0bd43f53 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0bd43f53 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0bd43f53 Branch: refs/heads/master Commit: 0bd43f536a13609b35629308c415aebc800d6799 Parents: e494fe9 Author: James Taylor <jamestay...@apache.org> Authored: Wed Aug 2 14:27:42 2017 -0700 Committer: James Taylor <jamestay...@apache.org> Committed: Wed Aug 2 14:40:31 2017 -0700 ---------------------------------------------------------------------- .../apache/phoenix/end2end/StoreNullsIT.java | 88 ++++++++++++++++++++ .../apache/phoenix/index/IndexMaintainer.java | 5 +- 2 files changed, 91 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bd43f53/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java index a37903f..120e25d 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java @@ -24,8 +24,11 @@ import static org.junit.Assert.assertTrue; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; +import java.sql.Timestamp; import java.util.Arrays; import java.util.Collection; import java.util.Properties; @@ -34,6 +37,7 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; @@ -48,6 +52,7 @@ import org.apache.phoenix.schema.tuple.ResultTuple; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Before; import org.junit.Test; @@ -242,4 +247,87 @@ public class StoreNullsIT extends ParallelStatsDisabledIT { } } + + private static long getRowCount(Connection conn, String tableName) throws SQLException { + ResultSet rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ count(*) FROM " + tableName); + assertTrue(rs.next()); + return rs.getLong(1); + } + + @Test + public void testSetIndexedColumnToNullTwiceWithStoreNulls() throws Exception { + if (!mutable) { + return; + } + + String tableName = generateUniqueName(); + String indexName = generateUniqueName(); + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + long ts = 1000; + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); + Connection conn = DriverManager.getConnection(getUrl(), props); + conn.createStatement().execute("CREATE TABLE " + tableName + "(k1 CHAR(2) NOT NULL, k2 CHAR(2) NOT NULL, ts TIMESTAMP, V VARCHAR, V2 VARCHAR, " + + "CONSTRAINT pk PRIMARY KEY (k1,k2)) STORE_NULLS=" + storeNulls + (columnEncoded ? "" : ",COLUMN_ENCODED_BYTES=0")); + conn.close(); + + ts = 1010; + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); + conn = DriverManager.getConnection(getUrl(), props); + conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + "(k2,k1,ts) INCLUDE (V, v2)"); + conn.close(); + + ts = 1020; + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); + conn = DriverManager.getConnection(getUrl(), props); + PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES('aa','aa',?, '0')"); + stmt.setTimestamp(1, new Timestamp(1000L)); + stmt.executeUpdate(); + conn.commit(); + conn.close(); + + Timestamp expectedTimestamp; + ts = 1030; + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); + conn = DriverManager.getConnection(getUrl(), props); + stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES('aa','aa',?, null)"); + expectedTimestamp = null; + stmt.setTimestamp(1, expectedTimestamp); + stmt.executeUpdate(); + conn.commit(); + + ts = 1040; + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); + conn = DriverManager.getConnection(getUrl(), props); + stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES('aa','aa',?, null)"); + expectedTimestamp = null; + stmt.setTimestamp(1, expectedTimestamp); + stmt.executeUpdate(); + conn.commit(); + + ts = 1050; + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); + conn = DriverManager.getConnection(getUrl(), props); + + TestUtil.dumpTable(conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(tableName))); + TestUtil.dumpTable(conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(indexName))); + + long count1 = getRowCount(conn, tableName); + long count2 = getRowCount(conn, indexName); + assertEquals("Table should have 1 row", 1, count1); + assertEquals("Index should have 1 row", 1, count2); + conn.close(); + + ResultSet rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ ts,v FROM " + tableName); + assertTrue(rs.next()); + assertEquals(expectedTimestamp, rs.getTimestamp(1)); + assertEquals(null, rs.getString(2)); + assertFalse(rs.next()); + + rs = conn.createStatement().executeQuery("SELECT \"0:TS\", \"0:V\" FROM " + indexName); + assertTrue(rs.next()); + assertEquals(expectedTimestamp, rs.getTimestamp(1)); + assertEquals(null, rs.getString(2)); + assertFalse(rs.next()); + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/0bd43f53/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java index a7ea99a..3b4faa9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java @@ -1081,12 +1081,13 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> { if (newValue != null) { // Indexed column has potentially changed ImmutableBytesWritable oldValue = oldState.getLatestValue(ref); boolean newValueSetAsNull = (newValue.getTypeByte() == Type.DeleteColumn.getCode() || newValue.getTypeByte() == Type.Delete.getCode() || CellUtil.matchingValue(newValue, HConstants.EMPTY_BYTE_ARRAY)); + boolean oldValueSetAsNull = oldValue == null || oldValue.getLength() == 0; //If the new column value has to be set as null and the older value is null too, //then just skip to the next indexed column. - if (newValueSetAsNull && oldValue == null) { + if (newValueSetAsNull && oldValueSetAsNull) { continue; } - if ((oldValue == null && !newValueSetAsNull) || (oldValue != null && newValueSetAsNull)) { + if (oldValueSetAsNull || newValueSetAsNull) { return true; } // If the old value is different than the new value, the index row needs to be deleted
