PHOENIX-4988 Incorrect index rowkey generated when updating only non-indexed columns after a delete
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/578b8402 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/578b8402 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/578b8402 Branch: refs/heads/4.14-cdh5.13 Commit: 578b8402ea4c5f000ec6c6046b656767e92ce96a Parents: 0952475 Author: Vincent Poon <vincentp...@apache.org> Authored: Mon Oct 22 21:20:10 2018 +0100 Committer: pboado <pedro.bo...@gmail.com> Committed: Mon Dec 3 20:32:49 2018 +0000 ---------------------------------------------------------------------- .../phoenix/end2end/index/MutableIndexIT.java | 36 ++++++++++++++++++++ .../filter/ApplyAndFilterDeletesFilter.java | 9 +++-- 2 files changed, 43 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/578b8402/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java index e968e99..1b9b8df 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java @@ -62,6 +62,7 @@ import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.EnvironmentEdgeManager; +import org.apache.phoenix.util.IndexScrutiny; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; @@ -910,6 +911,41 @@ public class MutableIndexIT extends ParallelStatsDisabledIT { } } + /** + * PHOENIX-4988 + * Test updating only a non-indexed column after two successive deletes to an indexed row + */ + @Test + public void testUpdateNonIndexedColumn() throws Exception { + String tableName = "TBL_" + generateUniqueName(); + String indexName = "IDX_" + generateUniqueName(); + String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName); + String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName); + try (Connection conn = getConnection()) { + conn.setAutoCommit(false); + conn.createStatement().execute("CREATE TABLE " + fullTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) " + tableDDLOptions); + conn.createStatement().execute("CREATE " + (localIndex ? " LOCAL " : "") + " INDEX " + indexName + " ON " + fullTableName + " (v2)"); + conn.createStatement().executeUpdate("UPSERT INTO " + fullTableName + "(k,v1,v2) VALUES ('testKey','v1_1','v2_1')"); + conn.commit(); + conn.createStatement().executeUpdate("DELETE FROM " + fullTableName); + conn.commit(); + conn.createStatement().executeUpdate("UPSERT INTO " + fullTableName + "(k,v1,v2) VALUES ('testKey','v1_2','v2_2')"); + conn.commit(); + conn.createStatement().executeUpdate("DELETE FROM " + fullTableName); + conn.commit(); + conn.createStatement().executeUpdate("UPSERT INTO " + fullTableName + "(k,v1) VALUES ('testKey','v1_3')"); + conn.commit(); + IndexScrutiny.scrutinizeIndex(conn, fullTableName, fullIndexName); + // PHOENIX-4980 + // When there is a flush after a data table update of non-indexed columns, the + // index gets out of sync on the next write + getUtility().getHBaseAdmin().flush(TableName.valueOf(fullTableName)); + conn.createStatement().executeUpdate("UPSERT INTO " + fullTableName + "(k,v1,v2) VALUES ('testKey','v1_4','v2_3')"); + conn.commit(); + IndexScrutiny.scrutinizeIndex(conn, fullTableName, fullIndexName); + } + } + private void upsertRow(String dml, Connection tenantConn, int i) throws SQLException { PreparedStatement stmt = tenantConn.prepareStatement(dml); stmt.setString(1, "00000000000000" + String.valueOf(i)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/578b8402/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java index a1f01ed..b5c3414 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/covered/filter/ApplyAndFilterDeletesFilter.java @@ -123,8 +123,13 @@ public class ApplyAndFilterDeletesFilter extends FilterBase { // kvs in the last column, so we can safely ignore the last deleteFamily, and just use this // one. In fact, it means that all the previous deletes can be ignored because the family must // not match anymore. - this.coveringDelete.reset(); - this.coveringDelete.deleteFamily = nextKV; + // We could potentially have multiple deleteFamily for the same row and family + // (e.g. upsert row+family, delete it, upsert again, delete again), + // in which case we keep the first one since its timestamp dominates + if (coveringDelete.deleteFamily == null || !CellUtil.matchingFamily(coveringDelete.deleteFamily, nextKV)) { + this.coveringDelete.reset(); + this.coveringDelete.deleteFamily = nextKV; + } return ReturnCode.SKIP; case DeleteColumn: // similar to deleteFamily, all the newer deletes/puts would have been seen at this point, so