[ https://issues.apache.org/jira/browse/PHOENIX-2613?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
chenglei updated PHOENIX-2613: ------------------------------ Description: In pheonix 4.6,any column of multi-part primary key can be null.If a table has one row which has a column of multi-part primary key is null, and the java assert is disable, when we do a query,the Skip Scan may cause RegionServer scan indefinite loop,just as the following unit test: {code:borderStyle=solid} @Test public void testNullInfiniteLoop() throws Exception { this.jdbcTemplate.update("drop table if exists NULL_TEST "); this.jdbcTemplate.update( "create table NULL_TEST"+ "("+ "CREATETIME VARCHAR,"+ "ACCOUNTID VARCHAR,"+ "SERVICENAME VARCHAR,"+ "SPAN.APPID VARCHAR,"+ "CONSTRAINT pk PRIMARY KEY(CREATETIME,ACCOUNTID,SERVICENAME)"+ ")"); this.jdbcTemplate.update("upsert into NULL_TEST(CREATETIME,SERVICENAME,SPAN.APPID) values('20160116141006','servlet','android')"); this.jdbcTemplate.update("upsert into NULL_TEST(CREATETIME,ACCOUNTID,SERVICENAME,SPAN.APPID) values('20160116151006','2404787','jdbc','ios')"); this.jdbcTemplate.queryForList("select * from NULL_TEST where CREATETIME>='20160116121006' and CREATETIME<='20160116181006' and ACCOUNTID='2404787'"); } {code} As above unit test explained,we create a NULL_TEST table, and insert a row which ACCOUNTID column is null, When we do a query which condition is CREATETIME column and ACCOUNTID column, Phoenix will use SkipScanFilter to scan the table. Unfortunately,the query will run forever,can not return result. If we construct a SkipScanFilter using the above query condition,and we can see after the SkipScanFilter's filterKeyValue method is called on the KeyValue which rowKey is(the ACCOUNTID column is null) : {noformat} 20160116141006\\x00\\x00servlet {noformat} the SkipScanFilter will return a erroneous NextHintCell value(2404787 is the ACCOUNTID): {noformat} 20160116141006\\x00\\x002404787 {noformat} which should be: {noformat} 20160116141006\\x002404787 {noformat} Just As the following unit test on the SkipScanFilter: {code:borderStyle=solid} @Test public void testNextCellHintError() throws Exception { List<List<KeyRange>> keyRanges=Arrays.asList( Collections.singletonList(KeyRange.getKeyRange(Bytes.toBytes("20160116121006"),true,Bytes.toBytes("20160116181006"),true)), Collections.singletonList(PChar.INSTANCE.getKeyRange(Bytes.toBytes("2404787"), true, Bytes.toBytes("2404787"), true))); RowKeySchemaBuilder rowKeySchemaBuilder = new RowKeySchemaBuilder(3); for(int i=1;i<=3;i++) { rowKeySchemaBuilder.addField(new PDatum() { @Override public boolean isNullable() { return true; } @Override public PDataType getDataType() { return PVarchar.INSTANCE; } @Override public Integer getMaxLength() { return null; } @Override public Integer getScale() { return null; } @Override public SortOrder getSortOrder() { return SortOrder.getDefault(); } }, true, SortOrder.getDefault()); } RowKeySchema rowKeySchema=rowKeySchemaBuilder.build(); byte[] rowKey=Bytes.toBytesBinary("20160116141006\\x00\\x00servlet"); KeyValue keyValue=new KeyValue( rowKey, Bytes.toBytes("SPAN"), Bytes.toBytes("APPID"), 1453117575829L, org.apache.hadoop.hbase.KeyValue.Type.Put); SkipScanFilter skipScanFilter=new SkipScanFilter(keyRanges, rowKeySchema); skipScanFilter.filterKeyValue(keyValue); Cell nextCellHint=skipScanFilter.getNextCellHint(keyValue); assertTrue(Bytes.toStringBinary(CellUtil.cloneRow(nextCellHint)).equals("20160116141006\\x00\\x002404787")); //try once again... skipScanFilter.filterKeyValue(keyValue); nextCellHint=skipScanFilter.getNextCellHint(keyValue); assertTrue(Bytes.toStringBinary(CellUtil.cloneRow(nextCellHint)).equals("20160116141006\\x00\\x002404787")); } {code} was: In pheonix 4.6,any column of multi-part primary key can be null.If a table has one row which has a column of multi-part primary key is null, and the java assert is disable, when we do a query,the Skip Scan may cause RegionServer scan indefinite loop,just as the following unit test: {code:borderStyle=solid} @Test public void testNullInfiniteLoop() throws Exception { this.jdbcTemplate.update("drop table if exists NULL_TEST "); this.jdbcTemplate.update( "create table NULL_TEST"+ "("+ "CREATETIME VARCHAR,"+ "ACCOUNTID VARCHAR,"+ "SERVICENAME VARCHAR,"+ "SPAN.APPID VARCHAR,"+ "CONSTRAINT pk PRIMARY KEY(CREATETIME,ACCOUNTID,SERVICENAME)"+ ")"); this.jdbcTemplate.update("upsert into NULL_TEST(CREATETIME,SERVICENAME,SPAN.APPID) values('20160116141006','servlet','android')"); this.jdbcTemplate.update("upsert into NULL_TEST(CREATETIME,ACCOUNTID,SERVICENAME,SPAN.APPID) values('20160116151006','2404787','jdbc','ios')"); this.jdbcTemplate.queryForList("select * from NULL_TEST where CREATETIME>='20160116121006' and CREATETIME<='20160116181006' and ACCOUNTID='2404787'"); } {code} As above unit test explained,we create a NULL_TEST table, and insert a row which ACCOUNT column is null, When we do a query which condition is CREATETIME column and ACCOUNTID column, Phoenix will use SkipScanFilter to scan the table. Unfortunately,the query will run forever,can not return result. If we construct a SkipScanFilter using the above query condition,and we can see after the SkipScanFilter's filterKeyValue method is called on the KeyValue which rowKey is : {noformat} 20160116141006\\x00\\x00servlet(the ACCOUNT column is null) {noformat} the SkipScanFilter will return a erroneous NextHintCell value,2404787 is the ACCOUNTID: {noformat} 20160116141006\\x00\\x002404787 {noformat} which should be: {noformat} 20160116141006\\x002404787 {noformat} Just As the following unit test on the SkipScanFilter: {code:borderStyle=solid} @Test public void testNextCellHintError() throws Exception { List<List<KeyRange>> keyRanges=Arrays.asList( Collections.singletonList(KeyRange.getKeyRange(Bytes.toBytes("20160116121006"),true,Bytes.toBytes("20160116181006"),true)), Collections.singletonList(PChar.INSTANCE.getKeyRange(Bytes.toBytes("2404787"), true, Bytes.toBytes("2404787"), true))); RowKeySchemaBuilder rowKeySchemaBuilder = new RowKeySchemaBuilder(3); for(int i=1;i<=3;i++) { rowKeySchemaBuilder.addField(new PDatum() { @Override public boolean isNullable() { return true; } @Override public PDataType getDataType() { return PVarchar.INSTANCE; } @Override public Integer getMaxLength() { return null; } @Override public Integer getScale() { return null; } @Override public SortOrder getSortOrder() { return SortOrder.getDefault(); } }, true, SortOrder.getDefault()); } RowKeySchema rowKeySchema=rowKeySchemaBuilder.build(); byte[] rowKey=Bytes.toBytesBinary("20160116141006\\x00\\x00servlet"); KeyValue keyValue=new KeyValue( rowKey, Bytes.toBytes("SPAN"), Bytes.toBytes("APPID"), 1453117575829L, org.apache.hadoop.hbase.KeyValue.Type.Put); SkipScanFilter skipScanFilter=new SkipScanFilter(keyRanges, rowKeySchema); skipScanFilter.filterKeyValue(keyValue); Cell nextCellHint=skipScanFilter.getNextCellHint(keyValue); assertTrue(Bytes.toStringBinary(CellUtil.cloneRow(nextCellHint)).equals("20160116141006\\x00\\x002404787")); //try once again... skipScanFilter.filterKeyValue(keyValue); nextCellHint=skipScanFilter.getNextCellHint(keyValue); assertTrue(Bytes.toStringBinary(CellUtil.cloneRow(nextCellHint)).equals("20160116141006\\x00\\x002404787")); } {code} > if any column of multi-part primary key is null, the Skip Scan may cause > RegionServer scan indefinite loop > ---------------------------------------------------------------------------------------------------------- > > Key: PHOENIX-2613 > URL: https://issues.apache.org/jira/browse/PHOENIX-2613 > Project: Phoenix > Issue Type: Bug > Affects Versions: 4.6.0 > Environment: HBase 0.98.6-cdh5.3.2, Phoenix 4.6.0-HBase-0.98 > Reporter: chenglei > > In pheonix 4.6,any column of multi-part primary key can be null.If a table > has one row which has a column of multi-part primary key is null, and the > java assert is disable, when we do a query,the Skip Scan may cause > RegionServer scan indefinite loop,just as the following unit test: > {code:borderStyle=solid} > @Test > public void testNullInfiniteLoop() throws Exception > { > this.jdbcTemplate.update("drop table if exists NULL_TEST "); > > this.jdbcTemplate.update( > "create table NULL_TEST"+ > "("+ > "CREATETIME VARCHAR,"+ > "ACCOUNTID VARCHAR,"+ > "SERVICENAME VARCHAR,"+ > "SPAN.APPID VARCHAR,"+ > "CONSTRAINT pk PRIMARY > KEY(CREATETIME,ACCOUNTID,SERVICENAME)"+ > ")"); > > this.jdbcTemplate.update("upsert into > NULL_TEST(CREATETIME,SERVICENAME,SPAN.APPID) > values('20160116141006','servlet','android')"); > this.jdbcTemplate.update("upsert into > NULL_TEST(CREATETIME,ACCOUNTID,SERVICENAME,SPAN.APPID) > values('20160116151006','2404787','jdbc','ios')"); > this.jdbcTemplate.queryForList("select * from NULL_TEST where > CREATETIME>='20160116121006' and CREATETIME<='20160116181006' and > ACCOUNTID='2404787'"); > > } > {code} > As above unit test explained,we create a NULL_TEST table, and insert a row > which ACCOUNTID column is null, When we do a query which condition is > CREATETIME column and ACCOUNTID column, Phoenix will use SkipScanFilter to > scan the table. Unfortunately,the query will run forever,can not return > result. > If we construct a SkipScanFilter using the above query condition,and we can > see after the SkipScanFilter's filterKeyValue method is called on the > KeyValue which rowKey is(the ACCOUNTID column is null) : > {noformat} > 20160116141006\\x00\\x00servlet > {noformat} > the SkipScanFilter will return a erroneous NextHintCell value(2404787 is the > ACCOUNTID): > {noformat} > 20160116141006\\x00\\x002404787 > {noformat} > which should be: > {noformat} > 20160116141006\\x002404787 > {noformat} > Just As the following unit test on the SkipScanFilter: > {code:borderStyle=solid} > @Test > public void testNextCellHintError() throws Exception > { > List<List<KeyRange>> keyRanges=Arrays.asList( > > Collections.singletonList(KeyRange.getKeyRange(Bytes.toBytes("20160116121006"),true,Bytes.toBytes("20160116181006"),true)), > > Collections.singletonList(PChar.INSTANCE.getKeyRange(Bytes.toBytes("2404787"), > true, Bytes.toBytes("2404787"), true))); > > RowKeySchemaBuilder rowKeySchemaBuilder = new RowKeySchemaBuilder(3); > > for(int i=1;i<=3;i++) > { > rowKeySchemaBuilder.addField(new PDatum() { > @Override > public boolean isNullable() { > return true; > } > @Override > public PDataType getDataType() { > return PVarchar.INSTANCE; > } > @Override > public Integer getMaxLength() { > return null; > } > @Override > public Integer getScale() { > return null; > } > @Override > public SortOrder getSortOrder() { > return SortOrder.getDefault(); > } > }, true, SortOrder.getDefault()); > } > > RowKeySchema rowKeySchema=rowKeySchemaBuilder.build(); > > byte[] rowKey=Bytes.toBytesBinary("20160116141006\\x00\\x00servlet"); > KeyValue keyValue=new KeyValue( > rowKey, > Bytes.toBytes("SPAN"), > Bytes.toBytes("APPID"), > 1453117575829L, > org.apache.hadoop.hbase.KeyValue.Type.Put); > SkipScanFilter skipScanFilter=new SkipScanFilter(keyRanges, > rowKeySchema); > skipScanFilter.filterKeyValue(keyValue); > Cell nextCellHint=skipScanFilter.getNextCellHint(keyValue); > > assertTrue(Bytes.toStringBinary(CellUtil.cloneRow(nextCellHint)).equals("20160116141006\\x00\\x002404787")); > //try once again... > skipScanFilter.filterKeyValue(keyValue); > nextCellHint=skipScanFilter.getNextCellHint(keyValue); > > assertTrue(Bytes.toStringBinary(CellUtil.cloneRow(nextCellHint)).equals("20160116141006\\x00\\x002404787")); > } > {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332)