[ https://issues.apache.org/jira/browse/PHOENIX-6156?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
hahafam updated PHOENIX-6156: ----------------------------- Description: Affects Version 4.14.1-cdh5.14.2 Bug in WhereOptimizer$KeyExpressionVisitor.intersectTrailing included in PHOENIX-3383(https://issues.apache.org/jira/browse/PHOENIX-3383) patch Using AndExpression with RVCExpression in a query results in an exception. - Test Query {code:java} CREATE TABLE TEST.TABLE1 (CREATE TABLE TEST.TABLE1 ( ID1 BIGINT NOT NULL, YYYYMMDD INTEGER NOT NULL, ID2 VARCHAR NOT NULL, ID3 VARCHAR NOT NULL, CONSTRAINT PK PRIMARY KEY (ID1,YYYYMMDD,ID2,ID3)); SELECT ID1, YYYYMMDD, ID2, ID3FROM TEST.TABLE1WHERE (ID1 = 100200300 AND YYYYMMDD = 20200101 AND ID2 = 'LENGTH_8_BYTE_OVER' AND ID3 = 'ABCDEFGHIJK') AND (ID1, YYYYMMDD, ID2, ID3) IN ((100200300, 20200101,'LENGTH_8_BYTE_OVER', 'ABCDEFGHIJK'),(200200300, 20200201,'ABC', 'ABCDEFGHIJK')); {code} - raise IndexOutOfExceptin {code:java} at org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.intersectTrailing(WhereOptimizer.java:1362) at org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.andKeySlots(WhereOptimizer.java:974) at org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.visitLeave(WhereOptimizer.java:1501) at org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.visitLeave(WhereOptimizer.java:512) at org.apache.phoenix.expression.AndExpression.accept(AndExpression.java:100) at org.apache.phoenix.compile.WhereOptimizer.pushKeyExpressionsToScan(WhereOptimizer.java:142) at org.apache.phoenix.compile.WhereCompiler.compile(WhereCompiler.java:157) at org.apache.phoenix.execute.HashJoinPlan.iterator(HashJoinPlan.java:232) at org.apache.phoenix.execute.DelegateQueryPlan.iterator(DelegateQueryPlan.java:144) at org.apache.phoenix.execute.DelegateQueryPlan.iterator(DelegateQueryPlan.java:139) at org.apache.phoenix.jdbc.PhoenixStatement$1.call(PhoenixStatement.java:312) at org.apache.phoenix.jdbc.PhoenixStatement$1.call(PhoenixStatement.java:291) at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53) at org.apache.phoenix.jdbc.PhoenixStatement.executeQuery(PhoenixStatement.java:290) at org.apache.phoenix.jdbc.PhoenixStatement.executeQuery(PhoenixStatement.java:283) at org.apache.phoenix.jdbc.PhoenixPreparedStatement.executeQuery(PhoenixPreparedStatement.java:194) {code} Exception occurs when the value of the ID2 column is 8 bytes or more. This is because of the way bytes are read in the function below. org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.intersectTrailing - My log {code:java} andExpression=((ID1 = 100200300 AND YYYYMMDD = 20200101 AND ID2 = 'LENGTH_8_BYTE_OVER' AND ID3 = 'ABCDEFGHIJK') AND (ID1, YYYYMMDD, ID2, ID3) IN ([128,0,0,0,5,248,239,108,129,52,58,165,76,69,78,71,84,72,95,56,95,66,89,84,69,95,79,86,69,82,0,65,66,67,68,69,70,71,72,73,74,75],[128,0,0,0,11,238,208,108,129,52,59,9,65,66,67,0,65,66,67,68,69,70,71,72,73,74,75],...)){code} - bytes, 8byte(bigint) + 4byte(int) + LENGTH_8_BYTE_OVER + delimeter(\0) + ABCDEFGHIJK {code:java} 128,0,0,0,5,248,239,108,129,52,58,165,76,69,78,71,84,72,95,56,95,66,89,84,69,95,79,86,69,82,0,65,66,67,68,69,70,71,72,73,74,75{code} intersectTrailing function reads byte : 8byte > 8byte > 4byte > remain If the value of the ID2 column is more than 8 bytes, the delimeter is encountered when reading the 'remain' byte, and the loop is executed once more. {code:java} 2020-09-23 17:27:36,907 compile.WhereOptimizer$KeyExpressionVisitor(1358) - MYLOG intersectTrailing : result=\x80\x00\x00\x00\x05\xF8\xEFl\x814:\xA5LENGTH_8_BYTE_OVER\x00ABCDEFGHIJK, pkPos=0, otherRange=\x814:\xA5, otherPKPos=1 2020-09-23 17:27:36,907 compile.WhereOptimizer$KeyExpressionVisitor(1429) - MYLOG intersectTrailing : pos=1, getOffset=8, getLength=4, ptr=[-127, 52, 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1429) - MYLOG intersectTrailing : pos=2, getOffset=12, getLength=10, ptr=[-127, 52, 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1429) - MYLOG intersectTrailing : pos=3, getOffset=23, getLength=11, ptr=[-127, 52, 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1429) - MYLOG intersectTrailing : pos=4, getOffset=34, getLength=0, ptr=[-127, 52, 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1445) - MYLOG intersectTrailing : error=index (4) must be less than size (4) {code} * Test Code {code:java} @Test public void testAndWithRVC2() throws Exception { String ddl; String query; StatementContext context; Connection conn = DriverManager.getConnection(getUrl()); ddl = "CREATE TABLE TEST.TABLE1 ( " + "ID1 BIGINT NOT NULL, YYYYMMDD INTEGER NOT NULL, " + "ID2 VARCHAR NOT NULL, ID3 VARCHAR NOT NULL, " + "CONSTRAINT PK PRIMARY KEY (ID1,YYYYMMDD,ID2,ID3) " + ")"; conn.createStatement().execute(ddl); query = "SELECT ID1, YYYYMMDD, ID2, ID3 " + "FROM TEST.TABLE1 " + "WHERE " + "(ID1 = 100200300 AND YYYYMMDD = 20200101 AND ID2 = 'LENGTH_8_BYTE_OVER' AND ID3 = 'ABCDEFGHIJK') AND " + "(ID1, YYYYMMDD, ID2, ID3) IN " + "((100200300, 20200101,'LENGTH_8_BYTE_OVER', 'ABCDEFGHIJK'), (200200300, 20200201,'ABC', 'ABCDEFGHIJK'))" + ""; context = compileStatement(query, Collections.<Object>emptyList()); Scan scan = context.getScan(); conn.close(); } {code} was: Bug in WhereOptimizer$KeyExpressionVisitor.intersectTrailing included in PHOENIX-3383(https://issues.apache.org/jira/browse/PHOENIX-3383) patch Using AndExpression with RVCExpression in a query results in an exception. - Test Query {code:java} CREATE TABLE TEST.TABLE1 (CREATE TABLE TEST.TABLE1 ( ID1 BIGINT NOT NULL, YYYYMMDD INTEGER NOT NULL, ID2 VARCHAR NOT NULL, ID3 VARCHAR NOT NULL, CONSTRAINT PK PRIMARY KEY (ID1,YYYYMMDD,ID2,ID3)); SELECT ID1, YYYYMMDD, ID2, ID3FROM TEST.TABLE1WHERE (ID1 = 100200300 AND YYYYMMDD = 20200101 AND ID2 = 'LENGTH_8_BYTE_OVER' AND ID3 = 'ABCDEFGHIJK') AND (ID1, YYYYMMDD, ID2, ID3) IN ((100200300, 20200101,'LENGTH_8_BYTE_OVER', 'ABCDEFGHIJK'),(200200300, 20200201,'ABC', 'ABCDEFGHIJK')); {code} - raise IndexOutOfExceptin {code:java} at org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.intersectTrailing(WhereOptimizer.java:1362) at org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.andKeySlots(WhereOptimizer.java:974) at org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.visitLeave(WhereOptimizer.java:1501) at org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.visitLeave(WhereOptimizer.java:512) at org.apache.phoenix.expression.AndExpression.accept(AndExpression.java:100) at org.apache.phoenix.compile.WhereOptimizer.pushKeyExpressionsToScan(WhereOptimizer.java:142) at org.apache.phoenix.compile.WhereCompiler.compile(WhereCompiler.java:157) at org.apache.phoenix.execute.HashJoinPlan.iterator(HashJoinPlan.java:232) at org.apache.phoenix.execute.DelegateQueryPlan.iterator(DelegateQueryPlan.java:144) at org.apache.phoenix.execute.DelegateQueryPlan.iterator(DelegateQueryPlan.java:139) at org.apache.phoenix.jdbc.PhoenixStatement$1.call(PhoenixStatement.java:312) at org.apache.phoenix.jdbc.PhoenixStatement$1.call(PhoenixStatement.java:291) at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53) at org.apache.phoenix.jdbc.PhoenixStatement.executeQuery(PhoenixStatement.java:290) at org.apache.phoenix.jdbc.PhoenixStatement.executeQuery(PhoenixStatement.java:283) at org.apache.phoenix.jdbc.PhoenixPreparedStatement.executeQuery(PhoenixPreparedStatement.java:194) {code} Exception occurs when the value of the ID2 column is 8 bytes or more. This is because of the way bytes are read in the function below. org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.intersectTrailing - My log {code:java} andExpression=((ID1 = 100200300 AND YYYYMMDD = 20200101 AND ID2 = 'LENGTH_8_BYTE_OVER' AND ID3 = 'ABCDEFGHIJK') AND (ID1, YYYYMMDD, ID2, ID3) IN ([128,0,0,0,5,248,239,108,129,52,58,165,76,69,78,71,84,72,95,56,95,66,89,84,69,95,79,86,69,82,0,65,66,67,68,69,70,71,72,73,74,75],[128,0,0,0,11,238,208,108,129,52,59,9,65,66,67,0,65,66,67,68,69,70,71,72,73,74,75],...)){code} - bytes, 8byte(bigint) + 4byte(int) + LENGTH_8_BYTE_OVER + delimeter(\0) + ABCDEFGHIJK {code:java} 128,0,0,0,5,248,239,108,129,52,58,165,76,69,78,71,84,72,95,56,95,66,89,84,69,95,79,86,69,82,0,65,66,67,68,69,70,71,72,73,74,75{code} intersectTrailing function reads byte : 8byte > 8byte > 4byte > remain If the value of the ID2 column is more than 8 bytes, the delimeter is encountered when reading the 'remain' byte, and the loop is executed once more. {code:java} 2020-09-23 17:27:36,907 compile.WhereOptimizer$KeyExpressionVisitor(1358) - MYLOG intersectTrailing : result=\x80\x00\x00\x00\x05\xF8\xEFl\x814:\xA5LENGTH_8_BYTE_OVER\x00ABCDEFGHIJK, pkPos=0, otherRange=\x814:\xA5, otherPKPos=1 2020-09-23 17:27:36,907 compile.WhereOptimizer$KeyExpressionVisitor(1429) - MYLOG intersectTrailing : pos=1, getOffset=8, getLength=4, ptr=[-127, 52, 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1429) - MYLOG intersectTrailing : pos=2, getOffset=12, getLength=10, ptr=[-127, 52, 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1429) - MYLOG intersectTrailing : pos=3, getOffset=23, getLength=11, ptr=[-127, 52, 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1429) - MYLOG intersectTrailing : pos=4, getOffset=34, getLength=0, ptr=[-127, 52, 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1445) - MYLOG intersectTrailing : error=index (4) must be less than size (4) {code} > IndexOutOfBoundsException, Expression 'AND' with 'RVC' > ------------------------------------------------------ > > Key: PHOENIX-6156 > URL: https://issues.apache.org/jira/browse/PHOENIX-6156 > Project: Phoenix > Issue Type: Bug > Components: core > Reporter: hahafam > Priority: Major > > Affects Version 4.14.1-cdh5.14.2 > > Bug in WhereOptimizer$KeyExpressionVisitor.intersectTrailing included in > PHOENIX-3383(https://issues.apache.org/jira/browse/PHOENIX-3383) patch > > Using AndExpression with RVCExpression in a query results in an exception. > > - Test Query > > {code:java} > CREATE TABLE TEST.TABLE1 (CREATE TABLE TEST.TABLE1 ( ID1 BIGINT NOT NULL, > YYYYMMDD INTEGER NOT NULL, ID2 VARCHAR NOT NULL, ID3 VARCHAR NOT NULL, > CONSTRAINT PK PRIMARY KEY (ID1,YYYYMMDD,ID2,ID3)); > SELECT ID1, YYYYMMDD, ID2, ID3FROM TEST.TABLE1WHERE (ID1 = 100200300 AND > YYYYMMDD = 20200101 AND ID2 = 'LENGTH_8_BYTE_OVER' AND ID3 = 'ABCDEFGHIJK') > AND (ID1, YYYYMMDD, ID2, ID3) IN ((100200300, > 20200101,'LENGTH_8_BYTE_OVER', 'ABCDEFGHIJK'),(200200300, 20200201,'ABC', > 'ABCDEFGHIJK')); > {code} > > - raise IndexOutOfExceptin > > {code:java} > at > org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.intersectTrailing(WhereOptimizer.java:1362) > at > org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.andKeySlots(WhereOptimizer.java:974) > at > org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.visitLeave(WhereOptimizer.java:1501) > at > org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.visitLeave(WhereOptimizer.java:512) > at > org.apache.phoenix.expression.AndExpression.accept(AndExpression.java:100) > at > org.apache.phoenix.compile.WhereOptimizer.pushKeyExpressionsToScan(WhereOptimizer.java:142) > at > org.apache.phoenix.compile.WhereCompiler.compile(WhereCompiler.java:157) > at org.apache.phoenix.execute.HashJoinPlan.iterator(HashJoinPlan.java:232) > at > org.apache.phoenix.execute.DelegateQueryPlan.iterator(DelegateQueryPlan.java:144) > at > org.apache.phoenix.execute.DelegateQueryPlan.iterator(DelegateQueryPlan.java:139) > at > org.apache.phoenix.jdbc.PhoenixStatement$1.call(PhoenixStatement.java:312) > at > org.apache.phoenix.jdbc.PhoenixStatement$1.call(PhoenixStatement.java:291) > at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53) > at > org.apache.phoenix.jdbc.PhoenixStatement.executeQuery(PhoenixStatement.java:290) > at > org.apache.phoenix.jdbc.PhoenixStatement.executeQuery(PhoenixStatement.java:283) > at > org.apache.phoenix.jdbc.PhoenixPreparedStatement.executeQuery(PhoenixPreparedStatement.java:194) > {code} > > > Exception occurs when the value of the ID2 column is 8 bytes or more. > This is because of the way bytes are read in the function below. > > org.apache.phoenix.compile.WhereOptimizer$KeyExpressionVisitor.intersectTrailing > > > - My log > > {code:java} > andExpression=((ID1 = 100200300 AND YYYYMMDD = 20200101 AND ID2 = > 'LENGTH_8_BYTE_OVER' AND ID3 = 'ABCDEFGHIJK') AND (ID1, YYYYMMDD, ID2, ID3) > IN > ([128,0,0,0,5,248,239,108,129,52,58,165,76,69,78,71,84,72,95,56,95,66,89,84,69,95,79,86,69,82,0,65,66,67,68,69,70,71,72,73,74,75],[128,0,0,0,11,238,208,108,129,52,59,9,65,66,67,0,65,66,67,68,69,70,71,72,73,74,75],...)){code} > > > - bytes, 8byte(bigint) + 4byte(int) + LENGTH_8_BYTE_OVER + delimeter(\0) + > ABCDEFGHIJK > > {code:java} > 128,0,0,0,5,248,239,108,129,52,58,165,76,69,78,71,84,72,95,56,95,66,89,84,69,95,79,86,69,82,0,65,66,67,68,69,70,71,72,73,74,75{code} > > > intersectTrailing function reads byte : 8byte > 8byte > 4byte > remain > If the value of the ID2 column is more than 8 bytes, the delimeter is > encountered when reading the 'remain' byte, and the loop is executed once > more. > > {code:java} > 2020-09-23 17:27:36,907 compile.WhereOptimizer$KeyExpressionVisitor(1358) - > MYLOG intersectTrailing : > result=\x80\x00\x00\x00\x05\xF8\xEFl\x814:\xA5LENGTH_8_BYTE_OVER\x00ABCDEFGHIJK, > pkPos=0, otherRange=\x814:\xA5, otherPKPos=1 > 2020-09-23 17:27:36,907 compile.WhereOptimizer$KeyExpressionVisitor(1429) - > MYLOG intersectTrailing : pos=1, getOffset=8, getLength=4, ptr=[-127, 52, 58, > -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, 82, > 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], > str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK > 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1429) - > MYLOG intersectTrailing : pos=2, getOffset=12, getLength=10, ptr=[-127, 52, > 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, > 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], > str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK > 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1429) - > MYLOG intersectTrailing : pos=3, getOffset=23, getLength=11, ptr=[-127, 52, > 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, > 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], > str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK > 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1429) - > MYLOG intersectTrailing : pos=4, getOffset=34, getLength=0, ptr=[-127, 52, > 58, -91, 76, 69, 78, 71, 84, 72, 95, 56, 95, 66, 89, 84, 69, 95, 79, 86, 69, > 82, 0, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75], > str=�4:�LENGTH_8_BYTE_OVERABCDEFGHIJK > 2020-09-23 17:27:36,908 compile.WhereOptimizer$KeyExpressionVisitor(1445) - > MYLOG intersectTrailing : error=index (4) must be less than size (4) > {code} > > * Test Code > > {code:java} > @Test > public void testAndWithRVC2() throws Exception { > String ddl; > String query; > StatementContext context; > Connection conn = DriverManager.getConnection(getUrl()); > ddl = "CREATE TABLE TEST.TABLE1 ( " + > "ID1 BIGINT NOT NULL, YYYYMMDD INTEGER NOT NULL, " + > "ID2 VARCHAR NOT NULL, ID3 VARCHAR NOT NULL, " + > "CONSTRAINT PK PRIMARY KEY (ID1,YYYYMMDD,ID2,ID3) " + > ")"; > conn.createStatement().execute(ddl); > query = "SELECT ID1, YYYYMMDD, ID2, ID3 " + > "FROM TEST.TABLE1 " + > "WHERE " + > "(ID1 = 100200300 AND YYYYMMDD = 20200101 AND ID2 = > 'LENGTH_8_BYTE_OVER' AND ID3 = 'ABCDEFGHIJK') AND " + > "(ID1, YYYYMMDD, ID2, ID3) IN " + > "((100200300, 20200101,'LENGTH_8_BYTE_OVER', 'ABCDEFGHIJK'), > (200200300, 20200201,'ABC', 'ABCDEFGHIJK'))" + > ""; > context = compileStatement(query, Collections.<Object>emptyList()); > Scan scan = context.getScan(); > conn.close(); > } > {code} > > > > > -- This message was sent by Atlassian Jira (v8.3.4#803005)