jean-claude created DRILL-7045: ---------------------------------- Summary: UDF string_binary java.lang.IndexOutOfBoundsException: Key: DRILL-7045 URL: https://issues.apache.org/jira/browse/DRILL-7045 Project: Apache Drill Issue Type: Bug Components: Functions - Drill Affects Versions: 1.15.0 Reporter: jean-claude
Given a large field like cat input.json { "col0": "lajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjjflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfjlajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfj" } { "col0": "lajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfj" } { "col0": "lajsldfjlkajflksdjfjasdlfjsalk;dfjasdlkfj" } drop table dfs.tmp.`converted`; create table dfs.tmp.`converted` as (select convert_to(col0, 'UTF8') as col0_bin from dfs.root.`prog/input.json`); select col0_bin from dfs.tmp.`converted`; select convert_from(col0_bin, 'UTF8') from dfs.tmp.`converted`; select string_binary(col0_bin) from dfs.tmp.`converted`; The last statement fails with Caused by: java.lang.IndexOutOfBoundsException: index: 0, length: 9068 (expected: range(0, 256)) at io.netty.buffer.AbstractByteBuf.checkIndex0(AbstractByteBuf.java:1125) at io.netty.buffer.AbstractByteBuf.checkIndex(AbstractByteBuf.java:1120) at io.netty.buffer.UnsafeByteBufUtil.setBytes(UnsafeByteBufUtil.java:349) at io.netty.buffer.PooledUnsafeDirectByteBuf.setBytes(PooledUnsafeDirectByteBuf.java:199) at io.netty.buffer.WrappedByteBuf.setBytes(WrappedByteBuf.java:397) at io.netty.buffer.UnsafeDirectLittleEndian.setBytes(UnsafeDirectLittleEndian.java:36) at io.netty.buffer.DrillBuf.setBytes(DrillBuf.java:735) at io.netty.buffer.AbstractByteBuf.setBytes(AbstractByteBuf.java:528) at org.apache.drill.exec.test.generated.ProjectorGen15.doEval(ProjectorTemplate.java:49) at org.apache.drill.exec.test.generated.ProjectorGen15.projectRecords(ProjectorTemplate.java:67) at org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.doWork(ProjectRecordBatch.java:232) This is probably due to the fact that the string_binary UDF does not reallocate it's buffer based on the size of the converted byte array. {code:java} public void eval() { byte[] buf = org.apache.drill.common.util.DrillStringUtils.toBinaryString(in.buffer, in.start, in.end).getBytes(charset); buffer.setBytes(0, buf); buffer.setIndex(0, buf.length); out.start = 0; out.end = buf.length; out.buffer = buffer; }{code} is missing an {code:java} out.buffer = buffer.reallocIfNeeded(in.end - in.start);{code} -- This message was sent by Atlassian JIRA (v7.6.3#76005)