This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new e9b59e7014 Table output fails with StackOverflowError on wide Oracle
tables since, fixes #9420 (#9421)
e9b59e7014 is described below
commit e9b59e7014f0f00da9385a2461ad3b5722011aa8
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Nov 12 08:12:38 2026 +0100
Table output fails with StackOverflowError on wide Oracle tables since,
fixes #9420 (#9421)
---
integration-tests/oracle/0012-wide-table-write.hpl | 109 +++++++++
.../oracle/main-0012-wide-table-write.hwf | 245 +++++++++++++++++++++
.../oracle/OraclePreparedStatementBinding.java | 41 +++-
.../oracle/OraclePreparedStatementBindingTest.java | 71 ++++++
4 files changed, 464 insertions(+), 2 deletions(-)
diff --git a/integration-tests/oracle/0012-wide-table-write.hpl
b/integration-tests/oracle/0012-wide-table-write.hpl
new file mode 100644
index 0000000000..b8873fc4ce
--- /dev/null
+++ b/integration-tests/oracle/0012-wide-table-write.hpl
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0012-wide-table-write</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Copies a 1000-column table with a Table input and a Table
output, mapping every field by name. The INSERT this produces is wide enough
that the Oracle driver's parse of it, done to find each parameter's column
type, used to overflow the transform thread's stack.</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2026/11/11 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/11 12:00:00.000</modified_date>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Wide table in</from>
+ <to>Wide table out</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Wide table in</name>
+ <type>TableInput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <connection>oracle-service-name</connection>
+ <execute_each_row>N</execute_each_row>
+ <limit>0</limit>
+ <sql>SELECT * FROM HOP_WIDE_SOURCE ORDER BY ID</sql>
+ <variables_active>N</variables_active>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Wide table out</name>
+ <type>TableOutput</type>
+ <description>Fields are not specified, so all 1000 go into the INSERT by
name.</description>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <commit>1000</commit>
+ <connection>oracle-service-name</connection>
+ <fields>
+ </fields>
+ <ignore_errors>N</ignore_errors>
+ <only_when_have_rows>N</only_when_have_rows>
+ <partitioning_daily>N</partitioning_daily>
+ <partitioning_enabled>N</partitioning_enabled>
+ <partitioning_field/>
+ <partitioning_monthly>Y</partitioning_monthly>
+ <return_field/>
+ <return_keys>N</return_keys>
+ <schema/>
+ <specify_fields>N</specify_fields>
+ <table>HOP_WIDE_TARGET</table>
+ <tablename_field/>
+ <tablename_in_field>N</tablename_in_field>
+ <tablename_in_table>Y</tablename_in_table>
+ <truncate>N</truncate>
+ <use_batch>Y</use_batch>
+ <attributes/>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>144</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/oracle/main-0012-wide-table-write.hwf
b/integration-tests/oracle/main-0012-wide-table-write.hwf
new file mode 100644
index 0000000000..a0e89a2465
--- /dev/null
+++ b/integration-tests/oracle/main-0012-wide-table-write.hwf
@@ -0,0 +1,245 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0012-wide-table-write</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>A Table input to Table output copy of a 1000-column table, the
most Oracle allows. To bind a string, Oracle needs to know whether its column
is VARCHAR2, NVARCHAR2 or CLOB, and Hop asks the driver for the statement's
parameter metadata. The driver answers by parsing the INSERT with a recursive
parser whose depth grows with the number of columns, which used to overflow the
1 MB stack of a transform thread on Linux with a StackOverflowError (#9420).
The table mixes VARCHAR2, [...]
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/11 12:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/11 12:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Init tables</name>
+ <description>Source and target have the same 1000 columns: ID, the CLOB
C0001, an NVARCHAR2 for every tenth column and a VARCHAR2 for the rest. Row 1
fills every column, with national text in the NVARCHAR2 columns and a 64000
character CLOB; row 2 is all nulls; row 3 has short values
throughout.</description>
+ <type>SQL</type>
+ <attributes/>
+ <sql>DECLARE
+ table_missing EXCEPTION;
+ PRAGMA EXCEPTION_INIT(table_missing, -942);
+ -- Written as escapes so nothing depends on the client's NLS settings.
+ japanese_literal CONSTANT VARCHAR2(100) :=
'UNISTR(''\65E5\672C\8A9E\30C6\30AD\30B9\30C8'')';
+ column_name VARCHAR2(10);
+ ddl CLOB := 'ID NUMBER(10)';
+ column_list CLOB := 'ID';
+ row_one CLOB := '1';
+ row_three CLOB := '3';
+ long_text CLOB;
+
+ FUNCTION column_type(i PLS_INTEGER) RETURN VARCHAR2 IS
+ BEGIN
+ IF i = 1 THEN
+ RETURN 'CLOB';
+ ELSIF MOD(i, 10) = 0 THEN
+ RETURN 'NVARCHAR2(20)';
+ ELSE
+ RETURN 'VARCHAR2(20)';
+ END IF;
+ END;
+BEGIN
+ BEGIN
+ EXECUTE IMMEDIATE 'DROP TABLE HOP_WIDE_SOURCE PURGE';
+ EXCEPTION
+ WHEN table_missing THEN NULL;
+ END;
+ BEGIN
+ EXECUTE IMMEDIATE 'DROP TABLE HOP_WIDE_TARGET PURGE';
+ EXCEPTION
+ WHEN table_missing THEN NULL;
+ END;
+
+ FOR i IN 1..999 LOOP
+ column_name := 'C' || TO_CHAR(i, 'FM0000');
+ ddl := ddl || ', ' || column_name || ' ' || column_type(i);
+ column_list := column_list || ', ' || column_name;
+ IF i = 1 THEN
+ row_one := row_one || ', :long_text';
+ row_three := row_three || ', TO_CLOB(''short clob'')';
+ ELSIF MOD(i, 10) = 0 THEN
+ row_one := row_one || ', ' || japanese_literal;
+ row_three := row_three || ', ' || japanese_literal;
+ ELSE
+ row_one := row_one || ', ''v' || i || '''';
+ row_three := row_three || ', ''w' || i || '''';
+ END IF;
+ END LOOP;
+
+ EXECUTE IMMEDIATE 'CREATE TABLE HOP_WIDE_SOURCE (' || ddl || ')';
+ EXECUTE IMMEDIATE 'CREATE TABLE HOP_WIDE_TARGET (' || ddl || ')';
+
+ -- 64000 characters: past the 4000 a VARCHAR2 holds, so it only arrives if
the CLOB column was
+ -- recognised and the value streamed.
+ long_text := TO_CLOB(RPAD('0123456789', 4000, '0123456789'));
+ FOR i IN 1..4 LOOP
+ long_text := long_text || long_text;
+ END LOOP;
+
+ EXECUTE IMMEDIATE 'INSERT INTO HOP_WIDE_SOURCE (' || column_list || ')
VALUES (' || row_one || ')'
+ USING long_text;
+ EXECUTE IMMEDIATE 'INSERT INTO HOP_WIDE_SOURCE (ID) VALUES (2)';
+ EXECUTE IMMEDIATE 'INSERT INTO HOP_WIDE_SOURCE (' || column_list || ')
VALUES (' || row_three || ')';
+ COMMIT;
+END;</sql>
+ <useVariableSubstitution>F</useVariableSubstitution>
+ <sqlfromfile>F</sqlfromfile>
+ <sqlfilename/>
+ <sendOneStatement>T</sendOneStatement>
+ <connection>oracle-service-name</connection>
+ <parallel>N</parallel>
+ <xloc>192</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0012-wide-table-write.hpl</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <clear_files>N</clear_files>
+ <clear_rows>N</clear_rows>
+ <create_parent_folder>N</create_parent_folder>
+ <exec_per_row>N</exec_per_row>
+ <filename>${PROJECT_HOME}/0012-wide-table-write.hpl</filename>
+ <logext/>
+ <logfile/>
+ <loglevel>Basic</loglevel>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <params_from_previous>N</params_from_previous>
+ <run_configuration>local</run_configuration>
+ <set_append_logfile>N</set_append_logfile>
+ <set_logfile>N</set_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <parallel>N</parallel>
+ <xloc>384</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>check what was stored</name>
+ <description/>
+ <type>SQL</type>
+ <attributes/>
+ <sql>DECLARE
+ found NUMBER;
+ column_name VARCHAR2(10);
+ column_list CLOB := 'ID';
+BEGIN
+ SELECT COUNT(*) INTO found FROM HOP_WIDE_TARGET;
+ IF found != 3 THEN
+ RAISE_APPLICATION_ERROR(-20120, 'expected 3 rows but found ' || found);
+ END IF;
+
+ -- Every column but the CLOB, compared both ways. MINUS treats two nulls as
equal, so this also
+ -- covers the row of nulls, and a national character turned into a question
mark shows up as a
+ -- difference.
+ FOR i IN 2..999 LOOP
+ column_name := 'C' || TO_CHAR(i, 'FM0000');
+ column_list := column_list || ', ' || column_name;
+ END LOOP;
+ EXECUTE IMMEDIATE
+ 'SELECT COUNT(*) FROM ('
+ || '(SELECT ' || column_list || ' FROM HOP_WIDE_SOURCE MINUS SELECT ' ||
column_list || ' FROM HOP_WIDE_TARGET)'
+ || ' UNION ALL '
+ || '(SELECT ' || column_list || ' FROM HOP_WIDE_TARGET MINUS SELECT ' ||
column_list || ' FROM HOP_WIDE_SOURCE))'
+ INTO found;
+ IF found != 0 THEN
+ RAISE_APPLICATION_ERROR(-20121,
+ found || ' rows differ between source and target in the VARCHAR2 and
NVARCHAR2 columns');
+ END IF;
+
+ -- The CLOB has to arrive whole: a value cut to 4000 would still leave the
row behind.
+ SELECT COUNT(*) INTO found
+ FROM HOP_WIDE_SOURCE s
+ JOIN HOP_WIDE_TARGET t ON t.ID = s.ID
+ WHERE (s.C0001 IS NULL AND t.C0001 IS NULL)
+ OR DBMS_LOB.COMPARE(s.C0001, t.C0001) = 0;
+ IF found != 3 THEN
+ RAISE_APPLICATION_ERROR(-20122, 'the CLOB column did not survive the
write');
+ END IF;
+
+ SELECT DBMS_LOB.GETLENGTH(C0001) INTO found FROM HOP_WIDE_TARGET WHERE ID =
1;
+ IF found != 64000 THEN
+ RAISE_APPLICATION_ERROR(-20123, 'the CLOB value should be 64000 characters
but is ' || found);
+ END IF;
+END;</sql>
+ <useVariableSubstitution>F</useVariableSubstitution>
+ <sqlfromfile>F</sqlfromfile>
+ <sqlfilename/>
+ <sendOneStatement>T</sendOneStatement>
+ <connection>oracle-service-name</connection>
+ <parallel>N</parallel>
+ <xloc>576</xloc>
+ <yloc>48</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Init tables</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Init tables</from>
+ <to>0012-wide-table-write.hpl</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ <hop>
+ <from>0012-wide-table-write.hpl</from>
+ <to>check what was stored</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/plugins/databases/oracle/src/main/java/org/apache/hop/databases/oracle/OraclePreparedStatementBinding.java
b/plugins/databases/oracle/src/main/java/org/apache/hop/databases/oracle/OraclePreparedStatementBinding.java
index 8dcdbe4d00..b53b3125a1 100644
---
a/plugins/databases/oracle/src/main/java/org/apache/hop/databases/oracle/OraclePreparedStatementBinding.java
+++
b/plugins/databases/oracle/src/main/java/org/apache/hop/databases/oracle/OraclePreparedStatementBinding.java
@@ -67,6 +67,15 @@ final class OraclePreparedStatementBinding implements
IValueBinding {
private static final Map<PreparedStatement, int[]> COLUMN_TYPES =
Collections.synchronizedMap(new WeakHashMap<>());
+ /**
+ * Stack of the thread that describes a statement's parameters. Reserved,
not committed: a parse
+ * that needs little stack touches little. A 1000-column INSERT, the most
Oracle allows in a
+ * table, needs well under this.
+ */
+ static final long DESCRIBE_STACK_SIZE = 64L * 1024 * 1024;
+
+ static final String DESCRIBE_THREAD_NAME = "Oracle bind parameter describe";
+
private OraclePreparedStatementBinding() {}
/**
@@ -135,8 +144,34 @@ final class OraclePreparedStatementBinding implements
IValueBinding {
* answer -- a driver too old to describe binds, a statement its parser does
not understand, a
* parameter it has no type for -- is recorded as unknown, so the question
is asked once per
* statement whatever the outcome.
+ *
+ * <p>The question is asked on a thread of its own with a large stack. The
Oracle driver answers
+ * it by parsing the SQL with a recursive parser whose depth grows with the
number of columns: an
+ * INSERT of a few hundred columns overflows the 1 MB stack a transform
thread gets by default on
+ * Linux. The calling thread waits for the answer, so the statement and its
connection are never
+ * used by two threads at once.
*/
- private static int[] describeColumns(PreparedStatement preparedStatement) {
+ static int[] describeColumns(PreparedStatement preparedStatement) {
+ int[][] types = {new int[0]};
+ Thread describer =
+ new Thread(
+ null,
+ () -> types[0] = describeColumnsOnThisThread(preparedStatement),
+ DESCRIBE_THREAD_NAME,
+ DESCRIBE_STACK_SIZE);
+ describer.setDaemon(true);
+ describer.start();
+ try {
+ describer.join();
+ } catch (InterruptedException e) {
+ // The transform is being stopped: the values go in the way Hop always
wrote them.
+ Thread.currentThread().interrupt();
+ return new int[0];
+ }
+ return types[0];
+ }
+
+ private static int[] describeColumnsOnThisThread(PreparedStatement
preparedStatement) {
try {
ParameterMetaData metaData = preparedStatement.getParameterMetaData();
int[] types = new int[metaData.getParameterCount()];
@@ -144,7 +179,9 @@ final class OraclePreparedStatementBinding implements
IValueBinding {
types[i] = columnType(metaData, i + 1);
}
return types;
- } catch (SQLException | RuntimeException e) {
+ } catch (SQLException | RuntimeException | StackOverflowError e) {
+ // A statement too wide even for the large stack is written the way Hop
always wrote it. The
+ // parser's state lives in this call only, so nothing is left behind by
unwinding it.
return new int[0];
}
}
diff --git
a/plugins/databases/oracle/src/test/java/org/apache/hop/databases/oracle/OraclePreparedStatementBindingTest.java
b/plugins/databases/oracle/src/test/java/org/apache/hop/databases/oracle/OraclePreparedStatementBindingTest.java
index c5dca736ee..d9b8a1d4b2 100644
---
a/plugins/databases/oracle/src/test/java/org/apache/hop/databases/oracle/OraclePreparedStatementBindingTest.java
+++
b/plugins/databases/oracle/src/test/java/org/apache/hop/databases/oracle/OraclePreparedStatementBindingTest.java
@@ -19,6 +19,7 @@ package org.apache.hop.databases.oracle;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
@@ -34,6 +35,8 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.database.DatabaseMeta;
import org.apache.hop.core.database.types.DatabaseTypeMapper;
@@ -56,6 +59,11 @@ class OraclePreparedStatementBindingTest {
private static final String LOG_FIELD = "LOG_FIELD";
+ /** Recursion depth that overflows {@link #SMALL_TRANSFORM_STACK} but not
the describe stack. */
+ private static final int DEEP_PARSE = 50_000;
+
+ private static final long SMALL_TRANSFORM_STACK = 256L * 1024;
+
private OracleDatabaseMeta oracleDatabaseMeta;
private PreparedStatement preparedStatementMock;
private ParameterMetaData parameterMetaData;
@@ -234,6 +242,69 @@ class OraclePreparedStatementBindingTest {
verify(preparedStatementMock, times(2)).setString(anyInt(), any());
}
+ /**
+ * The Oracle driver describes parameters by parsing the SQL recursively,
and a wide enough
+ * statement overflows even a large stack. That is an Error, not an
exception, but it must still
+ * leave the value written the way Hop always wrote it rather than kill the
transform -- and not
+ * be retried for every row.
+ */
+ @Test
+ void testAStackOverflowWhileDescribingFallsBackToSetString() throws
Exception {
+ when(preparedStatementMock.getParameterMetaData()).thenThrow(new
StackOverflowError());
+ IValueMeta valueMeta = new ValueMetaString(LOG_FIELD, 20, 0);
+ write(valueMeta, "one");
+ write(valueMeta, "two");
+
+ verify(preparedStatementMock, times(1)).getParameterMetaData();
+ verify(preparedStatementMock, times(1)).setString(1, "one");
+ verify(preparedStatementMock, times(1)).setString(1, "two");
+ }
+
+ /**
+ * The driver's parse of a 1000-column INSERT is deeper than the stack of a
transform thread on
+ * Linux (1 MB by default). The description is asked on a thread of its own
with a large stack, so
+ * a transform thread with a small stack still gets the column types.
+ */
+ @Test
+ void testDescribingNeedsNoStackFromTheTransformThread() throws Exception {
+ List<String> describedOn = new ArrayList<>();
+ when(parameterMetaData.getParameterType(1))
+ .thenAnswer(
+ invocation -> {
+ describedOn.add(Thread.currentThread().getName());
+ // As deep as the driver's parser goes on a very wide statement.
+ return recurse(DEEP_PARSE) > 0 ? Types.NVARCHAR : Types.VARCHAR;
+ });
+ when(parameterMetaData.getParameterTypeName(1)).thenReturn("NVARCHAR2");
+
+ Throwable[] failure = {null};
+ Thread transform =
+ new Thread(
+ null,
+ () -> {
+ try {
+ write(new ValueMetaString(LOG_FIELD, 20, 0), "data");
+ } catch (Throwable t) {
+ failure[0] = t;
+ }
+ },
+ "Table output.0",
+ SMALL_TRANSFORM_STACK);
+ transform.start();
+ transform.join();
+
+ assertNull(failure[0], "Describing the statement must not overflow the
transform thread");
+ verify(preparedStatementMock, times(1)).setNString(1, "data");
+ assertEquals(
+ List.of(OraclePreparedStatementBinding.DESCRIBE_THREAD_NAME),
+ describedOn,
+ "The parameters are described on the large-stack thread");
+ }
+
+ private static int recurse(int depth) {
+ return depth == 0 ? 1 : 1 + recurse(depth - 1);
+ }
+
/**
* Reading was never broken, so the binding declines it and the caller falls
back to the value
* type's own handling.