This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new a58120f264 Fix#9286 Vector CSV round trip fails with comma separator
and parse accepts Java literals (#9365)
a58120f264 is described below
commit a58120f264653613592038841a11d6152ed87140
Author: Lance <[email protected]>
AuthorDate: Sun Nov 15 14:47:37 2026 +0800
Fix#9286 Vector CSV round trip fails with comma separator and parse
accepts Java literals (#9365)
* Fix Vector CSV round trip fails with comma separator and parse accepts
Java literals
Signed-off-by: leehaut <[email protected]>
* Fix Vector CSV round trip fails with comma separator and parse accepts
Java literals
Signed-off-by: leehaut <[email protected]>
* Fix Vector CSV round trip fails with comma separator and parse accepts
Java literals
Signed-off-by: leehaut <[email protected]>
* Fix Vector CSV round trip fails with comma separator and parse accepts
Java literals
Signed-off-by: leehaut <[email protected]>
---------
Signed-off-by: leehaut <[email protected]>
---
.../pages/snippets/hop-concepts/data-types.adoc | 3 +
.../0126-vector-csv-round-trip-read-back.hpl | 161 ++++++++++++++++++
.../transforms/0126-vector-csv-round-trip.hpl | 181 +++++++++++++++++++++
.../datasets/golden-vector-csv-round-trip.csv | 6 +
.../transforms/main-0126-vector-csv-round-trip.hwf | 115 +++++++++++++
.../dataset/golden-vector-csv-round-trip.json | 23 +++
.../0126-vector-csv-round-trip-read-back UNIT.json | 32 ++++
.../transforms/textfileoutput/TextFileOutput.java | 22 +--
.../textfileoutput/TextFileOutputTest.java | 110 +++++++++++++
.../org/apache/hop/vector/ValueMetaVector.java | 37 ++++-
.../org/apache/hop/vector/ValueMetaVectorTest.java | 25 +++
11 files changed, 701 insertions(+), 14 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/snippets/hop-concepts/data-types.adoc
b/docs/hop-user-manual/modules/ROOT/pages/snippets/hop-concepts/data-types.adoc
index 5ee1754cea..0a43922c96 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/snippets/hop-concepts/data-types.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/snippets/hop-concepts/data-types.adoc
@@ -165,6 +165,9 @@ The canonical text form is a bracketed, comma separated
list of numbers:
```
That form is both valid JSON and the literal syntax pgvector accepts, so a
vector survives a round trip through a text file, a Data Grid, a JSON document
or a database column without a conversion step in between.
+Each element must be a plain decimal or scientific number (`0.1`, `-2`,
`1.5e-3`).
+Java float literals (`1f`, `2d`, `0x1p3`) and non-finite values (`NaN`,
`Infinity`) are rejected.
+Text File Output quotes Vector values when they contain the field separator,
so a comma-separated file can be read back.
*Conversions*
diff --git
a/integration-tests/transforms/0126-vector-csv-round-trip-read-back.hpl
b/integration-tests/transforms/0126-vector-csv-round-trip-read-back.hpl
new file mode 100644
index 0000000000..654115b248
--- /dev/null
+++ b/integration-tests/transforms/0126-vector-csv-round-trip-read-back.hpl
@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <pipeline_version/>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0126-vector-csv-round-trip-read-back</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Issue #9286: reads the comma-separated file written by
0126-vector-csv-round-trip with CSV File Input, field type Vector, then
converts each vector back to its canonical text so the unit test can compare it
with the data set.</description>
+ <extended_description/>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/11/14 11:30:00.000</created_date>
+ <modified_date>2026/11/14 11:30:00.000</modified_date>
+ </info>
+ <transform>
+ <type>CSVInput</type>
+ <name>Read vectors</name>
+ <filename>${java.io.tmpdir}/0126/vector-csv-round-trip.csv</filename>
+ <filename_field/>
+ <include_filename>N</include_filename>
+ <rownum_field/>
+ <header>Y</header>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <buffer_size>50000</buffer_size>
+ <lazy_conversion>N</lazy_conversion>
+ <add_filename_result>N</add_filename_result>
+ <parallel>N</parallel>
+ <encoding>UTF-8</encoding>
+ <newline_possible>N</newline_possible>
+ <schemaDefinition/>
+ <ignoreFields>N</ignoreFields>
+ <namingScheme/>
+ <fields>
+ <field>
+ <name>id</name>
+ <length>-1</length>
+ <type>String</type>
+ <format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ </field>
+ <field>
+ <name>embedding</name>
+ <length>-1</length>
+ <type>Vector</type>
+ <format/>
+ <trim_type>none</trim_type>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ </field>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>SelectValues</type>
+ <name>Back to text</name>
+ <fields>
+ <select_unspecified>N</select_unspecified>
+ <meta>
+ <name>embedding</name>
+ <rename>embedding</rename>
+ <type>String</type>
+ <length>-2</length>
+ <precision>-2</precision>
+ <storage_type/>
+ <conversion_mask/>
+ <date_format_lenient>N</date_format_lenient>
+ <date_format_locale/>
+ <date_format_timezone/>
+ <lenient_string_to_number>N</lenient_string_to_number>
+ <decimal_symbol/>
+ <grouping_symbol/>
+ <currency_symbol/>
+ <encoding/>
+ </meta>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>352</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>Dummy</type>
+ <name>OUTPUT</name>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>576</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>Read vectors</from>
+ <to>Back to text</to>
+ <enabled>Y</enabled>
+ </hop>
+ <hop>
+ <from>Back to text</from>
+ <to>OUTPUT</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git a/integration-tests/transforms/0126-vector-csv-round-trip.hpl
b/integration-tests/transforms/0126-vector-csv-round-trip.hpl
new file mode 100644
index 0000000000..b477dc7751
--- /dev/null
+++ b/integration-tests/transforms/0126-vector-csv-round-trip.hpl
@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <pipeline_version/>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <pipeline_type>Normal</pipeline_type>
+ <pipeline_status>0</pipeline_status>
+ <parameters/>
+ <name>0126-vector-csv-round-trip</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Issue #9286: writes Vector values with Text File Output using
a comma separator and a double-quote enclosure. A vector whose canonical text
contains the separator, such as [0.1,0.2,0.3], is quoted so the line is
a,"[0.1,0.2,0.3]". A one-element vector has no separator and stays unquoted.
Integer components are normalised, so [1,2,3] is written as [1.0,2.0,3.0], and
1.5e-3 is written as 0.0015. 0126-vector-csv-round-trip-read-back reads the
file with CSV File Input as Vec [...]
+ <extended_description/>
+ <created_user>-</created_user>
+ <modified_user>-</modified_user>
+ <created_date>2026/11/14 11:30:00.000</created_date>
+ <modified_date>2026/11/14 11:30:00.000</modified_date>
+ </info>
+ <transform>
+ <type>DataGrid</type>
+ <name>Vectors</name>
+ <data>
+ <line>
+ <item>a</item>
+ <item>[0.1,0.2,0.3]</item>
+ </line>
+ <line>
+ <item>b</item>
+ <item>[-0.5,0.25]</item>
+ </line>
+ <line>
+ <item>c</item>
+ <item>[1,2,3]</item>
+ </line>
+ <line>
+ <item>d</item>
+ <item>[0.5]</item>
+ </line>
+ <line>
+ <item>e</item>
+ <item>[-0.5,1.5e-3]</item>
+ </line>
+ </data>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>String</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ <field>
+ <name>embedding</name>
+ <type>Vector</type>
+ <format/>
+ <length>-1</length>
+ <precision>-1</precision>
+ <currency/>
+ <decimal/>
+ <group/>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>128</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <transform>
+ <type>TextFileOutput</type>
+ <name>Write comma-separated vectors</name>
+ <separator>,</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>Y</header>
+ <footer>N</footer>
+ <format>UNIX</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>${java.io.tmpdir}/0126/vector-csv-round-trip</name>
+ <do_not_open_new_file_init>N</do_not_open_new_file_init>
+ <extension>csv</extension>
+ <append>N</append>
+ <split>N</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery>0</splitevery>
+ </file>
+ <fields>
+ <field>
+ <name>id</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ <field>
+ <name>embedding</name>
+ <type>Vector</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <distribute>Y</distribute>
+ <copies>1</copies>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ <description/>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <attributes/>
+ </transform>
+ <order>
+ <hop>
+ <from>Vectors</from>
+ <to>Write comma-separated vectors</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <notepads/>
+ <attributes/>
+ <transform_error_handling/>
+</pipeline>
diff --git
a/integration-tests/transforms/datasets/golden-vector-csv-round-trip.csv
b/integration-tests/transforms/datasets/golden-vector-csv-round-trip.csv
new file mode 100644
index 0000000000..e4d27eb93e
--- /dev/null
+++ b/integration-tests/transforms/datasets/golden-vector-csv-round-trip.csv
@@ -0,0 +1,6 @@
+id,embedding
+a,"[0.1,0.2,0.3]"
+b,"[-0.5,0.25]"
+c,"[1.0,2.0,3.0]"
+d,[0.5]
+e,"[-0.5,0.0015]"
diff --git a/integration-tests/transforms/main-0126-vector-csv-round-trip.hwf
b/integration-tests/transforms/main-0126-vector-csv-round-trip.hwf
new file mode 100644
index 0000000000..ce76d40b1b
--- /dev/null
+++ b/integration-tests/transforms/main-0126-vector-csv-round-trip.hwf
@@ -0,0 +1,115 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0126-vector-csv-round-trip</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description/>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2026/11/14 11:30:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2026/11/14 11:30:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>START</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <intervalSeconds>0</intervalSeconds>
+ <intervalMinutes>60</intervalMinutes>
+ <hour>12</hour>
+ <minutes>0</minutes>
+ <weekDay>1</weekDay>
+ <DayOfMonth>1</DayOfMonth>
+ <parallel>N</parallel>
+ <xloc>96</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0126-vector-csv-round-trip</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0126-vector-csv-round-trip.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <clear_rows>N</clear_rows>
+ <clear_files>N</clear_files>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <set_append_logfile>N</set_append_logfile>
+ <wait_until_finished>Y</wait_until_finished>
+ <follow_abort_remote>N</follow_abort_remote>
+ <create_parent_folder>N</create_parent_folder>
+ <run_configuration>local</run_configuration>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <parallel>N</parallel>
+ <xloc>288</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>0126-vector-csv-round-trip-read-back UNIT</name>
+ <description/>
+ <type>RunPipelineTests</type>
+ <attributes/>
+ <test_names>
+ <test_name>
+ <name>0126-vector-csv-round-trip-read-back UNIT</name>
+ </test_name>
+ </test_names>
+ <parallel>N</parallel>
+ <xloc>544</xloc>
+ <yloc>80</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>START</from>
+ <to>0126-vector-csv-round-trip</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>0126-vector-csv-round-trip</from>
+ <to>0126-vector-csv-round-trip-read-back UNIT</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/transforms/metadata/dataset/golden-vector-csv-round-trip.json
b/integration-tests/transforms/metadata/dataset/golden-vector-csv-round-trip.json
new file mode 100644
index 0000000000..43461e5eac
--- /dev/null
+++
b/integration-tests/transforms/metadata/dataset/golden-vector-csv-round-trip.json
@@ -0,0 +1,23 @@
+{
+ "base_filename": "golden-vector-csv-round-trip.csv",
+ "name": "golden-vector-csv-round-trip",
+ "description": "Canonical text after a Vector is written with a comma
separator and read back by CSV File Input (#9286).",
+ "dataset_fields": [
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "id",
+ "field_format": ""
+ },
+ {
+ "field_comment": "",
+ "field_length": -1,
+ "field_type": 2,
+ "field_precision": -1,
+ "field_name": "embedding",
+ "field_format": ""
+ }
+ ]
+}
diff --git
a/integration-tests/transforms/metadata/unit-test/0126-vector-csv-round-trip-read-back
UNIT.json
b/integration-tests/transforms/metadata/unit-test/0126-vector-csv-round-trip-read-back
UNIT.json
new file mode 100644
index 0000000000..5f989324cd
--- /dev/null
+++
b/integration-tests/transforms/metadata/unit-test/0126-vector-csv-round-trip-read-back
UNIT.json
@@ -0,0 +1,32 @@
+{
+ "variableValues": [],
+ "database_replacements": [],
+ "autoOpening": false,
+ "basePath": "",
+ "golden_data_sets": [
+ {
+ "field_mappings": [
+ {
+ "transform_field": "id",
+ "data_set_field": "id"
+ },
+ {
+ "transform_field": "embedding",
+ "data_set_field": "embedding"
+ }
+ ],
+ "field_order": [
+ "id"
+ ],
+ "transform_name": "OUTPUT",
+ "data_set_name": "golden-vector-csv-round-trip"
+ }
+ ],
+ "input_data_sets": [],
+ "name": "0126-vector-csv-round-trip-read-back UNIT",
+ "description": "Reads the comma-separated file written by
0126-vector-csv-round-trip as Vector and compares the canonical text with the
data set",
+ "trans_test_tweaks": [],
+ "persist_filename": "",
+ "pipeline_filename": "./0126-vector-csv-round-trip-read-back.hpl",
+ "test_type": "UNIT_TEST"
+}
diff --git
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutput.java
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutput.java
index 7f594700dd..cdecd39928 100644
---
a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutput.java
+++
b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutput.java
@@ -687,16 +687,18 @@ public class TextFileOutput extends
BaseTransform<TextFileOutputMeta, TextFileOu
List<Integer> enclosures = null;
boolean writeEnclosures = false;
- if (v.isString()) {
- // Quoting stays tied to the Right pad fields checkbox.
isPaddingFields() is also true
- // for saved transforms that never selected that checkbox, and using
it here would drop
- // enclosures from existing files.
- if (meta.isEnclosureForced() && !meta.getFileSettings().isPadded()) {
- writeEnclosures = true;
- } else if (!meta.isEnclosureFixDisabled()
- && containsSeparatorOrEnclosure(str, data.binarySeparator,
data.binaryEnclosure)) {
- writeEnclosures = true;
- }
+ // Forced enclosure stays a String-only option. isPaddingFields() is
also true for saved
+ // transforms that never selected that checkbox, and using it here
would drop enclosures
+ // from existing files.
+ if (v.isString() && meta.isEnclosureForced() &&
!meta.getFileSettings().isPadded()) {
+ writeEnclosures = true;
+ } else if ((v.isString() || v.getType() == IValueMeta.TYPE_VECTOR)
+ && !meta.isEnclosureFixDisabled()
+ && containsSeparatorOrEnclosure(str, data.binarySeparator,
data.binaryEnclosure)) {
+ // Quote String and Vector when the rendered text embeds the
separator or enclosure.
+ // Vector's canonical form [0.1,0.2,0.3] needs this for a
comma-separated round trip.
+ // Other types (JSON, Binary, Number, Date) keep their previous
unquoted output.
+ writeEnclosures = true;
}
if (writeEnclosures) {
diff --git
a/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutputTest.java
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutputTest.java
index 56ac24bd5a..f212462ef0 100644
---
a/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutputTest.java
+++
b/plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/textfileoutput/TextFileOutputTest.java
@@ -45,6 +45,7 @@ import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.row.RowMeta;
+import org.apache.hop.core.row.value.ValueMetaBase;
import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.variables.IVariables;
@@ -840,6 +841,115 @@ class TextFileOutputTest {
assertEquals("abcdefghij", writeStringField("abcdefghijklmnop", 10, true,
true));
}
+ @Test
+ void enclosesVectorWhenValueContainsSeparator() throws Exception {
+ // Vector's canonical form embeds commas; quoting is required for CSV
round trip (#9286).
+ assertEquals(
+ "a,\"[0.1,0.2,0.3]\"",
+ writeIdAndRenderedField(",", IValueMeta.TYPE_VECTOR, "[0.1,0.2,0.3]",
false));
+ }
+
+ @Test
+ void doesNotEncloseVectorWhenSeparatorIsAbsentFromValue() throws Exception {
+ assertEquals(
+ "a;[0.1,0.2,0.3]",
+ writeIdAndRenderedField(";", IValueMeta.TYPE_VECTOR, "[0.1,0.2,0.3]",
false));
+ }
+
+ @Test
+ void enclosureFixDisabledSkipsQuotingVectorWithSeparator() throws Exception {
+ assertEquals(
+ "a,[0.1,0.2,0.3]",
+ writeIdAndRenderedField(",", IValueMeta.TYPE_VECTOR, "[0.1,0.2,0.3]",
true));
+ }
+
+ @Test
+ void stillEnclosesStringWhenValueContainsSeparator() throws Exception {
+ assertEquals(
+ "a,\"hello,world\"",
+ writeIdAndRenderedField(",", IValueMeta.TYPE_STRING, "hello,world",
false));
+ }
+
+ @Test
+ void doesNotEncloseNumberWhenFormattedValueContainsSeparator() throws
Exception {
+ // Number/Date/JSON/Binary must keep pre-#9286 output (no auto-quote on
separator).
+ assertEquals(
+ "a,1,234.50", writeIdAndRenderedField(",", IValueMeta.TYPE_NUMBER,
"1,234.50", false));
+ }
+
+ @Test
+ void doesNotEncloseDateWhenFormattedValueContainsSeparator() throws
Exception {
+ assertEquals(
+ "a,Jan 1, 1970", writeIdAndRenderedField(",", IValueMeta.TYPE_DATE,
"Jan 1, 1970", false));
+ }
+
+ @Test
+ void doesNotEncloseJsonWhenValueContainsEnclosure() throws Exception {
+ assertEquals(
+ "a;{\"a\" : 1}", writeIdAndRenderedField(";", IValueMeta.TYPE_JSON,
"{\"a\" : 1}", false));
+ }
+
+ @Test
+ void doesNotEncloseBinaryWhenValueContainsEnclosure() throws Exception {
+ assertEquals(
+ "a;bin\"ary", writeIdAndRenderedField(";", IValueMeta.TYPE_BINARY,
"bin\"ary", false));
+ }
+
+ /**
+ * Writes {@code id} as a String and a second field whose value meta reports
{@code fieldType} and
+ * whose {@code getBinaryString} returns {@code rendered} — the write path
Vector (and others)
+ * take once formatting is done.
+ */
+ private String writeIdAndRenderedField(
+ String separator, int fieldType, String rendered, boolean
enclosureFixDisabled)
+ throws Exception {
+ TextFileOutputMeta meta = new TextFileOutputMeta();
+ meta.setEncoding(Const.UTF_8);
+ meta.setSeparator(separator);
+ meta.setEnclosure("\"");
+ meta.setEnclosureForced(false);
+ meta.setEnclosureFixDisabled(enclosureFixDisabled);
+ meta.getFileSettings().setDoNotPadFields(false);
+ meta.getFileSettings().setPadded(false);
+
+ TextFileOutputData data = new TextFileOutputData();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ data.writer = baos;
+ data.binarySeparator = separator.getBytes(StandardCharsets.UTF_8);
+ data.binaryEnclosure = "\"".getBytes(StandardCharsets.UTF_8);
+ data.binaryNewline = "\n".getBytes(StandardCharsets.UTF_8);
+
+ IValueMeta renderedMeta =
+ new ValueMetaBase("field", fieldType) {
+ @Override
+ public byte[] getBinaryString(Object object) {
+ return object == null ? null :
object.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ @Override
+ public boolean isNull(Object object) {
+ return object == null;
+ }
+ };
+
+ RowMeta rowMeta = new RowMeta();
+ rowMeta.addValueMeta(new ValueMetaString("id"));
+ rowMeta.addValueMeta(renderedMeta);
+
+ TextFileOutput transform =
+ new TextFileOutput(
+ transformMockHelper.transformMeta,
+ meta,
+ data,
+ 0,
+ transformMockHelper.pipelineMeta,
+ transformMockHelper.pipeline);
+ transform.writeRow(rowMeta, new Object[] {"a", rendered});
+ String written = baos.toString(StandardCharsets.UTF_8);
+ assertTrue(written.endsWith("\n"), written);
+ return written.substring(0, written.length() - 1);
+ }
+
private String writeStringField(String value, int length, boolean doNotPad,
boolean rightPad)
throws Exception {
TextFileOutputMeta meta = new TextFileOutputMeta();
diff --git
a/plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java
b/plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java
index bb1e30e28f..99c4674586 100644
---
a/plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java
+++
b/plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java
@@ -29,6 +29,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.util.Arrays;
+import java.util.regex.Pattern;
import org.apache.hop.core.database.DatabaseMeta;
import org.apache.hop.core.database.IDatabase;
import org.apache.hop.core.exception.HopDatabaseException;
@@ -42,10 +43,11 @@ import org.apache.hop.core.row.value.ValueMetaPlugin;
/**
* A dense floating point vector, as produced by an embedding model and
consumed by a vector store.
*
- * <p>The canonical text form is a bracketed, comma separated list of numbers:
{@code
+ * <p>The canonical text form is a bracketed, comma separated list of plain
decimal numbers: {@code
* [0.1,0.2,0.3]}. That form is both valid JSON and the literal syntax
pgvector accepts, so a vector
* survives a round trip through a text file, a Data Grid, a JSON document or
a database column
- * without a conversion step in between.
+ * without a conversion step in between. Parsing rejects Java float literals
and non-finite values
+ * so the text stays in that portable form.
*
* <p>The field's {@code length} carries the number of dimensions, the way it
carries the size of a
* String. A length of zero or less means the dimension is unspecified. Hop
never pads or truncates
@@ -61,6 +63,14 @@ public class ValueMetaVector extends ValueMetaBase {
public static final int TYPE_VECTOR = IValueMeta.TYPE_VECTOR;
+ /**
+ * Plain decimal / scientific token accepted by {@link #parse(String)}.
Rejects Java float
+ * suffixes ({@code 1f}, {@code 2d}), hex floats ({@code 0x1p3}), and named
constants ({@code
+ * NaN}, {@code Infinity}).
+ */
+ private static final Pattern PLAIN_DECIMAL =
+ Pattern.compile("[+-]?(?:\\d+\\.\\d*|\\.\\d+|\\d+)(?:[eE][+-]?\\d+)?");
+
public ValueMetaVector() {
super(null, TYPE_VECTOR);
}
@@ -140,9 +150,13 @@ public class ValueMetaVector extends ValueMetaBase {
* Parse the canonical text form. Surrounding brackets are optional and
whitespace is ignored, so
* both {@code [0.1, 0.2]} and {@code 0.1,0.2} are accepted.
*
+ * <p>Each element must be a plain decimal or scientific number ({@code
0.1}, {@code -2}, {@code
+ * 1.5e-3}). Java float literals ({@code 1f}, {@code 2d}, {@code 0x1p3}) and
non-finite values
+ * ({@code NaN}, {@code Infinity}) are rejected so the text stays valid JSON
and pgvector input.
+ *
* @param string the text to parse, or null
* @return the parsed vector, or null when the input is null or blank
- * @throws HopValueException when the text is not a list of numbers
+ * @throws HopValueException when the text is not a list of plain decimal
numbers
*/
public static float[] parse(String string) throws HopValueException {
if (string == null) {
@@ -166,8 +180,23 @@ public class ValueMetaVector extends ValueMetaBase {
throw new HopValueException(
"Empty element at position " + i + " while parsing a vector from
'" + string + "'");
}
+ if (!PLAIN_DECIMAL.matcher(part).matches()) {
+ throw new HopValueException(
+ "'" + part + "' at position " + i + " is not a number in vector '"
+ string + "'");
+ }
try {
- vector[i] = Float.parseFloat(part);
+ float value = Float.parseFloat(part);
+ if (Float.isNaN(value) || Float.isInfinite(value)) {
+ throw new HopValueException(
+ "'"
+ + part
+ + "' at position "
+ + i
+ + " is not a finite number in vector '"
+ + string
+ + "'");
+ }
+ vector[i] = value;
} catch (NumberFormatException e) {
throw new HopValueException(
"'" + part + "' at position " + i + " is not a number in vector '"
+ string + "'", e);
diff --git
a/plugins/valuetypes/vector/src/test/java/org/apache/hop/vector/ValueMetaVectorTest.java
b/plugins/valuetypes/vector/src/test/java/org/apache/hop/vector/ValueMetaVectorTest.java
index 2afdcf868b..d2693c63b6 100644
---
a/plugins/valuetypes/vector/src/test/java/org/apache/hop/vector/ValueMetaVectorTest.java
+++
b/plugins/valuetypes/vector/src/test/java/org/apache/hop/vector/ValueMetaVectorTest.java
@@ -46,6 +46,7 @@ import org.apache.hop.core.row.value.ValueMetaString;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+/** Unit test for {@link ValueMetaVector} */
class ValueMetaVectorTest {
@BeforeAll
@@ -90,6 +91,30 @@ class ValueMetaVectorTest {
assertThrows(HopValueException.class, () ->
ValueMetaVector.parse("[0.1,,0.2]"));
}
+ @Test
+ void testParseRejectsJavaFloatLiterals() {
+ // Float.parseFloat accepts these, but they are not the canonical / JSON /
pgvector form.
+ assertThrows(HopValueException.class, () -> ValueMetaVector.parse("[1f,
2d, 0x1p3]"));
+ assertThrows(HopValueException.class, () -> ValueMetaVector.parse("[1F]"));
+ assertThrows(HopValueException.class, () -> ValueMetaVector.parse("[2D]"));
+ assertThrows(HopValueException.class, () ->
ValueMetaVector.parse("[0x1p3]"));
+ }
+
+ @Test
+ void testParseRejectsNaNAndInfinity() {
+ assertThrows(HopValueException.class, () -> ValueMetaVector.parse("[NaN,
Infinity]"));
+ assertThrows(HopValueException.class, () ->
ValueMetaVector.parse("[NaN]"));
+ assertThrows(HopValueException.class, () ->
ValueMetaVector.parse("[Infinity]"));
+ assertThrows(HopValueException.class, () ->
ValueMetaVector.parse("[-Infinity]"));
+ // Overflow to Infinity must not slip through as a finite vector element.
+ assertThrows(HopValueException.class, () ->
ValueMetaVector.parse("[1e309]"));
+ }
+
+ @Test
+ void testParseAcceptsLeadingDotAndPlusSign() throws Exception {
+ assertArrayEquals(new float[] {0.5f, 1.0f},
ValueMetaVector.parse("[.5,+1]"));
+ }
+
@Test
void testRenderRoundTrip() throws Exception {
float[] vector = {0.1f, -2.5f, 3.0f};