mattcasters commented on code in PR #8414: URL: https://github.com/apache/hop/pull/8414#discussion_r4028636332
########## plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java: ########## @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vector; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Arrays; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.database.IDatabase; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopEofException; +import org.apache.hop.core.exception.HopFileException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaPlugin; + +/** + * A dense floating point vector, as produced by an embedding model and consumed by a vector store. + * + * <p>The canonical text form is a bracketed, comma separated list of numbers: {@code + * [0.1,0.2,0.3]}. That form is both valid JSON and the literal syntax pgvector accepts, so a vector + * survives a round trip through a text file, a Data Grid, a JSON document or a database column + * without a conversion step in between. + */ +@ValueMetaPlugin( + id = "1536", // the dimension of OpenAI's text-embedding-3-small + name = "Vector", + description = "Dense floating point vector (embedding)", + image = "vector.svg") +public class ValueMetaVector extends ValueMetaBase { + + public static final int TYPE_VECTOR = 1536; + + public ValueMetaVector() { + super(null, TYPE_VECTOR); + } + + public ValueMetaVector(String name) { + super(name, TYPE_VECTOR); + } + + public ValueMetaVector(ValueMetaVector meta) { + super(meta.name, TYPE_VECTOR); + } + + @Override + public ValueMetaVector clone() { + return (ValueMetaVector) super.clone(); + } + + @Override + public Class<?> getNativeDataTypeClass() { + return float[].class; + } + + @Override + public Object convertData(IValueMeta meta2, Object data2) throws HopValueException { + return toVector(meta2, data2); + } + + /** + * Convert the specified data to a vector. Used internally instead of convertData() to avoid + * upcasts and casts. + */ + private float[] toVector(IValueMeta meta2, Object data2) throws HopValueException { + if (data2 == null) { + return null; + } + // Already a vector? Done. + if (data2 instanceof float[] vector) { + return vector; + } + try { + switch (meta2.getType()) { + case TYPE_VECTOR: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + // Only reached when the storage type is normal and the data is still a String. + // A float[] returns above. + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return toVector(this, meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + case TYPE_STRING: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + // convertBinaryStringToNativeType recurses through convertData, which already + // produces a float[], so there is nothing left to parse here. + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return parse((String) meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + default: + break; Review Comment: **Bug:** this uses `this.convertBinaryStringToNativeType`, which always does `convertData(this.storageMetadata, string)`. Select Values builds a *new* NORMAL Vector with `storageMetadata == null`. Converting a lazy String field (`STORAGE_TYPE_BINARY_STRING` + `byte[]`) therefore NPEs on `meta2.getType()`. The NPE is swallowed below and becomes `I can't convert the specified value to data type : Vector`. The unit test only covers the case where the **Vector** meta itself is BINARY_STRING (CSV field typed as Vector). The documented path — lazy String → Select Values → Vector — is untested and broken. For String sources, `parse(meta2.getString(data2))` already handles NORMAL / BINARY_STRING / INDEXED. Use `this.convertBinaryStringToNativeType` only when converting the Vector field's own lazy bytes (`meta2 == this` and storage metadata is set). ########## plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java: ########## @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vector; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Arrays; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.database.IDatabase; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopEofException; +import org.apache.hop.core.exception.HopFileException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaPlugin; + +/** + * A dense floating point vector, as produced by an embedding model and consumed by a vector store. + * + * <p>The canonical text form is a bracketed, comma separated list of numbers: {@code + * [0.1,0.2,0.3]}. That form is both valid JSON and the literal syntax pgvector accepts, so a vector + * survives a round trip through a text file, a Data Grid, a JSON document or a database column + * without a conversion step in between. + */ +@ValueMetaPlugin( + id = "1536", // the dimension of OpenAI's text-embedding-3-small + name = "Vector", + description = "Dense floating point vector (embedding)", + image = "vector.svg") +public class ValueMetaVector extends ValueMetaBase { + + public static final int TYPE_VECTOR = 1536; + + public ValueMetaVector() { + super(null, TYPE_VECTOR); + } + + public ValueMetaVector(String name) { + super(name, TYPE_VECTOR); + } + + public ValueMetaVector(ValueMetaVector meta) { + super(meta.name, TYPE_VECTOR); + } + + @Override + public ValueMetaVector clone() { + return (ValueMetaVector) super.clone(); + } + + @Override + public Class<?> getNativeDataTypeClass() { + return float[].class; + } + + @Override + public Object convertData(IValueMeta meta2, Object data2) throws HopValueException { + return toVector(meta2, data2); + } + + /** + * Convert the specified data to a vector. Used internally instead of convertData() to avoid + * upcasts and casts. + */ + private float[] toVector(IValueMeta meta2, Object data2) throws HopValueException { + if (data2 == null) { + return null; + } + // Already a vector? Done. + if (data2 instanceof float[] vector) { + return vector; + } + try { + switch (meta2.getType()) { + case TYPE_VECTOR: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + // Only reached when the storage type is normal and the data is still a String. + // A float[] returns above. + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return toVector(this, meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + case TYPE_STRING: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + // convertBinaryStringToNativeType recurses through convertData, which already + // produces a float[], so there is nothing left to parse here. + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return parse((String) meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + default: + break; + } + } catch (HopValueException e) { + throw e; + } catch (RuntimeException ignore) { + // Fall through to the exception below. + } + throw new HopValueException( + this + " : I can't convert the specified value to data type : Vector"); + } + + /** + * Parse the canonical text form. Surrounding brackets are optional and whitespace is ignored, so + * both {@code [0.1, 0.2]} and {@code 0.1,0.2} are accepted. + * + * @param string the text to parse, or null + * @return the parsed vector, or null when the input is null or blank + * @throws HopValueException when the text is not a list of numbers + */ + public static float[] parse(String string) throws HopValueException { + if (string == null) { + return null; + } + String trimmed = string.trim(); + if (trimmed.isEmpty()) { + return null; + } + if (trimmed.startsWith("[") && trimmed.endsWith("]")) { + trimmed = trimmed.substring(1, trimmed.length() - 1).trim(); + } + if (trimmed.isEmpty()) { + return new float[0]; + } + String[] parts = trimmed.split(",", -1); + float[] vector = new float[parts.length]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + if (part.isEmpty()) { + throw new HopValueException( + "Empty element at position " + i + " while parsing a vector from '" + string + "'"); + } + try { + vector[i] = Float.parseFloat(part); + } catch (NumberFormatException e) { + throw new HopValueException( + "'" + part + "' at position " + i + " is not a number in vector '" + string + "'", e); + } + } + return vector; + } + + /** + * Render a vector in the canonical text form. + * + * @param vector the vector, or null + * @return the text form, or null when the vector is null + */ + public static String render(float[] vector) { + if (vector == null) { + return null; + } + StringBuilder builder = new StringBuilder(vector.length * 12 + 2); + builder.append('['); + for (int i = 0; i < vector.length; i++) { + if (i > 0) { + builder.append(','); + } + builder.append(vector[i]); + } + builder.append(']'); + return builder.toString(); + } + + @Override + public int hashCode(Object object) throws HopValueException { + float[] vector = toVector(this, object); + return vector == null ? 0 : Arrays.hashCode(vector); + } + + @Override + public Object cloneValueData(Object object) throws HopValueException { + // Unlike the scalar types, a vector is mutable: hand out a copy so that two rows sharing a + // value can not write through each other. + if (object instanceof float[] vector) { + return vector.clone(); + } + return toVector(this, object); Review Comment: **Bug:** when the value is not already a `float[]`, this always converts via `toVector`. For `STORAGE_TYPE_BINARY_STRING` the row holds `byte[]`; cloning therefore returns a `float[]` while `getStorageType()` stays binary-string. `ValueMetaBase.cloneValueData` leaves non-NORMAL storage alone so lazy bytes stay bytes. After `RowMeta.cloneRow` (error handling, some rowsets), `writeData` delegates to `super.writeData` → `writeBinaryString((byte[]) object)` and ClassCastException. Only clone for NORMAL storage: ```java if (object == null) { return null; } if (storageType != STORAGE_TYPE_NORMAL) { return object; } if (object instanceof float[] vector) { return vector.clone(); } return toVector(this, object); ``` ########## plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java: ########## @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vector; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Arrays; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.database.IDatabase; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopEofException; +import org.apache.hop.core.exception.HopFileException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaPlugin; + +/** + * A dense floating point vector, as produced by an embedding model and consumed by a vector store. + * + * <p>The canonical text form is a bracketed, comma separated list of numbers: {@code + * [0.1,0.2,0.3]}. That form is both valid JSON and the literal syntax pgvector accepts, so a vector + * survives a round trip through a text file, a Data Grid, a JSON document or a database column + * without a conversion step in between. + */ +@ValueMetaPlugin( + id = "1536", // the dimension of OpenAI's text-embedding-3-small + name = "Vector", + description = "Dense floating point vector (embedding)", + image = "vector.svg") +public class ValueMetaVector extends ValueMetaBase { + + public static final int TYPE_VECTOR = 1536; + + public ValueMetaVector() { + super(null, TYPE_VECTOR); + } + + public ValueMetaVector(String name) { + super(name, TYPE_VECTOR); + } + + public ValueMetaVector(ValueMetaVector meta) { + super(meta.name, TYPE_VECTOR); + } + + @Override + public ValueMetaVector clone() { + return (ValueMetaVector) super.clone(); + } + + @Override + public Class<?> getNativeDataTypeClass() { + return float[].class; + } + + @Override + public Object convertData(IValueMeta meta2, Object data2) throws HopValueException { + return toVector(meta2, data2); + } + + /** + * Convert the specified data to a vector. Used internally instead of convertData() to avoid + * upcasts and casts. + */ + private float[] toVector(IValueMeta meta2, Object data2) throws HopValueException { + if (data2 == null) { + return null; + } + // Already a vector? Done. + if (data2 instanceof float[] vector) { + return vector; + } + try { + switch (meta2.getType()) { + case TYPE_VECTOR: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + // Only reached when the storage type is normal and the data is still a String. + // A float[] returns above. + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return toVector(this, meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + case TYPE_STRING: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + // convertBinaryStringToNativeType recurses through convertData, which already + // produces a float[], so there is nothing left to parse here. + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return parse((String) meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + default: + break; + } + } catch (HopValueException e) { + throw e; + } catch (RuntimeException ignore) { + // Fall through to the exception below. Review Comment: **Suggestion:** UUID only catches `IllegalArgumentException` from `UUID.fromString`. Catching every `RuntimeException` is why the NPE in the BINARY_STRING conversion path becomes a generic “can't convert to Vector” with no cause. Catch the conversion-specific exceptions, or at least attach `ignore` as the cause; do not ignore `NullPointerException`. ########## plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java: ########## @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vector; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Arrays; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.database.IDatabase; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopEofException; +import org.apache.hop.core.exception.HopFileException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaPlugin; + +/** + * A dense floating point vector, as produced by an embedding model and consumed by a vector store. + * + * <p>The canonical text form is a bracketed, comma separated list of numbers: {@code + * [0.1,0.2,0.3]}. That form is both valid JSON and the literal syntax pgvector accepts, so a vector + * survives a round trip through a text file, a Data Grid, a JSON document or a database column + * without a conversion step in between. + */ +@ValueMetaPlugin( + id = "1536", // the dimension of OpenAI's text-embedding-3-small + name = "Vector", + description = "Dense floating point vector (embedding)", + image = "vector.svg") +public class ValueMetaVector extends ValueMetaBase { + + public static final int TYPE_VECTOR = 1536; + + public ValueMetaVector() { + super(null, TYPE_VECTOR); + } + + public ValueMetaVector(String name) { + super(name, TYPE_VECTOR); + } + + public ValueMetaVector(ValueMetaVector meta) { + super(meta.name, TYPE_VECTOR); + } + + @Override + public ValueMetaVector clone() { + return (ValueMetaVector) super.clone(); + } + + @Override + public Class<?> getNativeDataTypeClass() { + return float[].class; + } + + @Override + public Object convertData(IValueMeta meta2, Object data2) throws HopValueException { + return toVector(meta2, data2); + } + + /** + * Convert the specified data to a vector. Used internally instead of convertData() to avoid + * upcasts and casts. + */ + private float[] toVector(IValueMeta meta2, Object data2) throws HopValueException { + if (data2 == null) { + return null; + } + // Already a vector? Done. + if (data2 instanceof float[] vector) { + return vector; + } + try { + switch (meta2.getType()) { + case TYPE_VECTOR: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + // Only reached when the storage type is normal and the data is still a String. + // A float[] returns above. + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return toVector(this, meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + case TYPE_STRING: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + // convertBinaryStringToNativeType recurses through convertData, which already + // produces a float[], so there is nothing left to parse here. + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return parse((String) meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + default: + break; + } + } catch (HopValueException e) { + throw e; + } catch (RuntimeException ignore) { + // Fall through to the exception below. + } + throw new HopValueException( + this + " : I can't convert the specified value to data type : Vector"); + } + + /** + * Parse the canonical text form. Surrounding brackets are optional and whitespace is ignored, so + * both {@code [0.1, 0.2]} and {@code 0.1,0.2} are accepted. + * + * @param string the text to parse, or null + * @return the parsed vector, or null when the input is null or blank + * @throws HopValueException when the text is not a list of numbers + */ + public static float[] parse(String string) throws HopValueException { + if (string == null) { + return null; + } + String trimmed = string.trim(); + if (trimmed.isEmpty()) { + return null; + } + if (trimmed.startsWith("[") && trimmed.endsWith("]")) { + trimmed = trimmed.substring(1, trimmed.length() - 1).trim(); + } + if (trimmed.isEmpty()) { + return new float[0]; + } + String[] parts = trimmed.split(",", -1); + float[] vector = new float[parts.length]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + if (part.isEmpty()) { + throw new HopValueException( + "Empty element at position " + i + " while parsing a vector from '" + string + "'"); + } + try { + vector[i] = Float.parseFloat(part); + } catch (NumberFormatException e) { + throw new HopValueException( + "'" + part + "' at position " + i + " is not a number in vector '" + string + "'", e); + } + } + return vector; + } + + /** + * Render a vector in the canonical text form. + * + * @param vector the vector, or null + * @return the text form, or null when the vector is null + */ + public static String render(float[] vector) { + if (vector == null) { + return null; + } + StringBuilder builder = new StringBuilder(vector.length * 12 + 2); + builder.append('['); + for (int i = 0; i < vector.length; i++) { + if (i > 0) { + builder.append(','); + } + builder.append(vector[i]); + } + builder.append(']'); + return builder.toString(); + } + + @Override + public int hashCode(Object object) throws HopValueException { + float[] vector = toVector(this, object); + return vector == null ? 0 : Arrays.hashCode(vector); + } + + @Override + public Object cloneValueData(Object object) throws HopValueException { + // Unlike the scalar types, a vector is mutable: hand out a copy so that two rows sharing a + // value can not write through each other. + if (object instanceof float[] vector) { + return vector.clone(); + } + return toVector(this, object); + } + + /** + * Vectors have no meaningful natural order, but sorting, grouping and distinct all need a total + * order that is stable. Shorter vectors sort first, then the first differing element decides. + */ + @Override + protected int typeCompare(Object object1, Object object2) throws HopValueException { + float[] vector1 = toVector(this, object1); + float[] vector2 = toVector(this, object2); + if (vector1 == null && vector2 == null) { + return 0; + } + if (vector1 == null) { + return -1; + } + if (vector2 == null) { + return 1; + } + if (vector1.length != vector2.length) { + return Integer.compare(vector1.length, vector2.length); + } + for (int i = 0; i < vector1.length; i++) { + int comparison = Float.compare(vector1[i], vector2[i]); + if (comparison != 0) { + return comparison; + } + } + return 0; + } + + @Override + public String getString(Object object) throws HopValueException { + return render(toVector(this, object)); + } + + @Override + public void setPreparedStatementValue( + DatabaseMeta databaseMeta, PreparedStatement preparedStatement, int index, Object data) + throws HopDatabaseException { + try { + float[] vector = toVector(this, data); + if (vector == null) { + preparedStatement.setNull(index, Types.VARCHAR); + return; + } + // The canonical text form. pgvector accepts it for a vector column, and a database without + // a vector type stores the same text in a character column. Handing the driver the float[] + // instead is not portable: most drivers reject it, and the ones that do not tend to write a + // serialized Java object into the column. + preparedStatement.setString(index, render(vector)); + } catch (Exception e) { + throw new HopDatabaseException( + "Error setting vector value #" + + index + + " [" + + toStringMeta() + + "] on prepared statement", + e); + } + } + + @Override + public Object getValueFromResultSet(IDatabase iDatabase, ResultSet resultSet, int index) + throws HopDatabaseException { + try { + Object object = resultSet.getObject(index + 1); + if (object == null) { + return null; + } + if (object instanceof float[] vector) { + return vector; + } + // pgvector hands back its own object type through getObject(); its toString() is the + // canonical form, which is also what a character column returns. + return parse(object.toString()); + } catch (SQLException e) { + throw new HopDatabaseException( + "Unable to get vector value '" Review Comment: **Suggestion:** `float[]` and pgvector/`String` (`toString()` → `[0.1,0.2]`) work. `double[]`, `Float[]`, `Double[]`, and `java.sql.Array` go through `Object.toString()` (`[D@…`), which `parse` rejects. Handle `float[]` / `double[]` (copy to float) / `String` / `Array` before falling back to `toString()`, and only if the text looks like the canonical form. ########## plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java: ########## @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vector; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Arrays; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.database.IDatabase; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopEofException; +import org.apache.hop.core.exception.HopFileException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaPlugin; + +/** + * A dense floating point vector, as produced by an embedding model and consumed by a vector store. + * + * <p>The canonical text form is a bracketed, comma separated list of numbers: {@code + * [0.1,0.2,0.3]}. That form is both valid JSON and the literal syntax pgvector accepts, so a vector + * survives a round trip through a text file, a Data Grid, a JSON document or a database column + * without a conversion step in between. + */ +@ValueMetaPlugin( + id = "1536", // the dimension of OpenAI's text-embedding-3-small + name = "Vector", + description = "Dense floating point vector (embedding)", + image = "vector.svg") +public class ValueMetaVector extends ValueMetaBase { + + public static final int TYPE_VECTOR = 1536; + + public ValueMetaVector() { + super(null, TYPE_VECTOR); + } + + public ValueMetaVector(String name) { + super(name, TYPE_VECTOR); + } + + public ValueMetaVector(ValueMetaVector meta) { + super(meta.name, TYPE_VECTOR); + } + + @Override + public ValueMetaVector clone() { + return (ValueMetaVector) super.clone(); + } + + @Override + public Class<?> getNativeDataTypeClass() { + return float[].class; + } + + @Override + public Object convertData(IValueMeta meta2, Object data2) throws HopValueException { + return toVector(meta2, data2); + } + + /** + * Convert the specified data to a vector. Used internally instead of convertData() to avoid + * upcasts and casts. + */ + private float[] toVector(IValueMeta meta2, Object data2) throws HopValueException { + if (data2 == null) { + return null; + } + // Already a vector? Done. + if (data2 instanceof float[] vector) { + return vector; + } + try { + switch (meta2.getType()) { + case TYPE_VECTOR: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + // Only reached when the storage type is normal and the data is still a String. + // A float[] returns above. + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return toVector(this, meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + case TYPE_STRING: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + // convertBinaryStringToNativeType recurses through convertData, which already + // produces a float[], so there is nothing left to parse here. + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return parse((String) meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + default: + break; + } + } catch (HopValueException e) { + throw e; + } catch (RuntimeException ignore) { + // Fall through to the exception below. + } + throw new HopValueException( + this + " : I can't convert the specified value to data type : Vector"); + } + + /** + * Parse the canonical text form. Surrounding brackets are optional and whitespace is ignored, so + * both {@code [0.1, 0.2]} and {@code 0.1,0.2} are accepted. + * + * @param string the text to parse, or null + * @return the parsed vector, or null when the input is null or blank + * @throws HopValueException when the text is not a list of numbers + */ + public static float[] parse(String string) throws HopValueException { + if (string == null) { + return null; + } + String trimmed = string.trim(); + if (trimmed.isEmpty()) { + return null; + } + if (trimmed.startsWith("[") && trimmed.endsWith("]")) { + trimmed = trimmed.substring(1, trimmed.length() - 1).trim(); + } + if (trimmed.isEmpty()) { + return new float[0]; + } + String[] parts = trimmed.split(",", -1); + float[] vector = new float[parts.length]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + if (part.isEmpty()) { + throw new HopValueException( + "Empty element at position " + i + " while parsing a vector from '" + string + "'"); + } + try { + vector[i] = Float.parseFloat(part); + } catch (NumberFormatException e) { + throw new HopValueException( + "'" + part + "' at position " + i + " is not a number in vector '" + string + "'", e); + } + } + return vector; + } + + /** + * Render a vector in the canonical text form. + * + * @param vector the vector, or null + * @return the text form, or null when the vector is null + */ + public static String render(float[] vector) { + if (vector == null) { + return null; + } + StringBuilder builder = new StringBuilder(vector.length * 12 + 2); + builder.append('['); + for (int i = 0; i < vector.length; i++) { + if (i > 0) { + builder.append(','); + } + builder.append(vector[i]); + } + builder.append(']'); + return builder.toString(); + } + + @Override + public int hashCode(Object object) throws HopValueException { + float[] vector = toVector(this, object); + return vector == null ? 0 : Arrays.hashCode(vector); + } + + @Override + public Object cloneValueData(Object object) throws HopValueException { + // Unlike the scalar types, a vector is mutable: hand out a copy so that two rows sharing a + // value can not write through each other. + if (object instanceof float[] vector) { + return vector.clone(); + } + return toVector(this, object); + } + + /** + * Vectors have no meaningful natural order, but sorting, grouping and distinct all need a total + * order that is stable. Shorter vectors sort first, then the first differing element decides. + */ Review Comment: **Suggestion:** the code compares length first and only then elements of equal-length vectors, so `[2]` sorts before `[1, 0]`. That is not “shorter first, then the first differing element” (lexicographic). Any total order is fine for group/distinct; please match the comment to the implementation, or switch to `Arrays.compare`. ########## plugins/valuetypes/vector/src/main/java/org/apache/hop/vector/ValueMetaVector.java: ########## @@ -0,0 +1,380 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vector; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Arrays; +import org.apache.hop.core.database.DatabaseMeta; +import org.apache.hop.core.database.IDatabase; +import org.apache.hop.core.exception.HopDatabaseException; +import org.apache.hop.core.exception.HopEofException; +import org.apache.hop.core.exception.HopFileException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaPlugin; + +/** + * A dense floating point vector, as produced by an embedding model and consumed by a vector store. + * + * <p>The canonical text form is a bracketed, comma separated list of numbers: {@code + * [0.1,0.2,0.3]}. That form is both valid JSON and the literal syntax pgvector accepts, so a vector + * survives a round trip through a text file, a Data Grid, a JSON document or a database column + * without a conversion step in between. + */ +@ValueMetaPlugin( + id = "1536", // the dimension of OpenAI's text-embedding-3-small + name = "Vector", + description = "Dense floating point vector (embedding)", + image = "vector.svg") +public class ValueMetaVector extends ValueMetaBase { + + public static final int TYPE_VECTOR = 1536; + + public ValueMetaVector() { + super(null, TYPE_VECTOR); + } + + public ValueMetaVector(String name) { + super(name, TYPE_VECTOR); + } + + public ValueMetaVector(ValueMetaVector meta) { + super(meta.name, TYPE_VECTOR); + } + + @Override + public ValueMetaVector clone() { + return (ValueMetaVector) super.clone(); + } + + @Override + public Class<?> getNativeDataTypeClass() { + return float[].class; + } + + @Override + public Object convertData(IValueMeta meta2, Object data2) throws HopValueException { + return toVector(meta2, data2); + } + + /** + * Convert the specified data to a vector. Used internally instead of convertData() to avoid + * upcasts and casts. + */ + private float[] toVector(IValueMeta meta2, Object data2) throws HopValueException { + if (data2 == null) { + return null; + } + // Already a vector? Done. + if (data2 instanceof float[] vector) { + return vector; + } + try { + switch (meta2.getType()) { + case TYPE_VECTOR: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + // Only reached when the storage type is normal and the data is still a String. + // A float[] returns above. + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return toVector(this, meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + case TYPE_STRING: + switch (meta2.getStorageType()) { + case STORAGE_TYPE_NORMAL: + return parse((String) data2); + case STORAGE_TYPE_BINARY_STRING: + // convertBinaryStringToNativeType recurses through convertData, which already + // produces a float[], so there is nothing left to parse here. + return (float[]) convertBinaryStringToNativeType((byte[]) data2); + case STORAGE_TYPE_INDEXED: + return parse((String) meta2.getIndex()[(Integer) data2]); + default: + break; + } + break; + default: + break; + } + } catch (HopValueException e) { + throw e; + } catch (RuntimeException ignore) { + // Fall through to the exception below. + } + throw new HopValueException( + this + " : I can't convert the specified value to data type : Vector"); + } + + /** + * Parse the canonical text form. Surrounding brackets are optional and whitespace is ignored, so + * both {@code [0.1, 0.2]} and {@code 0.1,0.2} are accepted. + * + * @param string the text to parse, or null + * @return the parsed vector, or null when the input is null or blank + * @throws HopValueException when the text is not a list of numbers + */ + public static float[] parse(String string) throws HopValueException { + if (string == null) { + return null; + } + String trimmed = string.trim(); + if (trimmed.isEmpty()) { + return null; + } + if (trimmed.startsWith("[") && trimmed.endsWith("]")) { + trimmed = trimmed.substring(1, trimmed.length() - 1).trim(); + } + if (trimmed.isEmpty()) { + return new float[0]; + } + String[] parts = trimmed.split(",", -1); + float[] vector = new float[parts.length]; + for (int i = 0; i < parts.length; i++) { + String part = parts[i].trim(); + if (part.isEmpty()) { + throw new HopValueException( + "Empty element at position " + i + " while parsing a vector from '" + string + "'"); + } + try { + vector[i] = Float.parseFloat(part); + } catch (NumberFormatException e) { + throw new HopValueException( + "'" + part + "' at position " + i + " is not a number in vector '" + string + "'", e); + } + } + return vector; + } + + /** + * Render a vector in the canonical text form. + * + * @param vector the vector, or null + * @return the text form, or null when the vector is null + */ + public static String render(float[] vector) { + if (vector == null) { + return null; + } + StringBuilder builder = new StringBuilder(vector.length * 12 + 2); + builder.append('['); + for (int i = 0; i < vector.length; i++) { + if (i > 0) { + builder.append(','); + } + builder.append(vector[i]); + } + builder.append(']'); + return builder.toString(); + } + + @Override + public int hashCode(Object object) throws HopValueException { + float[] vector = toVector(this, object); + return vector == null ? 0 : Arrays.hashCode(vector); + } + + @Override + public Object cloneValueData(Object object) throws HopValueException { + // Unlike the scalar types, a vector is mutable: hand out a copy so that two rows sharing a + // value can not write through each other. + if (object instanceof float[] vector) { + return vector.clone(); + } + return toVector(this, object); + } + + /** + * Vectors have no meaningful natural order, but sorting, grouping and distinct all need a total + * order that is stable. Shorter vectors sort first, then the first differing element decides. + */ + @Override + protected int typeCompare(Object object1, Object object2) throws HopValueException { + float[] vector1 = toVector(this, object1); + float[] vector2 = toVector(this, object2); + if (vector1 == null && vector2 == null) { + return 0; + } + if (vector1 == null) { + return -1; + } + if (vector2 == null) { + return 1; + } + if (vector1.length != vector2.length) { + return Integer.compare(vector1.length, vector2.length); + } + for (int i = 0; i < vector1.length; i++) { + int comparison = Float.compare(vector1[i], vector2[i]); + if (comparison != 0) { + return comparison; + } + } + return 0; + } + + @Override + public String getString(Object object) throws HopValueException { + return render(toVector(this, object)); + } + + @Override + public void setPreparedStatementValue( + DatabaseMeta databaseMeta, PreparedStatement preparedStatement, int index, Object data) + throws HopDatabaseException { + try { + float[] vector = toVector(this, data); + if (vector == null) { + preparedStatement.setNull(index, Types.VARCHAR); + return; + } + // The canonical text form. pgvector accepts it for a vector column, and a database without + // a vector type stores the same text in a character column. Handing the driver the float[] + // instead is not portable: most drivers reject it, and the ones that do not tend to write a + // serialized Java object into the column. + preparedStatement.setString(index, render(vector)); + } catch (Exception e) { + throw new HopDatabaseException( + "Error setting vector value #" + + index + + " [" + + toStringMeta() + + "] on prepared statement", + e); + } + } + + @Override + public Object getValueFromResultSet(IDatabase iDatabase, ResultSet resultSet, int index) + throws HopDatabaseException { + try { + Object object = resultSet.getObject(index + 1); + if (object == null) { + return null; + } + if (object instanceof float[] vector) { + return vector; + } + // pgvector hands back its own object type through getObject(); its toString() is the + // canonical form, which is also what a character column returns. + return parse(object.toString()); + } catch (SQLException e) { + throw new HopDatabaseException( + "Unable to get vector value '" + + toStringMeta() + + "' from database resultset, index " + + index, + e); + } catch (Exception e) { + throw new HopDatabaseException("Unable to read vector value", e); + } + } + + @Override + public byte[] getBinaryString(Object object) throws HopValueException { + if (isStorageBinaryString() && identicalFormat) { + return (byte[]) object; + } + float[] vector = toVector(this, object); + if (vector == null) { + return null; + } + try { + String encode = getStringEncoding(); + Charset charset = encode == null ? StandardCharsets.UTF_8 : Charset.forName(encode); + return render(vector).getBytes(charset); + } catch (Exception e) { + throw new HopValueException("Unable to get binary string for vector", e); + } + } + + @Override + public void writeData(DataOutputStream outputStream, Object object) throws HopFileException { + // Delegate non-NORMAL cases to the base class + if (getStorageType() != STORAGE_TYPE_NORMAL) { + super.writeData(outputStream, object); + return; + } + try { + outputStream.writeBoolean(object == null); + if (object != null) { + float[] vector = toVector(this, object); Review Comment: **Suggestion:** the null flag is `object == null`, but `parse` maps blank/whitespace to null, and TYPE_VECTOR NORMAL is allowed to still hold a String. `vector.length` then NPEs (caught as `HopFileException`), so a value that `getString`/`hashCode` treat as null cannot be serialised. Convert first, then write the null flag from `vector == null`. ########## plugins/valuetypes/vector/src/test/java/org/apache/hop/vector/ValueMetaVectorTest.java: ########## @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vector; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.nio.charset.StandardCharsets; +import org.apache.hop.core.HopClientEnvironment; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class ValueMetaVectorTest { + + @BeforeAll + static void setUpBeforeClass() throws Exception { + HopClientEnvironment.init(); + } + + @Test + void testTypeAndNativeClass() { + ValueMetaVector meta = new ValueMetaVector("embedding"); + assertEquals(ValueMetaVector.TYPE_VECTOR, meta.getType()); + assertEquals("embedding", meta.getName()); + assertEquals(float[].class, meta.getNativeDataTypeClass()); + } + + @Test + void testParseCanonicalForm() throws Exception { + assertArrayEquals(new float[] {0.1f, 0.2f, 0.3f}, ValueMetaVector.parse("[0.1,0.2,0.3]")); + } + + @Test + void testParseToleratesWhitespaceAndMissingBrackets() throws Exception { + assertArrayEquals(new float[] {0.1f, 0.2f}, ValueMetaVector.parse(" [ 0.1 , 0.2 ] ")); + assertArrayEquals(new float[] {0.1f, 0.2f}, ValueMetaVector.parse("0.1, 0.2")); + } + + @Test + void testParseNegativeAndScientificNotation() throws Exception { + assertArrayEquals(new float[] {-0.5f, 1.5e-3f}, ValueMetaVector.parse("[-0.5,1.5e-3]")); + } + + @Test + void testParseNullAndBlankAndEmpty() throws Exception { + assertNull(ValueMetaVector.parse(null)); + assertNull(ValueMetaVector.parse(" ")); + assertArrayEquals(new float[0], ValueMetaVector.parse("[]")); + } + + @Test + void testParseRejectsNonNumbers() { + assertThrows(HopValueException.class, () -> ValueMetaVector.parse("[0.1,abc]")); + assertThrows(HopValueException.class, () -> ValueMetaVector.parse("[0.1,,0.2]")); + } + + @Test + void testRenderRoundTrip() throws Exception { + float[] vector = {0.1f, -2.5f, 3.0f}; + String rendered = ValueMetaVector.render(vector); + assertTrue(rendered.startsWith("[") && rendered.endsWith("]")); + assertArrayEquals(vector, ValueMetaVector.parse(rendered)); + assertNull(ValueMetaVector.render(null)); + } + + @Test + void testGetStringUsesCanonicalForm() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + assertEquals("[1.0,2.0]", meta.getString(new float[] {1f, 2f})); + assertNull(meta.getString(null)); + } + + @Test + void testConvertFromString() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + Object converted = meta.convertData(new ValueMetaString("s"), "[1,2,3]"); + assertInstanceOf(float[].class, converted); + assertArrayEquals(new float[] {1f, 2f, 3f}, (float[]) converted); + } + + @Test + void testConvertPassesThroughVector() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + float[] vector = {1f, 2f}; + assertArrayEquals(vector, (float[]) meta.convertData(meta, vector)); + } + + @Test + void testConvertNullStaysNull() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + assertNull(meta.convertData(new ValueMetaString("s"), null)); + } + + @Test + void testCloneValueDataCopiesTheArray() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + float[] original = {1f, 2f, 3f}; + float[] copy = (float[]) meta.cloneValueData(original); + assertArrayEquals(original, copy); + assertNotSame(original, copy); + copy[0] = 99f; + assertEquals(1f, original[0], 0.0f); + } + + @Test + void testCompareOrdersByLengthThenElement() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + assertTrue(meta.compare(new float[] {1f}, new float[] {1f, 2f}) < 0); + assertTrue(meta.compare(new float[] {1f, 3f}, new float[] {1f, 2f}) > 0); + assertEquals(0, meta.compare(new float[] {1f, 2f}, new float[] {1f, 2f})); + } + + @Test + void testCompareHandlesNulls() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + assertEquals(0, meta.compare(null, null)); + assertTrue(meta.compare(null, new float[] {1f}) < 0); + assertTrue(meta.compare(new float[] {1f}, null) > 0); + } + + @Test + void testHashCodeMatchesContent() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + assertEquals(meta.hashCode(new float[] {1f, 2f}), meta.hashCode(new float[] {1f, 2f})); + assertEquals(0, meta.hashCode(null)); + } + + @Test + void testBinaryStringRoundTrip() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + byte[] binary = meta.getBinaryString(new float[] {1f, 2f}); + assertEquals("[1.0,2.0]", new String(binary, StandardCharsets.UTF_8)); + assertNull(meta.getBinaryString(null)); + } + + @Test + void testWriteAndReadDataRoundTrip() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + float[] vector = new float[1536]; + for (int i = 0; i < vector.length; i++) { + vector[i] = i * 0.001f; + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + meta.writeData(out, vector); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + assertArrayEquals(vector, (float[]) meta.readData(in)); + } + } + + @Test + void testWriteAndReadNull() throws Exception { + ValueMetaVector meta = new ValueMetaVector("v"); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + meta.writeData(out, null); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + assertNull(meta.readData(in)); + } + } + + @Test + void testBinaryStringStorageConvertsBack() throws Exception { Review Comment: **Suggestion:** this only tests when the *Vector* meta is BINARY_STRING. Please add: - source `ValueMetaString` with `STORAGE_TYPE_BINARY_STRING` + storage metadata, target a fresh NORMAL `ValueMetaVector` (the Select Values case; currently broken) - `cloneValueData` on BINARY_STRING data returns the same `byte[]` (or a copy of the bytes), not a `float[]` - `ValueMetaFactory.getIdForValueMeta("Vector") == 1536` - `setPreparedStatementValue` writes the canonical text, never the `float[]` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
