cshuo commented on code in PR #19842:
URL: https://github.com/apache/hudi/pull/19842#discussion_r3948646317
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieRowDataLanceReader.java:
##########
@@ -204,7 +211,53 @@ public ClosableIterator<RowData>
getRowDataIterator(DataType dataType, HoodieSch
@Override
public HoodieSchema getSchema() {
RowType rowType = HoodieFlinkLanceArrowUtils.toRowType(arrowSchema);
- return HoodieSchemaConverter.convertToSchema(rowType);
+ Map<String, String> metadata = arrowSchema.getCustomMetadata();
+ Set<String> vectorColumnNames = HoodieSchema.parseVectorColumnNames(
+ metadata == null ? null :
metadata.get(HoodieSchema.VECTOR_COLUMNS_METADATA_KEY));
+ if (vectorColumnNames.isEmpty()) {
+ return HoodieSchemaConverter.convertToSchema(rowType);
+ }
+ String vectorColumns = vectorColumnNames.stream()
+ .map(name -> name + ":" +
vectorSchemaFromArrow(getTopLevelField(name)).getDimension())
+ .collect(Collectors.joining(","));
+ return HoodieSchemaConverter.convertToSchema(rowType, "record",
vectorColumns);
+ }
+
+ private void validateRequestedVectors(HoodieSchema requestedSchema) {
+ for (HoodieSchemaField field :
requestedSchema.getNonNullType().getFields()) {
+ HoodieSchema fieldSchema = field.schema().getNonNullType();
+ if (fieldSchema.getType() != HoodieSchemaType.VECTOR) {
+ continue;
+ }
+ HoodieSchema.Vector expected = (HoodieSchema.Vector) fieldSchema;
+ HoodieSchema.Vector actual =
vectorSchemaFromArrow(getTopLevelField(field.name()));
+ if (actual.getDimension() != expected.getDimension()
+ || actual.getVectorElementType() != expected.getVectorElementType())
{
+ throw new HoodieValidationException(
+ "Incompatible Lance VECTOR encoding for column '" + field.name()
+ + "': requested " + expected.toTypeDescriptor()
+ + " but file contains " + actual.toTypeDescriptor());
+ }
+ }
+ }
+
+ private Field getTopLevelField(String name) {
+ return arrowSchema.getFields().stream()
+ .filter(field -> field.getName().equals(name))
+ .findFirst()
+ .orElseThrow(() -> new HoodieValidationException(
+ "Missing Lance column in file schema: " + name));
+ }
+
+ private static HoodieSchema.Vector vectorSchemaFromArrow(Field field) {
+ ArrowType.FixedSizeList listType = (ArrowType.FixedSizeList)
field.getType();
+ ArrowType.FloatingPoint elementType =
+ (ArrowType.FloatingPoint) field.getChildren().get(0).getType();
+ HoodieSchema.Vector.VectorElementType vectorElementType =
+ elementType.getPrecision() == FloatingPointPrecision.SINGLE
Review Comment:
Fixed. Precision handling now explicitly accepts only and ; other
precisions throw a descriptive .
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
