coderfender commented on code in PR #2099:
URL: https://github.com/apache/datafusion-comet/pull/2099#discussion_r2271177077
##########
native/spark-expr/src/static_invoke/char_varchar_utils/read_side_padding.rs:
##########
@@ -71,44 +101,101 @@ fn spark_read_side_padding2(
}
}
+enum RPadArgument {
+ ConstLength(i32),
+ ColArray(ArrayRef),
+}
+
fn spark_read_side_padding_internal<T: OffsetSizeTrait>(
array: &ArrayRef,
- length: i32,
truncate: bool,
+ rpad_argument: RPadArgument,
) -> Result<ColumnarValue, DataFusionError> {
let string_array = as_generic_string_array::<T>(array)?;
- let length = 0.max(length) as usize;
- let space_string = " ".repeat(length);
+ match rpad_argument {
+ RPadArgument::ColArray(array_int) => {
+ let int_pad_array = array_int.as_primitive::<Int32Type>();
+ let mut str_pad_value_map = HashMap::new();
+ for i in 0..string_array.len() {
+ if string_array.is_null(i) || int_pad_array.is_null(i) {
+ continue; // skip nulls
+ }
+ str_pad_value_map.insert(string_array.value(i),
int_pad_array.value(i));
+ }
- let mut builder =
- GenericStringBuilder::<T>::with_capacity(string_array.len(),
string_array.len() * length);
+ let mut builder = GenericStringBuilder::<T>::with_capacity(
+ str_pad_value_map.len(),
+ str_pad_value_map.len() * int_pad_array.len(),
+ );
- for string in string_array.iter() {
- match string {
- Some(string) => {
- // It looks Spark's UTF8String is closer to chars rather than
graphemes
- // https://stackoverflow.com/a/46290728
- let char_len = string.chars().count();
- if length <= char_len {
- if truncate {
- let idx = string
- .char_indices()
- .nth(length)
- .map(|(i, _)| i)
- .unwrap_or(string.len());
- builder.append_value(&string[..idx]);
- } else {
- builder.append_value(string);
+ for string in string_array.iter() {
+ match string {
+ Some(string) => {
+ // It looks Spark's UTF8String is closer to chars
rather than graphemes
+ // https://stackoverflow.com/a/46290728
+ let char_len = string.chars().count();
Review Comment:
Thank you for the feedback @mbutrovich . I moved the string manipulation
logic to a separate function and leveraged it in both the use cases.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
