Jefffrey commented on code in PR #19738:
URL: https://github.com/apache/datafusion/pull/19738#discussion_r2679489989
##########
datafusion/spark/src/function/math/hex.rs:
##########
@@ -110,37 +111,88 @@ impl ScalarUDFImpl for SparkHex {
}
}
-fn hex_int64(num: i64) -> String {
- format!("{num:X}")
-}
-
/// Hex encoding lookup tables for fast byte-to-hex conversion
const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";
const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF";
#[inline]
-fn hex_encode<T: AsRef<[u8]>>(data: T, lower_case: bool) -> String {
- let bytes = data.as_ref();
- let mut s = String::with_capacity(bytes.len() * 2);
- let hex_chars = if lower_case {
+fn hex_int64(num: i64, buffer: &mut Vec<u8>) {
+ if num == 0 {
+ buffer.push(HEX_CHARS_UPPER[0]);
+ return;
+ }
+
+ let mut n = num as u64;
+ let mut temp = [0u8; 16];
+ let mut i = 16;
+ while n != 0 {
+ i -= 1;
+ let digest = (n & 0xF) as u8;
+ temp[i] = HEX_CHARS_UPPER[digest as usize];
+ n >>= 4;
+ }
+
+ buffer.extend_from_slice(&temp[i..]);
+}
+
+/// Generic hex encoding for byte array types
+fn hex_encode_bytes<'a, I, T>(
+ iter: I,
+ lowercase: bool,
+ len: usize,
+) -> Result<ColumnarValue, DataFusionError>
+where
+ I: Iterator<Item = Option<T>>,
+ T: AsRef<[u8]> + 'a,
+{
+ let mut builder = StringBuilder::with_capacity(len, len * 64);
+ let mut buffer = Vec::with_capacity(64);
+ let hex_chars = if lowercase {
HEX_CHARS_LOWER
} else {
HEX_CHARS_UPPER
};
- for &b in bytes {
- s.push(hex_chars[(b >> 4) as usize] as char);
- s.push(hex_chars[(b & 0x0f) as usize] as char);
+
+ for v in iter {
+ if let Some(b) = v {
+ buffer.clear();
+ let bytes = b.as_ref();
+ for &byte in bytes {
+ buffer.push(hex_chars[(byte >> 4) as usize]);
+ buffer.push(hex_chars[(byte & 0x0f) as usize]);
+ }
+ unsafe {
+ builder.append_value(from_utf8_unchecked(&buffer));
+ }
+ } else {
+ builder.append_null();
+ }
}
- s
+
+ Ok(ColumnarValue::Array(Arc::new(builder.finish())))
}
-#[inline(always)]
-fn hex_bytes<T: AsRef<[u8]>>(
- bytes: T,
- lowercase: bool,
-) -> Result<String, std::fmt::Error> {
- let hex_string = hex_encode(bytes, lowercase);
- Ok(hex_string)
+/// Generic hex encoding for int64 type
+fn hex_encode_int64<I>(iter: I, len: usize) -> Result<ColumnarValue,
DataFusionError>
Review Comment:
nit: this doesnt need to be generic since its only used for int64 arrays
##########
datafusion/spark/src/function/math/hex.rs:
##########
@@ -110,37 +111,88 @@ impl ScalarUDFImpl for SparkHex {
}
}
-fn hex_int64(num: i64) -> String {
- format!("{num:X}")
-}
-
/// Hex encoding lookup tables for fast byte-to-hex conversion
const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";
const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF";
#[inline]
-fn hex_encode<T: AsRef<[u8]>>(data: T, lower_case: bool) -> String {
- let bytes = data.as_ref();
- let mut s = String::with_capacity(bytes.len() * 2);
- let hex_chars = if lower_case {
+fn hex_int64(num: i64, buffer: &mut Vec<u8>) {
+ if num == 0 {
+ buffer.push(HEX_CHARS_UPPER[0]);
+ return;
+ }
+
+ let mut n = num as u64;
+ let mut temp = [0u8; 16];
+ let mut i = 16;
+ while n != 0 {
+ i -= 1;
+ let digest = (n & 0xF) as u8;
+ temp[i] = HEX_CHARS_UPPER[digest as usize];
+ n >>= 4;
+ }
Review Comment:
Would it make a performance difference if we unroll this loop manually,
given we know the upper limit of iterations needed?
##########
datafusion/spark/src/function/math/hex.rs:
##########
@@ -110,37 +111,88 @@ impl ScalarUDFImpl for SparkHex {
}
}
-fn hex_int64(num: i64) -> String {
- format!("{num:X}")
-}
-
/// Hex encoding lookup tables for fast byte-to-hex conversion
const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";
const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF";
#[inline]
-fn hex_encode<T: AsRef<[u8]>>(data: T, lower_case: bool) -> String {
- let bytes = data.as_ref();
- let mut s = String::with_capacity(bytes.len() * 2);
- let hex_chars = if lower_case {
+fn hex_int64(num: i64, buffer: &mut Vec<u8>) {
+ if num == 0 {
+ buffer.push(HEX_CHARS_UPPER[0]);
+ return;
+ }
+
+ let mut n = num as u64;
+ let mut temp = [0u8; 16];
+ let mut i = 16;
+ while n != 0 {
+ i -= 1;
+ let digest = (n & 0xF) as u8;
+ temp[i] = HEX_CHARS_UPPER[digest as usize];
+ n >>= 4;
+ }
+
+ buffer.extend_from_slice(&temp[i..]);
+}
+
+/// Generic hex encoding for byte array types
+fn hex_encode_bytes<'a, I, T>(
+ iter: I,
+ lowercase: bool,
+ len: usize,
+) -> Result<ColumnarValue, DataFusionError>
+where
+ I: Iterator<Item = Option<T>>,
+ T: AsRef<[u8]> + 'a,
+{
+ let mut builder = StringBuilder::with_capacity(len, len * 64);
+ let mut buffer = Vec::with_capacity(64);
+ let hex_chars = if lowercase {
HEX_CHARS_LOWER
} else {
HEX_CHARS_UPPER
};
- for &b in bytes {
- s.push(hex_chars[(b >> 4) as usize] as char);
- s.push(hex_chars[(b & 0x0f) as usize] as char);
+
+ for v in iter {
+ if let Some(b) = v {
+ buffer.clear();
+ let bytes = b.as_ref();
+ for &byte in bytes {
+ buffer.push(hex_chars[(byte >> 4) as usize]);
+ buffer.push(hex_chars[(byte & 0x0f) as usize]);
+ }
+ unsafe {
+ builder.append_value(from_utf8_unchecked(&buffer));
+ }
+ } else {
+ builder.append_null();
+ }
}
- s
+
+ Ok(ColumnarValue::Array(Arc::new(builder.finish())))
}
-#[inline(always)]
-fn hex_bytes<T: AsRef<[u8]>>(
- bytes: T,
- lowercase: bool,
-) -> Result<String, std::fmt::Error> {
- let hex_string = hex_encode(bytes, lowercase);
- Ok(hex_string)
+/// Generic hex encoding for int64 type
+fn hex_encode_int64<I>(iter: I, len: usize) -> Result<ColumnarValue,
DataFusionError>
+where
+ I: Iterator<Item = Option<i64>>,
+{
+ let mut builder = StringBuilder::with_capacity(len, len * 16);
+ let mut buffer = Vec::with_capacity(16);
+
+ for v in iter {
+ if let Some(num) = v {
+ buffer.clear();
+ hex_int64(num, &mut buffer);
Review Comment:
I find it a little interesting how we have this `buffer` across iterations
here, but then inside `hex_int64` we have another buffer, which is used to copy
into this outer buffer. Can we unify them somehow?
##########
datafusion/spark/src/function/math/hex.rs:
##########
@@ -110,37 +111,88 @@ impl ScalarUDFImpl for SparkHex {
}
}
-fn hex_int64(num: i64) -> String {
- format!("{num:X}")
-}
-
/// Hex encoding lookup tables for fast byte-to-hex conversion
const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef";
const HEX_CHARS_UPPER: &[u8; 16] = b"0123456789ABCDEF";
#[inline]
-fn hex_encode<T: AsRef<[u8]>>(data: T, lower_case: bool) -> String {
- let bytes = data.as_ref();
- let mut s = String::with_capacity(bytes.len() * 2);
- let hex_chars = if lower_case {
+fn hex_int64(num: i64, buffer: &mut Vec<u8>) {
+ if num == 0 {
+ buffer.push(HEX_CHARS_UPPER[0]);
+ return;
+ }
+
+ let mut n = num as u64;
+ let mut temp = [0u8; 16];
+ let mut i = 16;
+ while n != 0 {
+ i -= 1;
+ let digest = (n & 0xF) as u8;
+ temp[i] = HEX_CHARS_UPPER[digest as usize];
+ n >>= 4;
+ }
+
+ buffer.extend_from_slice(&temp[i..]);
+}
+
+/// Generic hex encoding for byte array types
+fn hex_encode_bytes<'a, I, T>(
+ iter: I,
+ lowercase: bool,
+ len: usize,
+) -> Result<ColumnarValue, DataFusionError>
+where
+ I: Iterator<Item = Option<T>>,
+ T: AsRef<[u8]> + 'a,
+{
+ let mut builder = StringBuilder::with_capacity(len, len * 64);
+ let mut buffer = Vec::with_capacity(64);
+ let hex_chars = if lowercase {
HEX_CHARS_LOWER
} else {
HEX_CHARS_UPPER
};
- for &b in bytes {
- s.push(hex_chars[(b >> 4) as usize] as char);
- s.push(hex_chars[(b & 0x0f) as usize] as char);
+
+ for v in iter {
+ if let Some(b) = v {
+ buffer.clear();
+ let bytes = b.as_ref();
+ for &byte in bytes {
+ buffer.push(hex_chars[(byte >> 4) as usize]);
+ buffer.push(hex_chars[(byte & 0x0f) as usize]);
+ }
+ unsafe {
+ builder.append_value(from_utf8_unchecked(&buffer));
+ }
+ } else {
+ builder.append_null();
+ }
}
- s
+
+ Ok(ColumnarValue::Array(Arc::new(builder.finish())))
}
-#[inline(always)]
-fn hex_bytes<T: AsRef<[u8]>>(
- bytes: T,
- lowercase: bool,
-) -> Result<String, std::fmt::Error> {
- let hex_string = hex_encode(bytes, lowercase);
- Ok(hex_string)
+/// Generic hex encoding for int64 type
+fn hex_encode_int64<I>(iter: I, len: usize) -> Result<ColumnarValue,
DataFusionError>
+where
+ I: Iterator<Item = Option<i64>>,
+{
+ let mut builder = StringBuilder::with_capacity(len, len * 16);
+ let mut buffer = Vec::with_capacity(16);
+
+ for v in iter {
+ if let Some(num) = v {
+ buffer.clear();
+ hex_int64(num, &mut buffer);
+ unsafe {
+ builder.append_value(from_utf8_unchecked(&buffer));
Review Comment:
nit: would be good to add a safety comment, even if its just something like
> SAFETY: all chars are from `HEX_CHARS_UPPER` which are valid ascii
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
