andygrove commented on code in PR #627:
URL: https://github.com/apache/datafusion-comet/pull/627#discussion_r1676051349
##########
native/core/src/execution/datafusion/expressions/cast.rs:
##########
@@ -500,19 +500,27 @@ impl Cast {
let to_type = &self.data_type;
let array = array_with_timezone(array, self.timezone.clone(),
Some(to_type))?;
let from_type = array.data_type().clone();
-
- // unpack dictionary string arrays first
- // TODO: we are unpacking a dictionary-encoded array and then
performing
- // the cast. We could potentially improve performance here by casting
the
- // dictionary values directly without unpacking the array first,
although this
- // would add more complexity to the code
let array = match &from_type {
DataType::Dictionary(key_type, value_type)
if key_type.as_ref() == &DataType::Int32
&& (value_type.as_ref() == &DataType::Utf8
|| value_type.as_ref() == &DataType::LargeUtf8) =>
{
- cast_with_options(&array, value_type.as_ref(), &CAST_OPTIONS)?
+ let dict_array = array
+ .as_any()
+ .downcast_ref::<DictionaryArray<Int32Type>>()
+ .expect("Expected a dictionary array");
+
+ let casted_dictionary = DictionaryArray::<Int32Type>::new(
+ dict_array.keys().clone(),
+ self.cast_array(dict_array.values().clone())?,
+ );
+
+ let casted_result = match to_type {
+ DataType::Dictionary(_, _) =>
Arc::new(casted_dictionary.clone()),
Review Comment:
I don't know if we would currently hit the use case of casting *to* a
dictionary type, or whether this is something that could help us in the future
(it seems like we could take advantage of this when running natively even
though Spark doesn't support keeping data dictionary-encoded after a cast, as
far as I know, but maybe @viirya could confirm).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
