dmitry-chirkov-dremio commented on code in PR #55253:
URL: https://github.com/apache/arrow/pull/55253#discussion_r4559987585


##########
cpp/src/arrow/compute/kernels/vector_hash.cc:
##########
@@ -513,15 +515,25 @@ class DictionaryHashKernel : public HashKernel {
       return first_dictionary_;
     }
 
-    auto out_dict_type = first_dictionary_->type();
+    std::shared_ptr<DataType> out_dict_type;
     std::shared_ptr<Array> out_dict;
     RETURN_NOT_OK(dictionary_unifier_->GetResult(&out_dict_type, &out_dict));
+    // The output keeps the input index type, so the highest index into the 
unified
+    // dictionary (its length minus one) must fit in it
+    const Int64Scalar max_index(out_dict->length() - 1);
+    if (out_dict->length() > 0 &&
+        !::arrow::internal::IntegersCanFit(max_index, *index_type_).ok()) {

Review Comment:
   Check the indices we use instead of the dictionary size perhaps?
   
   Counter example with `int8` indices:
   
   ```text
   chunk 1: dictionary = 0..63,    indices = [0]
   chunk 2: dictionary = 100..164, indices = [0]
   ```
   
   Also check the used non-null indices before narrowing them and add this test 
for both functions.



##########
cpp/src/arrow/compute/kernels/vector_hash_test.cc:
##########
@@ -764,6 +765,35 @@ TEST_F(TestHashKernel, DictionaryUniqueAndValueCounts) {
   }
 }
 
+TEST_F(TestHashKernel, DictionaryUniqueAndValueCountsIndexOverflow) {
+  // GH-45010: unifying the chunk dictionaries can produce more entries than 
the
+  // index type can address
+  auto dict_ty = dictionary(int8(), int64());
+  auto make_chunk = [&](int64_t first_value, int64_t dict_length) {
+    std::vector<int64_t> values(dict_length);
+    std::iota(values.begin(), values.end(), first_value);
+    std::shared_ptr<Array> dict;
+    ArrayFromVector<Int64Type, int64_t>(values, &dict);
+    auto indices = ArrayFromJSON(int8(), "[" + std::to_string(dict_length - 1) 
+ "]");
+    return std::make_shared<DictionaryArray>(dict_ty, indices, dict);
+  };
+
+  // 64 + 64 = 128 unified entries still fit in int8: the highest index is 127
+  auto fits = *ChunkedArray::Make({make_chunk(0, 64), make_chunk(100, 64)}, 
dict_ty);
+  ASSERT_OK_AND_ASSIGN(auto uniques, Unique(fits));
+  ASSERT_OK(uniques->ValidateFull());
+  ASSERT_EQ(checked_cast<const 
DictionaryArray&>(*uniques).dictionary()->length(), 128);
+  ASSERT_OK_AND_ASSIGN(auto counts, ValueCounts(fits));
+  ASSERT_OK(counts->ValidateFull());

Review Comment:
   Validate result values and counts (`CheckUnique`, `CheckValueCounts` ...)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to