wjones127 commented on code in PR #34668:
URL: https://github.com/apache/arrow/pull/34668#discussion_r1144035990
##########
cpp/src/parquet/encoding_test.cc:
##########
@@ -1639,6 +1639,79 @@ TYPED_TEST(TestDeltaLengthByteArrayEncoding,
BasicRoundTrip) {
/*null_probability*/ 0.1));
}
+std::shared_ptr<Buffer> DeltaEncode(std::vector<int32_t> lengths) {
+ auto encoder = MakeTypedEncoder<Int32Type>(Encoding::DELTA_BINARY_PACKED);
+ encoder->Put(lengths.data(), static_cast<int>(lengths.size()));
+ return encoder->FlushValues();
+}
+
+TEST(TestDeltaLengthByteArrayEncoding, AdHocRoundTrip) {
+ const std::shared_ptr<::arrow::Array> cases[] = {
+ ::arrow::ArrayFromJSON(::arrow::utf8(), R"([])"),
+ ::arrow::ArrayFromJSON(::arrow::utf8(), R"(["abc", "de", ""])"),
+ ::arrow::ArrayFromJSON(::arrow::utf8(), R"(["", "", ""])"),
+ ::arrow::ArrayFromJSON(::arrow::utf8(), R"(["abc", "",
"xyz"])")->Slice(1),
+ };
+
+ std::string expected_encoded_vals[] = {
+ DeltaEncode({})->ToString(),
+ DeltaEncode({3, 2, 0})->ToString() + "abcde",
+ DeltaEncode({0, 0, 0})->ToString(),
+ DeltaEncode({0, 3})->ToString() + "xyz",
+ };
+
+ auto encoder =
MakeTypedEncoder<ByteArrayType>(Encoding::DELTA_LENGTH_BYTE_ARRAY,
+ /*use_dictionary=*/false);
+ auto decoder =
MakeTypedDecoder<ByteArrayType>(Encoding::DELTA_LENGTH_BYTE_ARRAY);
+ for (int i = 0; i < 4; ++i) {
+ auto array = cases[i];
+ auto expected_encoded = expected_encoded_vals[i];
+
+ encoder->Put(*array);
+ std::shared_ptr<Buffer> buffer = encoder->FlushValues();
+
+ ASSERT_EQ(expected_encoded, buffer->ToString());
+
+ int num_values = static_cast<int>(array->length() - array->null_count());
+ decoder->SetData(num_values, buffer->data(),
static_cast<int>(buffer->size()));
+
+ typename EncodingTraits<ByteArrayType>::Accumulator acc;
+ acc.builder.reset(new ::arrow::StringBuilder);
+
+ int values_decoded = decoder->DecodeArrow(num_values, 0, nullptr, 0, &acc);
+ ASSERT_EQ(values_decoded, num_values);
+
+ std::shared_ptr<::arrow::Array> roundtripped_array;
+ ASSERT_OK(acc.builder->Finish(&roundtripped_array));
+ ASSERT_ARRAYS_EQUAL(*array, *roundtripped_array);
+ }
+}
+
+TEST(DeltaLengthByteArrayEncoding, RejectBadBuffer) {
Review Comment:
I'd welcome any other ideas for bad buffers we should try.
##########
cpp/src/parquet/encoding_test.cc:
##########
@@ -1759,12 +1832,10 @@ TEST(DeltaLengthByteArrayEncodingAdHoc, ArrowDirectPut)
{
CheckEncode(::arrow::ArrayFromJSON(::arrow::binary(), values), lengths);
CheckEncode(::arrow::ArrayFromJSON(::arrow::large_binary(), values),
lengths);
- const uint8_t data[] = {
- 0x80, 0x01, 0x04, 0x04, 0x0a, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00,
- 0x00, 0x00, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x57, 0x6f, 0x72, 0x6c, 0x64,
- 0x46, 0x6f, 0x6f, 0x62, 0x61, 0x72, 0x41, 0x44, 0x42, 0x43, 0x45, 0x46,
- };
- auto encoded = Buffer::Wrap(data, sizeof(data));
+ auto encoded =
+ ::arrow::ConcatenateBuffers(
+ {DeltaEncode({5, 5, 6, 6}),
std::make_shared<Buffer>("HelloWorldFoobarADBCEF")})
+ .ValueOrDie();
Review Comment:
Added for readability.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
