projjal commented on a change in pull request #9707:
URL: https://github.com/apache/arrow/pull/9707#discussion_r596529641
##########
File path: cpp/src/gandiva/gdv_function_stubs.cc
##########
@@ -122,6 +123,133 @@ int32_t gdv_fn_populate_varlen_vector(int64_t
context_ptr, int8_t* data_ptr,
return 0;
}
+#define SHA128_HASH_FUNCTION(TYPE)
\
+ GANDIVA_EXPORT
\
+ const char *gdv_fn_sha128_##TYPE(int64_t context, gdv_##TYPE value,
\
+ bool validity, int32_t *out_length) {
\
+ if (!validity) {
\
+ *out_length = 0;
\
+ return "";
\
Review comment:
Return a hash for null values too.
##########
File path: cpp/src/gandiva/tests/hash_test.cc
##########
@@ -147,4 +153,252 @@ TEST_F(TestHash, TestBuf) {
}
}
+TEST_F(TestHash, TestSha256Simple) {
Review comment:
I think you should also add unit test for HashUtils::GetHash() with
expected sha values.
You can also add java test in java/gandiva/ProjectorTest and match by
generating sha using MessageDigest class.
##########
File path: cpp/src/gandiva/hash_utils.cc
##########
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstring>
+#include "openssl/evp.h"
+#include "gandiva/hash_utils.h"
+#include "gandiva/execution_context.h"
+#include "gandiva/gdv_function_stubs.h"
+
+namespace gandiva {
+ const char* HashUtils::HashUsingSha256(int64_t context,
+ const void* message,
+ size_t message_length,
+ int32_t *out_length) {
+ // The buffer size is the hash size + null character
+ int sha256_result_length = 65;
+ return HashUtils::GetHash(context, message, message_length, EVP_sha256(),
+ sha256_result_length,
out_length);
+ }
+ const char* HashUtils::HashUsingSha128(int64_t context,
+ const void* message,
+ size_t message_length,
+ int32_t *out_length) {
+ // The buffer size is the hash size + null character
+ int sha128_result_length = 41;
+ return HashUtils::GetHash(context, message, message_length, EVP_sha1(),
+ sha128_result_length,
out_length);
+ }
+
+ const char* HashUtils::GetHash(int64_t context,
+ const void* message,
+ size_t message_length,
+ const EVP_MD *hash_type,
+ int result_buf_size,
+ int32_t *out_length) {
+ EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
+
+ if (md_ctx == nullptr) {
+ HashUtils::ErrorMessage(context, "Could not allocate memory "
+ "for
SHA processing.");
+ return "";
+ }
+
+ int evp_success_status = 1;
+
+ if (EVP_DigestInit_ex(md_ctx, hash_type, nullptr) != evp_success_status) {
+ HashUtils::ErrorMessage(context, "Could not obtain the hash "
+ "for
the defined value.");
+ EVP_MD_CTX_free(md_ctx);
+ return "";
+ }
+
+ if (EVP_DigestUpdate(md_ctx, message, message_length) !=
evp_success_status) {
+ HashUtils::ErrorMessage(context, "Could not obtain the hash for "
+ "the
defined value.");
+ EVP_MD_CTX_free(md_ctx);
+ return "";
+ }
+
+ int hash_size = EVP_MD_size(hash_type);
+ auto* result = static_cast<unsigned char*>(OPENSSL_malloc(hash_size));
+
+ if (result == nullptr) {
+ HashUtils::ErrorMessage(context, "Could not allocate memory "
+ "for
SHA processing.");
+ EVP_MD_CTX_free(md_ctx);
+ return "";
+ }
+
+ unsigned int result_length;
+ EVP_DigestFinal_ex(md_ctx, result, &result_length);
+
+ int tmp_buf_len = 4;
+
+ auto hex_buffer =
+ reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context,
tmp_buf_len));
+
+ auto result_buffer =
+ reinterpret_cast<char*>(gdv_fn_context_arena_malloc(context,
result_buf_size));
+
+ CleanCharArray(result_buffer);
+ CleanCharArray(hex_buffer);
+
+ if (hex_buffer == nullptr || result_buffer == nullptr) {
+ gdv_fn_context_set_error_msg(context, "Could not allocate memory "
+ "for the result buffers.");
+ EVP_MD_CTX_free(md_ctx);
+ return "";
+ }
+
+ for (unsigned int j = 0; j < result_length; j++) {
+ unsigned char hex_number = result[j];
+ snprintf(hex_buffer, tmp_buf_len, "%02x", hex_number);
+ strncat(result_buffer, hex_buffer, tmp_buf_len);
+ }
+
+ // Add the NULL character to shows the end of the string
+ result_buffer[result_buf_size - 1] = '\0';
+
+ // free the resources to avoid memory leaks
+ EVP_MD_CTX_free(md_ctx);
+ free(result);
+
+ *out_length = strlen(result_buffer);
Review comment:
*out_length = result_buf_size
##########
File path: cpp/src/gandiva/hash_utils.cc
##########
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstring>
+#include "openssl/evp.h"
+#include "gandiva/hash_utils.h"
+#include "gandiva/execution_context.h"
+#include "gandiva/gdv_function_stubs.h"
+
+namespace gandiva {
+ const char* HashUtils::HashUsingSha256(int64_t context,
+ const void* message,
+ size_t message_length,
+ int32_t *out_length) {
+ // The buffer size is the hash size + null character
+ int sha256_result_length = 65;
+ return HashUtils::GetHash(context, message, message_length, EVP_sha256(),
+ sha256_result_length,
out_length);
+ }
+ const char* HashUtils::HashUsingSha128(int64_t context,
+ const void* message,
+ size_t message_length,
+ int32_t *out_length) {
+ // The buffer size is the hash size + null character
+ int sha128_result_length = 41;
+ return HashUtils::GetHash(context, message, message_length, EVP_sha1(),
+ sha128_result_length,
out_length);
+ }
+
+ const char* HashUtils::GetHash(int64_t context,
+ const void* message,
+ size_t message_length,
+ const EVP_MD *hash_type,
+ int result_buf_size,
+ int32_t *out_length) {
+ EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
+
+ if (md_ctx == nullptr) {
+ HashUtils::ErrorMessage(context, "Could not allocate memory "
+ "for
SHA processing.");
+ return "";
+ }
+
+ int evp_success_status = 1;
+
+ if (EVP_DigestInit_ex(md_ctx, hash_type, nullptr) != evp_success_status) {
+ HashUtils::ErrorMessage(context, "Could not obtain the hash "
+ "for
the defined value.");
+ EVP_MD_CTX_free(md_ctx);
+ return "";
+ }
+
+ if (EVP_DigestUpdate(md_ctx, message, message_length) !=
evp_success_status) {
+ HashUtils::ErrorMessage(context, "Could not obtain the hash for "
+ "the
defined value.");
+ EVP_MD_CTX_free(md_ctx);
+ return "";
+ }
+
+ int hash_size = EVP_MD_size(hash_type);
+ auto* result = static_cast<unsigned char*>(OPENSSL_malloc(hash_size));
+
+ if (result == nullptr) {
+ HashUtils::ErrorMessage(context, "Could not allocate memory "
+ "for
SHA processing.");
+ EVP_MD_CTX_free(md_ctx);
+ return "";
+ }
+
+ unsigned int result_length;
+ EVP_DigestFinal_ex(md_ctx, result, &result_length);
Review comment:
add a check result_length == hash_size
and result_buf_size == 2 * hash_size
##########
File path: cpp/src/gandiva/function_registry_common.h
##########
@@ -199,6 +199,28 @@ typedef std::unordered_map<const FunctionSignature*, const
NativeFunction*, KeyH
DataTypeVector{TYPE(), int64()}, int64(), kResultNullNever, \
ARROW_STRINGIFY(NAME##WithSeed_##TYPE))
+// HashSHA128 functions that :
+// - NULL handling is of type NULL_NEVER
+// - can return errors
+//
+// The function name includes the base name & input type name.
gdv_fn_sha128_float64
+#define HASH_SHA128_NULL_NEVER(NAME, ALIASES, TYPE) \
Review comment:
I think it is called SHA1 and not SHA128. The digest bit size is also
160 and not 128
##########
File path: cpp/src/gandiva/hash_utils.cc
##########
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstring>
+#include "openssl/evp.h"
+#include "gandiva/hash_utils.h"
+#include "gandiva/execution_context.h"
+#include "gandiva/gdv_function_stubs.h"
+
+namespace gandiva {
+ const char* HashUtils::HashUsingSha256(int64_t context,
+ const void* message,
+ size_t message_length,
+ int32_t *out_length) {
+ // The buffer size is the hash size + null character
+ int sha256_result_length = 65;
Review comment:
Arrow doesn't use null terminated string so adding null character is not
required. assigning out_length is enough.
##########
File path: cpp/src/gandiva/tests/hash_test.cc
##########
@@ -147,4 +153,252 @@ TEST_F(TestHash, TestBuf) {
}
}
+TEST_F(TestHash, TestSha256Simple) {
+ // schema for input fields
+ auto field_a = field("a", int32());
+ auto field_b = field("b", int64());
+ auto field_c = field("c", float32());
+ auto field_d = field("d", float64());
+ auto schema = arrow::schema({field_a, field_b, field_c, field_d});
+
+ // output fields
+ auto res_0 = field("res0", utf8());
+ auto res_1 = field("res1", utf8());
+ auto res_2 = field("res2", utf8());
+ auto res_3 = field("res3", utf8());
+
+ // build expressions.
+ // hashSHA256(a)
+ auto node_a = TreeExprBuilder::MakeField(field_a);
+ auto hashSha256_1 = TreeExprBuilder::MakeFunction("hashSHA256",
+
{node_a}, utf8());
+ auto expr_0 = TreeExprBuilder::MakeExpression(hashSha256_1, res_0);
+
+ auto node_b = TreeExprBuilder::MakeField(field_b);
+ auto hashSha256_2 = TreeExprBuilder::MakeFunction("hashSHA256",
+
{node_b}, utf8());
+ auto expr_1 = TreeExprBuilder::MakeExpression(hashSha256_2, res_1);
+
+ auto node_c = TreeExprBuilder::MakeField(field_c);
+ auto hashSha256_3 = TreeExprBuilder::MakeFunction("hashSHA256",
+
{node_c}, utf8());
+ auto expr_2 = TreeExprBuilder::MakeExpression(hashSha256_3, res_2);
+
+ auto node_d = TreeExprBuilder::MakeField(field_d);
+ auto hashSha256_4 = TreeExprBuilder::MakeFunction("hashSHA256",
+
{node_d}, utf8());
+ auto expr_3 = TreeExprBuilder::MakeExpression(hashSha256_4, res_3);
+
+ // Build a projector for the expressions.
+ std::shared_ptr<Projector> projector;
+ auto status =
+ Projector::Make(schema, {expr_0, expr_1, expr_2, expr_3},
+ TestConfiguration(), &projector);
+ EXPECT_TRUE(status.ok()) << status.message();
+
+ // Create a row-batch with some sample data
+ int num_records = 2;
+ auto validity_array = {false, true};
+
+ auto array_int32 =
+ MakeArrowArrayInt32({1, 0}, validity_array);
+
+ auto array_int64 =
+ MakeArrowArrayInt64({1, 0}, validity_array);
+
+ auto array_float32 =
+ MakeArrowArrayFloat32({1.0, 0.0}, validity_array);
+
+ auto array_float64 =
+ MakeArrowArrayFloat64({1.0, 0.0}, validity_array);
+
+ // prepare input record batch
+ auto in_batch = arrow::RecordBatch::Make(schema, num_records,
+
{array_int32, array_int64,
+ array_float32, array_float64});
+
+ // Evaluate expression
+ arrow::ArrayVector outputs;
+ status = projector->Evaluate(*in_batch, pool_, &outputs);
+ EXPECT_TRUE(status.ok());
+
+ EXPECT_ARROW_ARRAY_EQUALS(outputs.at(0), outputs.at(1));
+ EXPECT_ARROW_ARRAY_EQUALS(outputs.at(1), outputs.at(2));
+ EXPECT_ARROW_ARRAY_EQUALS(outputs.at(2), outputs.at(3));
+}
+
+TEST_F(TestHash, TestSha256Varlen) {
+ // schema for input fields
+ auto field_a = field("a", utf8());
+ auto schema = arrow::schema({field_a});
+
+ // output fields
+ auto res_0 = field("res0", utf8());
+
+ // build expressions.
+ // hashSHA256(a)
+ auto node_a = TreeExprBuilder::MakeField(field_a);
+ auto hashSha256 = TreeExprBuilder::MakeFunction("hashSHA256",
+
{node_a}, utf8());
+ auto expr_0 = TreeExprBuilder::MakeExpression(hashSha256, res_0);
+
+ // Build a projector for the expressions.
+ std::shared_ptr<Projector> projector;
+ auto status =
+ Projector::Make(schema, {expr_0}, TestConfiguration(), &projector);
+ EXPECT_TRUE(status.ok()) << status.message();
+
+ // Create a row-batch with some sample data
+ int num_records = 3;
+
+ std::string first_string = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn\nY "
+ "[ˈʏpsilɔn], Yen
[jɛn], Yoga [ˈjoːgɑ]";
+ std::string second_string = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeın\nY "
+ "[ˈʏpsilɔn], Yen
[jɛn], Yoga [ˈjoːgɑ] コンニチハ";
+
+ auto array_a =
+ MakeArrowArrayUtf8({"foo", first_string, second_string}, {false, true,
true});
+
+ // prepare input record batch
+ auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_a});
+
+ // Evaluate expression
+ arrow::ArrayVector outputs;
+ status = projector->Evaluate(*in_batch, pool_, &outputs);
+ EXPECT_TRUE(status.ok());
+
+ auto response = outputs.at(0);
+ EXPECT_EQ(response->null_count(), 0);
+ EXPECT_EQ(response->GetScalar(0).ValueOrDie()->ToString(), "");
+ for (int i = 1; i < num_records; ++i) {
+ const auto &value_at_position =
response->GetScalar(i).ValueOrDie()->ToString();
+ EXPECT_NE(value_at_position,
+ response->GetScalar(i - 1).ValueOrDie()->ToString());
+ }
+}
+
+TEST_F(TestHash, TestSha128Simple) {
+ // schema for input fields
+ auto field_a = field("a", int32());
+ auto field_b = field("b", int64());
+ auto field_c = field("c", float32());
+ auto field_d = field("d", float64());
+ auto schema = arrow::schema({field_a, field_b, field_c, field_d});
+
+ // output fields
+ auto res_0 = field("res0", utf8());
+ auto res_1 = field("res1", utf8());
+ auto res_2 = field("res2", utf8());
+ auto res_3 = field("res3", utf8());
+
+ // build expressions.
+ // hashSHA128(a)
+ auto node_a = TreeExprBuilder::MakeField(field_a);
+ auto hashSha128_1 = TreeExprBuilder::MakeFunction("hashSHA128",
+
{node_a}, utf8());
+ auto expr_0 = TreeExprBuilder::MakeExpression(hashSha128_1, res_0);
+
+ auto node_b = TreeExprBuilder::MakeField(field_b);
+ auto hashSha128_2 = TreeExprBuilder::MakeFunction("hashSHA128",
+
{node_b}, utf8());
+ auto expr_1 = TreeExprBuilder::MakeExpression(hashSha128_2, res_1);
+
+ auto node_c = TreeExprBuilder::MakeField(field_c);
+ auto hashSha128_3 = TreeExprBuilder::MakeFunction("hashSHA128",
+
{node_c}, utf8());
+ auto expr_2 = TreeExprBuilder::MakeExpression(hashSha128_3, res_2);
+
+ auto node_d = TreeExprBuilder::MakeField(field_d);
+ auto hashSha128_4 = TreeExprBuilder::MakeFunction("hashSHA128",
+
{node_d}, utf8());
+ auto expr_3 = TreeExprBuilder::MakeExpression(hashSha128_4, res_3);
+
+ // Build a projector for the expressions.
+ std::shared_ptr<Projector> projector;
+ auto status =
+ Projector::Make(schema, {expr_0, expr_1, expr_2, expr_3},
+ TestConfiguration(),
&projector);
+ EXPECT_TRUE(status.ok()) << status.message();
+
+ // Create a row-batch with some sample data
+ int num_records = 2;
+ auto validity_array = {false, true};
+
+ auto array_int32 =
+ MakeArrowArrayInt32({1, 0}, validity_array);
+
+ auto array_int64 =
+ MakeArrowArrayInt64({1, 0}, validity_array);
+
+ auto array_float32 =
+ MakeArrowArrayFloat32({1.0, 0.0}, validity_array);
+
+ auto array_float64 =
+ MakeArrowArrayFloat64({1.0, 0.0}, validity_array);
+
+ // prepare input record batch
+ auto in_batch = arrow::RecordBatch::Make(schema, num_records,
+
{array_int32, array_int64,
+ array_float32, array_float64});
+
+ // Evaluate expression
+ arrow::ArrayVector outputs;
+ status = projector->Evaluate(*in_batch, pool_, &outputs);
+ EXPECT_TRUE(status.ok());
+
+ EXPECT_ARROW_ARRAY_EQUALS(outputs.at(0), outputs.at(1));
+ EXPECT_ARROW_ARRAY_EQUALS(outputs.at(1), outputs.at(2));
+ EXPECT_ARROW_ARRAY_EQUALS(outputs.at(2), outputs.at(3));
+}
+
+TEST_F(TestHash, TestSha128Varlen) {
+ // schema for input fields
+ auto field_a = field("a", utf8());
+ auto schema = arrow::schema({field_a});
+
+ // output fields
+ auto res_0 = field("res0", utf8());
+
+ // build expressions.
+ // hashSHA128(a)
+ auto node_a = TreeExprBuilder::MakeField(field_a);
+ auto hashSha128 = TreeExprBuilder::MakeFunction("hashSHA128",
+
{node_a}, utf8());
+ auto expr_0 = TreeExprBuilder::MakeExpression(hashSha128, res_0);
+
+ // Build a projector for the expressions.
+ std::shared_ptr<Projector> projector;
+ auto status =
+ Projector::Make(schema, {expr_0}, TestConfiguration(), &projector);
+ EXPECT_TRUE(status.ok()) << status.message();
+
+ // Create a row-batch with some sample data
+ int num_records = 3;
+
+ std::string first_string = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn\nY
[ˈʏpsilɔn], "
+ "Yen [jɛn], Yoga
[ˈjoːgɑ]";
+ std::string second_string = "ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeın\nY
[ˈʏpsilɔn], "
+ "Yen [jɛn],
Yoga [ˈjoːgɑ] コンニチハ";
+
+ auto array_a =
+ MakeArrowArrayUtf8({"foo", first_string, second_string},
+ {false, true, true});
+
+ // prepare input record batch
+ auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array_a});
+
+ // Evaluate expression
+ arrow::ArrayVector outputs;
+ status = projector->Evaluate(*in_batch, pool_, &outputs);
+ EXPECT_TRUE(status.ok());
+
+ auto response = outputs.at(0);
+ EXPECT_EQ(response->null_count(), 0);
+ EXPECT_EQ(response->GetScalar(0).ValueOrDie()->ToString(), "");
+ for (int i = 1; i < num_records; ++i) {
+ const auto &value_at_position =
response->GetScalar(i).ValueOrDie()->ToString();
+ EXPECT_NE(value_at_position,
+ response->GetScalar(i -
1).ValueOrDie()->ToString());
+ }
Review comment:
May be also add assertion that output are sized correctly, that is each
output data is 40 and 64 length string for sha128 and sha256 resp.
##########
File path: cpp/src/gandiva/hash_utils.cc
##########
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstring>
+#include "openssl/evp.h"
+#include "gandiva/hash_utils.h"
+#include "gandiva/execution_context.h"
+#include "gandiva/gdv_function_stubs.h"
+
+namespace gandiva {
+ const char* HashUtils::HashUsingSha256(int64_t context,
+ const void* message,
+ size_t message_length,
+ int32_t *out_length) {
+ // The buffer size is the hash size + null character
+ int sha256_result_length = 65;
+ return HashUtils::GetHash(context, message, message_length, EVP_sha256(),
+ sha256_result_length,
out_length);
+ }
+ const char* HashUtils::HashUsingSha128(int64_t context,
+ const void* message,
+ size_t message_length,
+ int32_t *out_length) {
+ // The buffer size is the hash size + null character
+ int sha128_result_length = 41;
+ return HashUtils::GetHash(context, message, message_length, EVP_sha1(),
+ sha128_result_length,
out_length);
+ }
+
+ const char* HashUtils::GetHash(int64_t context,
+ const void* message,
+ size_t message_length,
+ const EVP_MD *hash_type,
+ int result_buf_size,
+ int32_t *out_length) {
+ EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
+
+ if (md_ctx == nullptr) {
+ HashUtils::ErrorMessage(context, "Could not allocate memory "
Review comment:
-> "Could not create context"
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
