This is an automated email from the ASF dual-hosted git repository.
xyz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pulsar-client-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new d2cfabd Support the base64 encoded credentials for OAuth2
authentication (#249)
d2cfabd is described below
commit d2cfabd3cb158f53605b781699b331c6c535456d
Author: Yunze Xu <[email protected]>
AuthorDate: Mon Apr 17 09:53:20 2023 +0800
Support the base64 encoded credentials for OAuth2 authentication (#249)
Fixes https://github.com/apache/pulsar-client-python/issues/101
### Motivation
Currently the `private_key` field of the JSON passed to `AuthOauth2`
only represents the path to the file, we need to support passing the
base64 encoded JSON string.
### Modifications
- Add the util methods `encode` and `decode` in namespace
`pulsar::base64` for base64 serialization. Then add `Base64Test.cc`
for it.
- Support the following URL representations for `private_key`:
1. `file:///path/to/key/file`
2. `data:application/json;base64,xxxx`
- Add `Oauth2Test` and set up the test environment for it independently
with a Docker compose file.
---
.github/workflows/ci-pr-validation.yaml | 9 ----
lib/Base64Utils.h | 59 +++++++++++++++++++++
lib/ProtobufNativeSchema.cc | 22 ++------
lib/auth/AuthOauth2.cc | 65 ++++++++++++++++++++++-
lib/auth/AuthOauth2.h | 1 +
run-unit-tests.sh | 24 ++++++++-
test-conf/cpp_credentials_file.json | 2 +
tests/Base64Test.cc | 38 ++++++++++++++
tests/CMakeLists.txt | 4 ++
tests/oauth2/Oauth2Test.cc | 91 +++++++++++++++++++++++++++++++++
tests/oauth2/docker-compose.yml | 46 +++++++++++++++++
11 files changed, 330 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/ci-pr-validation.yaml
b/.github/workflows/ci-pr-validation.yaml
index 1bb7cb3..a736bb6 100644
--- a/.github/workflows/ci-pr-validation.yaml
+++ b/.github/workflows/ci-pr-validation.yaml
@@ -93,18 +93,9 @@ jobs:
- name: Build
run: make -j8
- - name: Start Pulsar service
- run: ./pulsar-test-service-start.sh
-
- - name: Run ConnectionFailTest
- run: ./tests/ConnectionFailTest --gtest_repeat=20
-
- name: Run unit tests
run: RETRY_FAILED=3 ./run-unit-tests.sh
- - name: Stop Pulsar service
- run: ./pulsar-test-service-stop.sh
-
cpp-build-windows:
timeout-minutes: 120
diff --git a/lib/Base64Utils.h b/lib/Base64Utils.h
new file mode 100644
index 0000000..7706089
--- /dev/null
+++ b/lib/Base64Utils.h
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#pragma once
+
+#include <boost/archive/iterators/base64_from_binary.hpp>
+#include <boost/archive/iterators/binary_from_base64.hpp>
+#include <boost/archive/iterators/transform_width.hpp>
+#include <string>
+
+namespace pulsar {
+
+namespace base64 {
+
+inline std::string encode(const char* data, size_t size) {
+ using namespace boost::archive::iterators;
+ using Base64Iter = base64_from_binary<transform_width<const char*, 6, 8>>;
+
+ std::string encoded{Base64Iter(data), Base64Iter(data + size)};
+ const auto numPaddings = (4 - encoded.size()) % 4;
+ encoded.append(numPaddings, '=');
+ return encoded;
+}
+
+template <typename CharContainer>
+inline std::string encode(const CharContainer& container) {
+ return encode(container.data(), container.size());
+}
+
+inline std::string decode(const std::string& encoded) {
+ using namespace boost::archive::iterators;
+ using Base64Iter =
transform_width<binary_from_base64<std::string::const_iterator>, 8, 6>;
+
+ std::string result{Base64Iter(encoded.cbegin()),
Base64Iter(encoded.cend())};
+ // There could be '\0's at the tail, it could cause "garbage after data"
error when parsing JSON
+ while (!result.empty() && result.back() == '\0') {
+ result.pop_back();
+ }
+ return result;
+}
+
+} // namespace base64
+
+} // namespace pulsar
diff --git a/lib/ProtobufNativeSchema.cc b/lib/ProtobufNativeSchema.cc
index edae2ec..5cddf74 100644
--- a/lib/ProtobufNativeSchema.cc
+++ b/lib/ProtobufNativeSchema.cc
@@ -20,11 +20,11 @@
#include <google/protobuf/descriptor.pb.h>
-#include <boost/archive/iterators/base64_from_binary.hpp>
-#include <boost/archive/iterators/transform_width.hpp>
#include <stdexcept>
#include <vector>
+#include "Base64Utils.h"
+
using google::protobuf::FileDescriptor;
using google::protobuf::FileDescriptorSet;
@@ -45,26 +45,10 @@ SchemaInfo createProtobufNativeSchema(const
google::protobuf::Descriptor* descri
FileDescriptorSet fileDescriptorSet;
internalCollectFileDescriptors(fileDescriptor, fileDescriptorSet);
- using namespace boost::archive::iterators;
- using base64 = base64_from_binary<transform_width<const char*, 6, 8>>;
-
std::vector<char> bytes(fileDescriptorSet.ByteSizeLong());
fileDescriptorSet.SerializeToArray(bytes.data(), bytes.size());
- std::string base64String{base64(bytes.data()), base64(bytes.data() +
bytes.size())};
- // Pulsar broker only supports decoding Base64 with padding so we need to
add padding '=' here
- const size_t numPadding = 4 - base64String.size() % 4;
- if (numPadding <= 2) {
- for (size_t i = 0; i < numPadding; i++) {
- base64String.push_back('=');
- }
- } else if (numPadding == 3) {
- // The length of encoded Base64 string (without padding) should not be
4N+1
- throw std::runtime_error("Unexpected padding number (3), the encoded
Base64 string is:\n" +
- base64String);
- } // else numPadding == 4, which means no padding characters need to be
added
-
- const std::string schemaJson = R"({"fileDescriptorSet":")" + base64String +
+ const std::string schemaJson = R"({"fileDescriptorSet":")" +
base64::encode(bytes) +
R"(","rootMessageTypeName":")" +
rootMessageTypeName +
R"(","rootFileDescriptorName":")" +
rootFileDescriptorName + R"("})";
diff --git a/lib/auth/AuthOauth2.cc b/lib/auth/AuthOauth2.cc
index 1592827..919f2bf 100644
--- a/lib/auth/AuthOauth2.cc
+++ b/lib/auth/AuthOauth2.cc
@@ -26,6 +26,7 @@
#include <stdexcept>
#include "InitialAuthData.h"
+#include "lib/Base64Utils.h"
#include "lib/LogUtils.h"
DECLARE_LOG_OBJECT()
@@ -113,10 +114,52 @@ Oauth2Flow::~Oauth2Flow() {}
KeyFile KeyFile::fromParamMap(ParamMap& params) {
const auto it = params.find("private_key");
- if (it != params.cend()) {
+ if (it == params.cend()) {
+ return {params["client_id"], params["client_secret"]};
+ }
+
+ const std::string& url = it->second;
+ size_t startPos = 0;
+ auto getPrefix = [&url, &startPos](char separator) -> std::string {
+ const size_t endPos = url.find(separator, startPos);
+ if (endPos == std::string::npos) {
+ return "";
+ }
+ const auto prefix = url.substr(startPos, endPos - startPos);
+ startPos = endPos + 1;
+ return prefix;
+ };
+
+ const auto protocol = getPrefix(':');
+ // If the private key is not a URL, treat it as the file path
+ if (protocol.empty()) {
return fromFile(it->second);
+ }
+
+ if (protocol == "file") {
+ // URL is "file://..." or "file:..."
+ if (url.size() > startPos + 2 && url[startPos + 1] == '/' &&
url[startPos + 2] == '/') {
+ return fromFile(url.substr(startPos + 2));
+ } else {
+ return fromFile(url.substr(startPos));
+ }
+ } else if (protocol == "data") {
+ // Only support base64 encoded data from a JSON string. The URL should
be:
+ // "data:application/json;base64,..."
+ const auto contentType = getPrefix(';');
+ if (contentType != "application/json") {
+ LOG_ERROR("Unsupported content type: " << contentType);
+ return {};
+ }
+ const auto encodingType = getPrefix(',');
+ if (encodingType != "base64") {
+ LOG_ERROR("Unsupported encoding type: " << encodingType);
+ return {};
+ }
+ return fromBase64(url.substr(startPos));
} else {
- return {params["client_id"], params["client_secret"]};
+ LOG_ERROR("Unsupported protocol: " << protocol);
+ return {};
}
}
@@ -139,6 +182,24 @@ KeyFile KeyFile::fromFile(const std::string&
credentialsFilePath) {
}
}
+KeyFile KeyFile::fromBase64(const std::string& encoded) {
+ boost::property_tree::ptree root;
+ std::stringstream stream;
+ stream << base64::decode(encoded);
+ try {
+ boost::property_tree::read_json(stream, root);
+ } catch (const boost::property_tree::json_parser_error& e) {
+ LOG_ERROR("Failed to parse credentials from " << stream.str());
+ return {};
+ }
+ try {
+ return {root.get<std::string>("client_id"),
root.get<std::string>("client_secret")};
+ } catch (const boost::property_tree::ptree_error& e) {
+ LOG_ERROR("Failed to get client_id or client_secret in " <<
stream.str() << ": " << e.what());
+ return {};
+ }
+}
+
ClientCredentialFlow::ClientCredentialFlow(ParamMap& params)
: issuerUrl_(params["issuer_url"]),
keyFile_(KeyFile::fromParamMap(params)),
diff --git a/lib/auth/AuthOauth2.h b/lib/auth/AuthOauth2.h
index 31c6122..e65d0e2 100644
--- a/lib/auth/AuthOauth2.h
+++ b/lib/auth/AuthOauth2.h
@@ -48,6 +48,7 @@ class KeyFile {
KeyFile() : valid_(false) {}
static KeyFile fromFile(const std::string& filename);
+ static KeyFile fromBase64(const std::string& encoded);
};
class ClientCredentialFlow : public Oauth2Flow {
diff --git a/run-unit-tests.sh b/run-unit-tests.sh
index a5489a7..8be29f0 100755
--- a/run-unit-tests.sh
+++ b/run-unit-tests.sh
@@ -23,7 +23,27 @@ set -e
ROOT_DIR=$(git rev-parse --show-toplevel)
cd $ROOT_DIR
-pushd tests
+if [[ ! $CMAKE_BUILD_DIRECTORY ]]; then
+ CMAKE_BUILD_DIRECTORY=.
+fi
+
+export http_proxy=
+export https_proxy=
+
+# Run OAuth2 tests
+docker compose -f tests/oauth2/docker-compose.yml up -d
+# Wait until the namespace is created, currently there is no good way to check
it
+# because it's hard to configure OAuth2 authentication via CLI.
+sleep 15
+$CMAKE_BUILD_DIRECTORY/tests/Oauth2Test
+docker compose -f tests/oauth2/docker-compose.yml down
+
+./pulsar-test-service-start.sh
+
+pushd $CMAKE_BUILD_DIRECTORY/tests
+
+# Avoid this test is still flaky, see
https://github.com/apache/pulsar-client-cpp/pull/217
+./ConnectionFailTest --gtest_repeat=20
export RETRY_FAILED="${RETRY_FAILED:-1}"
@@ -54,4 +74,6 @@ fi
popd
+./pulsar-test-service-stop.sh
+
exit $RES
diff --git a/test-conf/cpp_credentials_file.json
b/test-conf/cpp_credentials_file.json
index db1eccd..cbc7e81 100644
--- a/test-conf/cpp_credentials_file.json
+++ b/test-conf/cpp_credentials_file.json
@@ -1,4 +1,6 @@
{
+ "issuer_url": "https://dev-kt-aa9ne.us.auth0.com";,
+ "audience": "https://dev-kt-aa9ne.us.auth0.com/api/v2/";,
"client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x",
"client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb"
}
diff --git a/tests/Base64Test.cc b/tests/Base64Test.cc
new file mode 100644
index 0000000..487dc08
--- /dev/null
+++ b/tests/Base64Test.cc
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <gtest/gtest.h>
+
+#include <cstring>
+
+#include "lib/Base64Utils.h"
+
+using namespace pulsar;
+
+TEST(Base64Test, testJsonEncodeDecode) {
+ const std::string s1 = R"("{"key":"value"}")";
+ const auto s2 = base64::decode(base64::encode(s1));
+ ASSERT_EQ(s1, s2);
+}
+
+TEST(Base64Test, testPaddings) {
+ auto encode = [](const char* s) { return base64::encode(s, strlen(s)); };
+ ASSERT_EQ(encode("x"), "eA=="); // 2 paddings
+ ASSERT_EQ(encode("xy"), "eHk="); // 1 padding
+ ASSERT_EQ(encode("xyz"), "eHl6"); // 0 padding
+}
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 05ca139..cfc9e27 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -64,3 +64,7 @@ if (UNIX)
add_executable(ConnectionFailTest unix/ConnectionFailTest.cc HttpHelper.cc)
target_link_libraries(ConnectionFailTest ${CLIENT_LIBS} pulsarStatic
${GTEST_LIBRARY_PATH})
endif ()
+
+add_executable(Oauth2Test oauth2/Oauth2Test.cc)
+target_compile_options(Oauth2Test PRIVATE
"-DTEST_ROOT_PATH=\"${CMAKE_CURRENT_SOURCE_DIR}\"")
+target_link_libraries(Oauth2Test ${CLIENT_LIBS} pulsarStatic
${GTEST_LIBRARY_PATH})
diff --git a/tests/oauth2/Oauth2Test.cc b/tests/oauth2/Oauth2Test.cc
new file mode 100644
index 0000000..6264620
--- /dev/null
+++ b/tests/oauth2/Oauth2Test.cc
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// Run `docker-compose up -d` to set up the test environment for this test.
+#include <gtest/gtest.h>
+#include <pulsar/Client.h>
+
+#include <boost/property_tree/json_parser.hpp>
+#include <boost/property_tree/ptree.hpp>
+
+#include "lib/Base64Utils.h"
+
+using namespace pulsar;
+
+#ifndef TEST_ROOT_PATH
+#define TEST_ROOT_PATH "."
+#endif
+
+static const std::string gKeyPath = std::string(TEST_ROOT_PATH) +
"/../test-conf/cpp_credentials_file.json";
+static std::string gClientId;
+static std::string gClientSecret;
+static ParamMap gCommonParams;
+
+static Result testCreateProducer(const std::string& privateKey);
+
+static std::string credentials(const std::string& clientId, const std::string&
clientSecret) {
+ return base64::encode(R"({"client_id":")" + clientId +
R"(","client_secret":")" + clientSecret + R"("})");
+}
+
+TEST(Oauth2Test, testBase64Key) {
+ ASSERT_EQ(ResultOk,
+ testCreateProducer("data:application/json;base64," +
credentials(gClientId, gClientSecret)));
+ ASSERT_EQ(ResultAuthenticationError,
+ testCreateProducer("data:application/json;base64," +
credentials("test-id", "test-secret")));
+}
+
+TEST(Oauth2Test, testFileKey) {
+ ASSERT_EQ(ResultOk, testCreateProducer("file://" + gKeyPath));
+ ASSERT_EQ(ResultOk, testCreateProducer("file:" + gKeyPath));
+ ASSERT_EQ(ResultOk, testCreateProducer(gKeyPath));
+ ASSERT_EQ(ResultAuthenticationError,
testCreateProducer("file:///tmp/file-not-exist"));
+}
+
+TEST(Oauth2Test, testWrongUrl) {
+ ASSERT_EQ(ResultAuthenticationError,
+ testCreateProducer("data:text/plain;base64," +
credentials(gClientId, gClientSecret)));
+ ASSERT_EQ(ResultAuthenticationError,
+ testCreateProducer("data:application/json;text," +
credentials(gClientId, gClientSecret)));
+ ASSERT_EQ(ResultAuthenticationError, testCreateProducer("my-protocol:" +
gKeyPath));
+}
+
+int main(int argc, char* argv[]) {
+ std::cout << "Load Oauth2 configs from " << gKeyPath << "..." << std::endl;
+ boost::property_tree::ptree root;
+ boost::property_tree::read_json(gKeyPath, root);
+ gClientId = root.get<std::string>("client_id");
+ gClientSecret = root.get<std::string>("client_secret");
+ gCommonParams["issuer_url"] = root.get<std::string>("issuer_url");
+ gCommonParams["audience"] = root.get<std::string>("audience");
+
+ ::testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
+ return 0;
+}
+
+static Result testCreateProducer(const std::string& privateKey) {
+ ClientConfiguration conf;
+ auto params = gCommonParams;
+ params["private_key"] = privateKey;
+ conf.setAuth(AuthOauth2::create(params));
+ Client client{"pulsar://localhost:6650", conf};
+ Producer producer;
+ const auto result = client.createProducer("oauth2-test", producer);
+ client.close();
+ return result;
+}
diff --git a/tests/oauth2/docker-compose.yml b/tests/oauth2/docker-compose.yml
new file mode 100644
index 0000000..0ab818e
--- /dev/null
+++ b/tests/oauth2/docker-compose.yml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+version: '3'
+networks:
+ pulsar:
+ driver: bridge
+services:
+ standalone:
+ image: apachepulsar/pulsar:latest
+ container_name: standalone
+ hostname: local
+ restart: "no"
+ networks:
+ - pulsar
+ environment:
+ - metadataStoreUrl=zk:localhost:2181
+ - clusterName=standalone-oauth2
+ - advertisedAddress=localhost
+ - advertisedListeners=external:pulsar://localhost:6650
+ - PULSAR_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m
+ - PULSAR_PREFIX_authenticationEnabled=true
+ -
PULSAR_PREFIX_authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken
+ -
PULSAR_PREFIX_tokenPublicKey=data:;base64,MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2tZd/4gJda3U2Pc3tpgRAN7JPGWx/Gn17v/0IiZlNNRbP/Mmf0Vc6G1qsnaRaWNWOR+t6/a6ekFHJMikQ1N2X6yfz4UjMc8/G2FDPRmWjA+GURzARjVhxc/BBEYGoD0Kwvbq/u9CZm2QjlKrYaLfg3AeB09j0btNrDJ8rBsNzU6AuzChRvXj9IdcE/A/4N/UQ+S9cJ4UXP6NJbToLwajQ5km+CnxdGE6nfB7LWHvOFHjn9C2Rb9e37CFlmeKmIVFkagFM0gbmGOb6bnGI8Bp/VNGV0APef4YaBvBTqwoZ1Z4aDHy5eRxXfAMdtBkBupmBXqL6bpd15XRYUbu/7ck9QIDAQAB
+ -
PULSAR_PREFIX_brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2
+ -
PULSAR_PREFIX_brokerClientAuthenticationParameters={"issuerUrl":"https://dev-kt-aa9ne.us.auth0.com","audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/","privateKey":"data:application/json;base64,ewogICAgICAgICAgICAiY2xpZW50X2lkIjoiWGQyM1JIc1VudlVsUDd3Y2hqTllPYUlmYXpnZUhkOXgiLAogICAgICAgICAgICAiY2xpZW50X3NlY3JldCI6InJUN3BzN1dZOHVoZFZ1QlRLV1prdHR3TGRRb3RtZEVsaWFNNXJMZm1nTmlidnF6aVotZzA3Wkg1Mk5fcG9HQWIiCiAgICAgICAgfQ=="}
+ ports:
+ - "6650:6650"
+ - "8080:8080"
+ command: bash -c "bin/apply-config-from-env.py conf/standalone.conf &&
exec bin/pulsar standalone -nss -nfw"
