HDFS-8764. Generate Hadoop RPC stubs from protobuf definitions. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f34bda72 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f34bda72 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f34bda72 Branch: refs/heads/8b02d962b291afe4b08c47f0398c1db0709419a1 Commit: f34bda720184cc2a7e3ac48916c693519387539b Parents: 6416b38 Author: Haohui Mai <whe...@apache.org> Authored: Mon Jul 13 16:53:13 2015 -0700 Committer: Haohui Mai <whe...@apache.org> Committed: Tue Jul 14 12:47:00 2015 -0700 ---------------------------------------------------------------------- .../native/libhdfspp/lib/proto/CMakeLists.txt | 46 +++++++++- .../native/libhdfspp/lib/proto/cpp_helpers.h | 82 +++++++++++++++++ .../libhdfspp/lib/proto/protoc_gen_hrpc.cc | 94 ++++++++++++++++++++ 3 files changed, 221 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f34bda72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt index 156a7f4..3f703b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt @@ -18,4 +18,48 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${COMMON_PROTO_DIR}/Security.proto ) -add_library(proto ${PROTO_SRCS} ${PROTO_HDRS}) +add_executable(protoc-gen-hrpc protoc_gen_hrpc.cc) +target_link_libraries(protoc-gen-hrpc ${PROTOBUF_PROTOC_LIBRARY} ${PROTOBUF_LIBRARY}) + +function(GEN_HRPC SRCS) + if(NOT ARGN) + message(SEND_ERROR "Error: GEN_HRPC() called without any proto files") + return() + endif() + + if(DEFINED PROTOBUF_IMPORT_DIRS) + foreach(DIR ${PROTOBUF_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.hrpc.inl") + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.hrpc.inl" + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS --plugin=protoc-gen-hrpc=${CMAKE_CURRENT_BINARY_DIR}/protoc-gen-hrpc --hrpc_out=${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL} + DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE} protoc-gen-hrpc + COMMENT "Running HRPC protocol buffer compiler on ${FIL}" + VERBATIM ) + endforeach() + + set_source_files_properties(${${SRCS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} PARENT_SCOPE) +endfunction() + +gen_hrpc(HRPC_SRCS + ${CLIENT_PROTO_DIR}/ClientNamenodeProtocol.proto +) + +add_library(proto ${PROTO_SRCS} ${PROTO_HDRS} ${HRPC_SRCS}) http://git-wip-us.apache.org/repos/asf/hadoop/blob/f34bda72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/cpp_helpers.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/cpp_helpers.h b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/cpp_helpers.h new file mode 100644 index 0000000..6f380ad --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/cpp_helpers.h @@ -0,0 +1,82 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: ken...@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + +#ifndef LIBHDFSPP_PROTO_CPP_HELPERS_H_ +#define LIBHDFSPP_PROTO_CPP_HELPERS_H_ + +#include <string> + +/** + * The functions in this file are derived from the original implementation of + *the protobuf library from Google. + **/ + +static inline std::string StripProto(const std::string &str) { + static const std::string kExtension = ".proto"; + if (str.size() >= kExtension.size() && + str.compare(str.size() - kExtension.size(), kExtension.size(), + kExtension) == 0) { + return str.substr(0, str.size() - kExtension.size()); + } else { + return str; + } +} + +static inline std::string ToCamelCase(const std::string &input) { + bool cap_next_letter = true; + std::string result; + // Note: I distrust ctype.h due to locales. + for (size_t i = 0; i < input.size(); i++) { + if ('a' <= input[i] && input[i] <= 'z') { + if (cap_next_letter) { + result += input[i] + ('A' - 'a'); + } else { + result += input[i]; + } + cap_next_letter = false; + } else if ('A' <= input[i] && input[i] <= 'Z') { + // Capital letters are left as-is. + result += input[i]; + cap_next_letter = false; + } else if ('0' <= input[i] && input[i] <= '9') { + result += input[i]; + cap_next_letter = true; + } else { + cap_next_letter = true; + } + } + return result; +} + +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/f34bda72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc new file mode 100644 index 0000000..f384e36 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cpp_helpers.h" + +#include <google/protobuf/compiler/code_generator.h> +#include <google/protobuf/compiler/plugin.h> +#include <google/protobuf/descriptor.h> +#include <google/protobuf/io/printer.h> +#include <google/protobuf/io/zero_copy_stream.h> +#include <google/protobuf/stubs/common.h> + +using ::google::protobuf::FileDescriptor; +using ::google::protobuf::MethodDescriptor; +using ::google::protobuf::ServiceDescriptor; +using ::google::protobuf::compiler::CodeGenerator; +using ::google::protobuf::compiler::GeneratorContext; +using ::google::protobuf::io::Emiter; +using ::google::protobuf::io::ZeroCopyOutputStream; + +class StubGenerator : public CodeGenerator { +public: + virtual bool Generate(const FileDescriptor *file, const std::string &, + GeneratorContext *ctx, + std::string *error) const override; + +private: + void EmitService(const ServiceDescriptor *service, Emiter *out) const; + void EmitMethod(const MethodDescriptor *method, Emiter *out) const; +}; + +bool StubGenerator::Generate(const FileDescriptor *file, const std::string &, + GeneratorContext *ctx, std::string *) const { + namespace pb = ::google::protobuf; + std::unique_ptr<ZeroCopyOutputStream> os( + ctx->Open(StripProto(file->name()) + ".hrpc.inl")); + Emiter out(os.get(), '$'); + for (int i = 0; i < file->service_count(); ++i) { + const ServiceDescriptor *service = file->service(i); + EmitService(service, &out); + } + return true; +} + +void StubGenerator::EmitService(const ServiceDescriptor *service, + Emiter *out) const { + out->Emit("\n// GENERATED AUTOMATICALLY. DO NOT MODIFY.\n" + "class $service$ {\n" + "private:\n" + " ::hdfs::RpcEngine *const engine_;\n" + "public:\n" + " typedef std::function<void(const ::hdfs::Status &)> Callback;\n" + " typedef ::google::protobuf::MessageLite Message;\n" + " inline $service$(::hdfs::RpcEngine *engine)\n" + " : engine_(engine) {}\n", + "service", service->name()); + for (int i = 0; i < service->method_count(); ++i) { + const MethodDescriptor *method = service->method(i); + EmitMethod(method, out); + } + out->Emit("};\n"); +} + +void StubGenerator::EmitMethod(const MethodDescriptor *method, + Emiter *out) const { + out->Emit("\n inline void $camel_method$(const Message *req, " + "const std::shared_ptr<Message> &resp, " + "Callback &&handler) {\n" + " engine_->AsyncRpc(\"$method$\", req, resp, std::move(handler));\n" + " }\n", + "camel_method", ToCamelCase(method->name()), + "method", method->name() + ); +} + +int main(int argc, char *argv[]) { + StubGenerator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); +}
