http://git-wip-us.apache.org/repos/asf/incubator-ratis/blob/7e71a2e0/ratis-client/src/main/java/org/apache/ratis/client/RaftClientRequestSender.java ---------------------------------------------------------------------- diff --git a/ratis-client/src/main/java/org/apache/ratis/client/RaftClientRequestSender.java b/ratis-client/src/main/java/org/apache/ratis/client/RaftClientRequestSender.java new file mode 100644 index 0000000..b2541e1 --- /dev/null +++ b/ratis-client/src/main/java/org/apache/ratis/client/RaftClientRequestSender.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.client; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.ratis.protocol.RaftClientReply; +import org.apache.ratis.protocol.RaftClientRequest; +import org.apache.ratis.protocol.RaftPeer; + +/** Send requests to a raft service. */ +public interface RaftClientRequestSender extends Closeable { + /** Send a request. */ + RaftClientReply sendRequest(RaftClientRequest request) throws IOException; + + /** Add the information of the given raft servers */ + void addServers(Iterable<RaftPeer> servers); +}
http://git-wip-us.apache.org/repos/asf/incubator-ratis/blob/7e71a2e0/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java ---------------------------------------------------------------------- diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java new file mode 100644 index 0000000..24bb4ec --- /dev/null +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.client.impl; + +import org.apache.ratis.shaded.com.google.protobuf.ByteString; +import org.apache.ratis.shaded.proto.RaftProtos.*; +import org.apache.ratis.protocol.*; +import org.apache.ratis.util.ProtoUtils; + +import java.util.Arrays; + +public class ClientProtoUtils { + public static RaftRpcReplyProto.Builder toRaftRpcReplyProtoBuilder( + String requestorId, String replyId, long seqNum, boolean success) { + return RaftRpcReplyProto.newBuilder() + .setRequestorId(requestorId).setReplyId(replyId).setSeqNum(seqNum) + .setSuccess(success); + } + + public static RaftRpcRequestProto.Builder toRaftRpcRequestProtoBuilder( + String requesterId, String replyId, long seqNum) { + return RaftRpcRequestProto.newBuilder() + .setRequestorId(requesterId).setReplyId(replyId).setSeqNum(seqNum); + } + + public static RaftClientRequest toRaftClientRequest(RaftClientRequestProto p) { + return new RaftClientRequest(p.getRpcRequest().getRequestorId(), + p.getRpcRequest().getReplyId(), p.getRpcRequest().getSeqNum(), + toMessage(p.getMessage()), p.getReadOnly()); + } + + public static RaftClientRequestProto toRaftClientRequestProto( + RaftClientRequest request) { + return RaftClientRequestProto.newBuilder() + .setRpcRequest(toRaftRpcRequestProtoBuilder(request.getRequestorId(), + request.getReplierId(), request.getSeqNum())) + .setMessage(toClientMessageEntryProto(request.getMessage())) + .setReadOnly(request.isReadOnly()) + .build(); + } + + public static RaftClientRequestProto genRaftClientRequestProto( + String requestorId, String replierId, long seqNum, ByteString content, + boolean readOnly) { + return RaftClientRequestProto.newBuilder() + .setRpcRequest(toRaftRpcRequestProtoBuilder(requestorId, replierId, seqNum)) + .setMessage(ClientMessageEntryProto.newBuilder().setContent(content)) + .setReadOnly(readOnly) + .build(); + } + + public static RaftClientReplyProto toRaftClientReplyProto( + RaftClientReply reply) { + final RaftClientReplyProto.Builder b = RaftClientReplyProto.newBuilder(); + if (reply != null) { + b.setRpcReply(toRaftRpcReplyProtoBuilder(reply.getRequestorId(), + reply.getReplierId(), reply.getSeqNum(), reply.isSuccess())); + if (reply.getMessage() != null) { + b.setMessage(toClientMessageEntryProto(reply.getMessage())); + } + if (reply.isNotLeader()) { + b.setIsNotLeader(true); + final RaftPeer suggestedLeader = reply.getNotLeaderException() + .getSuggestedLeader(); + if (suggestedLeader != null) { + b.setSuggestedLeader(ProtoUtils.toRaftPeerProto(suggestedLeader)); + } + b.addAllPeersInConf(ProtoUtils.toRaftPeerProtos( + Arrays.asList(reply.getNotLeaderException().getPeers()))); + } + } + return b.build(); + } + + public static RaftClientReply toRaftClientReply( + RaftClientReplyProto replyProto) { + final RaftRpcReplyProto rp = replyProto.getRpcReply(); + NotLeaderException e = null; + if (replyProto.getIsNotLeader()) { + final RaftPeer suggestedLeader = replyProto.hasSuggestedLeader() ? + ProtoUtils.toRaftPeer(replyProto.getSuggestedLeader()) : null; + final RaftPeer[] peers = ProtoUtils.toRaftPeerArray( + replyProto.getPeersInConfList()); + e = new NotLeaderException(rp.getReplyId(), suggestedLeader, peers); + } + return new RaftClientReply(rp.getRequestorId(), rp.getReplyId(), + rp.getSeqNum(), rp.getSuccess(), toMessage(replyProto.getMessage()), e); + } + + public static Message toMessage(final ClientMessageEntryProto p) { + return p::getContent; + } + + public static ClientMessageEntryProto toClientMessageEntryProto(Message message) { + return ClientMessageEntryProto.newBuilder() + .setContent(message.getContent()).build(); + } + + public static SetConfigurationRequest toSetConfigurationRequest( + SetConfigurationRequestProto p) { + final RaftRpcRequestProto m = p.getRpcRequest(); + final RaftPeer[] peers = ProtoUtils.toRaftPeerArray(p.getPeersList()); + return new SetConfigurationRequest(m.getRequestorId(), m.getReplyId(), + p.getRpcRequest().getSeqNum(), peers); + } + + public static SetConfigurationRequestProto toSetConfigurationRequestProto( + SetConfigurationRequest request) { + return SetConfigurationRequestProto.newBuilder() + .setRpcRequest(toRaftRpcRequestProtoBuilder(request.getRequestorId(), + request.getReplierId(), request.getSeqNum())) + .addAllPeers(ProtoUtils.toRaftPeerProtos( + Arrays.asList(request.getPeersInNewConf()))) + .build(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ratis/blob/7e71a2e0/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java ---------------------------------------------------------------------- diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java new file mode 100644 index 0000000..e101554 --- /dev/null +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.client.impl; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.ratis.client.RaftClient; +import org.apache.ratis.client.RaftClientConfigKeys; +import org.apache.ratis.client.RaftClientRequestSender; +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.protocol.*; +import org.apache.ratis.util.RaftUtils; +import org.apache.ratis.util.StringUtils; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** A client who sends requests to a raft service. */ +public final class RaftClientImpl implements RaftClient { + public static final long DEFAULT_SEQNUM = 0; + + private final String clientId; + private final RaftClientRequestSender requestSender; + private final Map<String, RaftPeer> peers; + private final int retryInterval; + + private volatile String leaderId; + + public RaftClientImpl( + String clientId, Collection<RaftPeer> peers, + RaftClientRequestSender requestSender, String leaderId, + RaftProperties properties) { + this.clientId = clientId; + this.requestSender = requestSender; + this.peers = peers.stream().collect( + Collectors.toMap(RaftPeer::getId, Function.identity())); + this.leaderId = leaderId != null? leaderId : peers.iterator().next().getId(); + this.retryInterval = properties.getInt( + RaftClientConfigKeys.RAFT_RPC_TIMEOUT_MS_KEY, + RaftClientConfigKeys.RAFT_RPC_TIMEOUT_MS_DEFAULT); + } + + @Override + public String getId() { + return clientId; + } + + @Override + public RaftClientReply send(Message message) throws IOException { + return send(message, false); + } + + @Override + public RaftClientReply sendReadOnly(Message message) throws IOException { + return send(message, true); + } + + private RaftClientReply send(Message message, boolean readOnly) throws IOException { + return sendRequestWithRetry(() -> new RaftClientRequest( + clientId, leaderId, DEFAULT_SEQNUM, message, readOnly)); + } + + @Override + public RaftClientReply setConfiguration(RaftPeer[] peersInNewConf) + throws IOException { + return sendRequestWithRetry(() -> new SetConfigurationRequest( + clientId, leaderId, DEFAULT_SEQNUM, peersInNewConf)); + } + + private RaftClientReply sendRequestWithRetry( + Supplier<RaftClientRequest> supplier) + throws InterruptedIOException, StateMachineException { + for(;;) { + final RaftClientRequest request = supplier.get(); + LOG.debug("{}: {}", clientId, request); + final RaftClientReply reply = sendRequest(request); + if (reply != null) { + LOG.debug("{}: {}", clientId, reply); + return reply; + } + + // sleep and then retry + try { + Thread.sleep(retryInterval); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw RaftUtils.toInterruptedIOException( + "Interrupted when sending " + request, ie); + } + } + } + + private RaftClientReply sendRequest(RaftClientRequest request) + throws StateMachineException { + try { + RaftClientReply reply = requestSender.sendRequest(request); + if (reply.isNotLeader()) { + handleNotLeaderException(request, reply.getNotLeaderException()); + return null; + } else { + return reply; + } + } catch (StateMachineException e) { + throw e; + } catch (IOException ioe) { + // TODO different retry policies for different exceptions + handleIOException(request, ioe, null); + } + return null; + } + + private void handleNotLeaderException(RaftClientRequest request, NotLeaderException nle) { + refreshPeers(nle.getPeers()); + final String newLeader = nle.getSuggestedLeader() == null? null + : nle.getSuggestedLeader().getId(); + handleIOException(request, nle, newLeader); + } + + private void refreshPeers(RaftPeer[] newPeers) { + if (newPeers != null && newPeers.length > 0) { + peers.clear(); + for (RaftPeer p : newPeers) { + peers.put(p.getId(), p); + } + // also refresh the rpc proxies for these peers + requestSender.addServers(Arrays.asList(newPeers)); + } + } + + private void handleIOException(RaftClientRequest request, IOException ioe, String newLeader) { + LOG.debug("{}: Failed with {}", clientId, ioe); + final String oldLeader = request.getReplierId(); + if (newLeader == null && oldLeader.equals(leaderId)) { + newLeader = StringUtils.next(oldLeader, peers.keySet()); + } + if (newLeader != null && oldLeader.equals(leaderId)) { + LOG.debug("{}: change Leader from {} to {}", clientId, oldLeader, newLeader); + this.leaderId = newLeader; + } + } + + @VisibleForTesting + public RaftClientRequestSender getRequestSender() { + return requestSender; + } + + @Override + public void close() throws IOException { + requestSender.close(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ratis/blob/7e71a2e0/ratis-common/RatisCommon.cmake ---------------------------------------------------------------------- diff --git a/ratis-common/RatisCommon.cmake b/ratis-common/RatisCommon.cmake new file mode 100644 index 0000000..e8f70eb --- /dev/null +++ b/ratis-common/RatisCommon.cmake @@ -0,0 +1,208 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Common CMake utilities and configuration, shared by all Native components. +# + +# +# Platform-specific prerequisite checks. +# + +if(CMAKE_SYSTEM_NAME STREQUAL "SunOS") + # Only 64-bit Java is supported. + if(NOT JVM_ARCH_DATA_MODEL EQUAL 64) + message(FATAL_ERROR "Unrecognised JVM_ARCH_DATA_MODEL '${JVM_ARCH_DATA_MODEL}'. " + "A 64-bit JVM must be used on Solaris, make sure that one is installed and, " + "if necessary, the MAVEN_OPTS environment variable includes '-d64'") + endif() + + # Only gcc is suported for now. + if(NOT(CMAKE_COMPILER_IS_GNUCC AND CMAKE_COMPILER_IS_GNUCXX)) + message(FATAL_ERROR "Only gcc is supported on Solaris") + endif() +endif() + +# +# Helper functions and macros. +# + +# Add flags to all the CMake compiler variables +macro(ratis_add_compiler_flags FLAGS) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAGS}") +endmacro() + +# Add flags to all the CMake linker variables. +macro(ratis_add_linker_flags FLAGS) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${FLAGS}") +endmacro() + +# Compile a library with both shared and static variants. +function(ratis_add_dual_library LIBNAME) + add_library(${LIBNAME} SHARED ${ARGN}) + add_library(${LIBNAME}_static STATIC ${ARGN}) + set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME}) +endfunction() + +# Link both a static and a dynamic target against some libraries. +function(ratis_target_link_dual_libraries LIBNAME) + target_link_libraries(${LIBNAME} ${ARGN}) + target_link_libraries(${LIBNAME}_static ${ARGN}) +endfunction() + +# Set all the output directories to the same place. +function(ratis_output_directory TGT DIR) + set_target_properties(${TGT} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + set_target_properties(${TGT} PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + set_target_properties(${TGT} PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") +endfunction() + +# Set the target directories for dynamic and static builds. +function(ratis_dual_output_directory TGT DIR) + ratis_output_directory(${TGT} "${DIR}") + ratis_output_directory(${TGT}_static "${DIR}") +endfunction() + +# Alter the behavior of find_package and find_library so that we find only +# shared libraries with a given version suffix. You should save +# CMAKE_FIND_LIBRARY_SUFFIXES before calling this function and restore it +# afterwards. On Windows this function is a no-op. Windows does not encode +# version number information information into library path names. +macro(ratis_set_find_shared_library_version LVERS) + if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + # Mac OS uses .dylib + set(CMAKE_FIND_LIBRARY_SUFFIXES ".${LVERS}.dylib") + elseif(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD") + # FreeBSD has always .so installed. + set(CMAKE_FIND_LIBRARY_SUFFIXES ".so") + elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + # Windows doesn't support finding shared libraries by version. + else() + # Most UNIX variants use .so + set(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${LVERS}") + endif() +endmacro() + +# Alter the behavior of find_package and find_library so that we find only +# shared libraries without any version suffix. You should save +# CMAKE_FIND_LIBRARY_SUFFIXES before calling this function and restore it +# afterwards. On Windows this function is a no-op. Windows does not encode +# version number information information into library path names. +macro(ratis_set_find_shared_library_without_version) + if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + # Mac OS uses .dylib + set(CMAKE_FIND_LIBRARY_SUFFIXES ".dylib") + elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + # No effect + else() + # Most UNIX variants use .so + set(CMAKE_FIND_LIBRARY_SUFFIXES ".so") + endif() +endmacro() + +# +# Configuration. +# + +# Initialise the shared gcc/g++ flags if they aren't already defined. +if(NOT DEFINED GCC_SHARED_FLAGS) + set(GCC_SHARED_FLAGS "-g -O2 -Wall -pthread -D_FILE_OFFSET_BITS=64") +endif() + +# Add in support other compilers here, if necessary, +# the assumption is that GCC or a GCC-compatible compiler is being used. + +# Set the shared GCC-compatible compiler and linker flags. +ratis_add_compiler_flags("${GCC_SHARED_FLAGS}") +ratis_add_linker_flags("${LINKER_SHARED_FLAGS}") + +# +# Linux-specific configuration. +# +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Make GNU extensions available. + ratis_add_compiler_flags("-D_GNU_SOURCE") + + # If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit. + if(JVM_ARCH_DATA_MODEL EQUAL 32) + # Force 32-bit code generation on amd64/x86_64, ppc64, sparc64 + if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_SYSTEM_PROCESSOR MATCHES ".*64") + ratis_add_compiler_flags("-m32") + ratis_add_linker_flags("-m32") + endif() + # Set CMAKE_SYSTEM_PROCESSOR to ensure that find_package(JNI) will use 32-bit libraries + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") + set(CMAKE_SYSTEM_PROCESSOR "i686") + endif() + endif() + + # Determine float ABI of JVM on ARM. + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") + find_program(READELF readelf) + if(READELF MATCHES "NOTFOUND") + message(WARNING "readelf not found; JVM float ABI detection disabled") + else(READELF MATCHES "NOTFOUND") + execute_process( + COMMAND ${READELF} -A ${JAVA_JVM_LIBRARY} + OUTPUT_VARIABLE JVM_ELF_ARCH + ERROR_QUIET) + if(NOT JVM_ELF_ARCH MATCHES "Tag_ABI_VFP_args: VFP registers") + # Test compilation with -mfloat-abi=softfp using an arbitrary libc function + # (typically fails with "fatal error: bits/predefs.h: No such file or directory" + # if soft-float dev libraries are not installed) + message("Soft-float JVM detected") + include(CMakePushCheckState) + cmake_push_check_state() + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -mfloat-abi=softfp") + include(CheckSymbolExists) + check_symbol_exists(exit stdlib.h SOFTFP_AVAILABLE) + if(NOT SOFTFP_AVAILABLE) + message(FATAL_ERROR "Soft-float dev libraries required (e.g. 'apt-get install libc6-dev-armel' on Debian/Ubuntu)") + endif() + cmake_pop_check_state() + ratis_add_compiler_flags("-mfloat-abi=softfp") + endif() + endif() + endif() + +# +# Solaris-specific configuration. +# +elseif(CMAKE_SYSTEM_NAME STREQUAL "SunOS") + # Solaris flags. 64-bit compilation is mandatory, and is checked earlier. + ratis_add_compiler_flags("-m64 -D_POSIX_C_SOURCE=200112L -D__EXTENSIONS__ -D_POSIX_PTHREAD_SEMANTICS") + set(CMAKE_C_FLAGS "-std=gnu99 ${CMAKE_C_FLAGS}") + set(CMAKE_CXX_FLAGS "-std=gnu++98 ${CMAKE_CXX_FLAGS}") + ratis_add_linker_flags("-m64") + + # CMAKE_SYSTEM_PROCESSOR is set to the output of 'uname -p', which on Solaris is + # the 'lowest' ISA supported, i.e. 'i386' or 'sparc'. However in order for the + # standard CMake modules to look in the right places it needs to reflect the required + # compilation mode, i.e. 64 bit. We therefore force it to either 'amd64' or 'sparcv9'. + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "i386") + set(CMAKE_SYSTEM_PROCESSOR "amd64") + set(CMAKE_LIBRARY_ARCHITECTURE "amd64") + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "sparc") + set(CMAKE_SYSTEM_PROCESSOR "sparcv9") + set(CMAKE_LIBRARY_ARCHITECTURE "sparcv9") + else() + message(FATAL_ERROR "Unrecognised CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}") + endif() +endif() http://git-wip-us.apache.org/repos/asf/incubator-ratis/blob/7e71a2e0/ratis-common/RatisJNI.cmake ---------------------------------------------------------------------- diff --git a/ratis-common/RatisJNI.cmake b/ratis-common/RatisJNI.cmake new file mode 100644 index 0000000..78d7ffd --- /dev/null +++ b/ratis-common/RatisJNI.cmake @@ -0,0 +1,97 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Common JNI detection for CMake, shared by all Native components. +# + +# Check the JVM_ARCH_DATA_MODEL variable as been set to 32 or 64 by maven. +if(NOT DEFINED JVM_ARCH_DATA_MODEL) + message(FATAL_ERROR "JVM_ARCH_DATA_MODEL is not defined") +elseif(NOT (JVM_ARCH_DATA_MODEL EQUAL 32 OR JVM_ARCH_DATA_MODEL EQUAL 64)) + message(FATAL_ERROR "JVM_ARCH_DATA_MODEL is not 32 or 64") +endif() + +# +# Linux-specific JNI configuration. +# +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Locate JNI_INCLUDE_DIRS and JNI_LIBRARIES. + # Since we were invoked from Maven, we know that the JAVA_HOME environment + # variable is valid. So we ignore system paths here and just use JAVA_HOME. + file(TO_CMAKE_PATH "$ENV{JAVA_HOME}" _java_home) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$") + set(_java_libarch "i386") + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") + set(_java_libarch "amd64") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") + set(_java_libarch "arm") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64le") + if(EXISTS "${_java_home}/jre/lib/ppc64le") + set(_java_libarch "ppc64le") + else() + set(_java_libarch "ppc64") + endif() + else() + set(_java_libarch ${CMAKE_SYSTEM_PROCESSOR}) + endif() + set(_JDK_DIRS "${_java_home}/jre/lib/${_java_libarch}/*" + "${_java_home}/jre/lib/${_java_libarch}" + "${_java_home}/jre/lib/*" + "${_java_home}/jre/lib" + "${_java_home}/lib/*" + "${_java_home}/lib" + "${_java_home}/include/*" + "${_java_home}/include" + "${_java_home}" + ) + find_path(JAVA_INCLUDE_PATH + NAMES jni.h + PATHS ${_JDK_DIRS} + NO_DEFAULT_PATH) + #In IBM java, it's jniport.h instead of jni_md.h + find_path(JAVA_INCLUDE_PATH2 + NAMES jni_md.h jniport.h + PATHS ${_JDK_DIRS} + NO_DEFAULT_PATH) + set(JNI_INCLUDE_DIRS ${JAVA_INCLUDE_PATH} ${JAVA_INCLUDE_PATH2}) + find_library(JAVA_JVM_LIBRARY + NAMES jvm JavaVM + PATHS ${_JDK_DIRS} + NO_DEFAULT_PATH) + set(JNI_LIBRARIES ${JAVA_JVM_LIBRARY}) + unset(_java_libarch) + unset(_java_home) + + message("JAVA_HOME=${JAVA_HOME}, JAVA_JVM_LIBRARY=${JAVA_JVM_LIBRARY}") + message("JAVA_INCLUDE_PATH=${JAVA_INCLUDE_PATH}, JAVA_INCLUDE_PATH2=${JAVA_INCLUDE_PATH2}") + if(JAVA_JVM_LIBRARY AND JAVA_INCLUDE_PATH AND JAVA_INCLUDE_PATH2) + message("Located all JNI components successfully.") + else() + message(FATAL_ERROR "Failed to find a viable JVM installation under JAVA_HOME.") + endif() + + # Use the standard FindJNI module to locate the JNI components. + find_package(JNI REQUIRED) + +# +# Otherwise, use the standard FindJNI module to locate the JNI components. +# +else() + find_package(JNI REQUIRED) +endif() http://git-wip-us.apache.org/repos/asf/incubator-ratis/blob/7e71a2e0/ratis-common/pom.xml ---------------------------------------------------------------------- diff --git a/ratis-common/pom.xml b/ratis-common/pom.xml new file mode 100644 index 0000000..1cbf058 --- /dev/null +++ b/ratis-common/pom.xml @@ -0,0 +1,143 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>ratis-project-dist</artifactId> + <groupId>org.apache.ratis</groupId> + <version>1.0-SNAPSHOT</version> + <relativePath>../ratis-project-dist</relativePath> + </parent> + + <artifactId>ratis-common</artifactId> + <name>Ratis Common</name> + + <dependencies> + <dependency> + <artifactId>ratis-proto-shaded</artifactId> + <groupId>org.apache.ratis</groupId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <id>native</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <properties> + <runningWithNative>true</runningWithNative> + </properties> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>enforce-os</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireOS> + <family>mac</family> + <family>unix</family> + <message>native build only supported on Mac or Unix</message> + </requireOS> + </rules> + <fail>true</fail> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>native-maven-plugin</artifactId> + <executions> + <execution> + <phase>compile</phase> + <goals> + <goal>javah</goal> + </goals> + <configuration> + <javahPath>${env.JAVA_HOME}/bin/javah</javahPath> + <javahClassNames> + <javahClassName>org.apache.ratis.io.nativeio.NativeIO</javahClassName> + <javahClassName>org.apache.ratis.util.NativeCrc32</javahClassName> + </javahClassNames> + <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> + <executions> + <execution> + <id>cmake-compile</id> + <phase>compile</phase> + <goals><goal>cmake-compile</goal></goals> + <configuration> + <source>${basedir}/src</source> + <vars> + <GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH> + <JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL> + </vars> + </configuration> + </execution> + <execution> + <id>test_bulk_crc32</id> + <goals><goal>cmake-test</goal></goals> + <phase>test</phase> + <configuration> + <binary>${project.build.directory}/native/test_bulk_crc32</binary> + <timeout>1200</timeout> + <results>${project.build.directory}/native-results</results> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/incubator-ratis/blob/7e71a2e0/ratis-common/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/ratis-common/src/CMakeLists.txt b/ratis-common/src/CMakeLists.txt new file mode 100644 index 0000000..460b3b1 --- /dev/null +++ b/ratis-common/src/CMakeLists.txt @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# CMake configuration. +# + +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/..) +include(RatisCommon) + +# Source and test locations. +set(SRC main/native/src/org/apache/ratis) +set(TST main/native/src/test/org/apache/ratis) + +# +# Main configuration. +# + +# The caller must specify where the generated headers have been placed. +if(NOT GENERATED_JAVAH) + message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH") +endif() + +# Configure JNI. +include(RatisJNI) + +# Build hardware CRC32 acceleration, if supported on the platform. +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") + set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_x86.c") +elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") + set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_aarch64.c") +else() + message("No HW CRC acceleration for ${CMAKE_SYSTEM_PROCESSOR}, falling back to SW") +endif() + +# Check for platform-specific functions and libraries. +include(CheckFunctionExists) +include(CheckLibraryExists) +check_function_exists(sync_file_range HAVE_SYNC_FILE_RANGE) +check_function_exists(posix_fadvise HAVE_POSIX_FADVISE) +check_library_exists(dl dlopen "" NEED_LINK_DL) + +# Configure the build. +include_directories( + ${GENERATED_JAVAH} + main/native/src + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_BINARY_DIR} + ${JNI_INCLUDE_DIRS} + ${SRC}/util +) +configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) + +set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) +ratis_add_dual_library(ratis + main/native/src/exception.c + ${SRC}/io/nativeio/NativeIO.c + ${SRC}/io/nativeio/errno_enum.c + ${SRC}/io/nativeio/file_descriptor.c + ${SRC}/util/NativeCodeLoader.c + ${SRC}/util/NativeCrc32.c + ${SRC}/util/bulk_crc32.c + ${BULK_CRC_ARCH_SOURCE_FIlE} +) +if(NEED_LINK_DL) + set(LIB_DL dl) +endif() + +ratis_target_link_dual_libraries(ratis ${LIB_DL} ${JAVA_JVM_LIBRARY}) +set(LIBRATIS_VERSION "1.0.0") +set_target_properties(ratis PROPERTIES SOVERSION ${LIBRATIS_VERSION}) +ratis_dual_output_directory(ratis target/usr/local/lib) + +# By embedding '$ORIGIN' into the RPATH of libratis.so, dlopen will look in +# the directory containing libratis.so. However, $ORIGIN is not supported by +# all operating systems. +if(${CMAKE_SYSTEM_NAME} MATCHES "Linux|SunOS") + set(RPATH "\$ORIGIN/") + if(EXTRA_LIBRATIS_RPATH) + set(RPATH "${RPATH}:${EXTRA_LIBRATIS_RPATH}/") + endif() + set_target_properties(ratis PROPERTIES INSTALL_RPATH "${RPATH}") +endif() + +# Build the CRC32 test executable. +add_executable(test_bulk_crc32 + ${SRC}/util/bulk_crc32.c + ${BULK_CRC_ARCH_SOURCE_FIlE} + ${TST}/util/test_bulk_crc32.c +) http://git-wip-us.apache.org/repos/asf/incubator-ratis/blob/7e71a2e0/ratis-common/src/config.h.cmake ---------------------------------------------------------------------- diff --git a/ratis-common/src/config.h.cmake b/ratis-common/src/config.h.cmake new file mode 100644 index 0000000..709fc75 --- /dev/null +++ b/ratis-common/src/config.h.cmake @@ -0,0 +1,24 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#ifndef CONFIG_H +#define CONFIG_H + +#cmakedefine HAVE_SYNC_FILE_RANGE +#cmakedefine HAVE_POSIX_FADVISE + +#endif http://git-wip-us.apache.org/repos/asf/incubator-ratis/blob/7e71a2e0/ratis-common/src/main/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/ratis-common/src/main/conf/log4j.properties b/ratis-common/src/main/conf/log4j.properties new file mode 100644 index 0000000..64c1922 --- /dev/null +++ b/ratis-common/src/main/conf/log4j.properties @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +ratis.root.logger=INFO,console +ratis.log.dir=. +ratis.log.file=ratis.log + +# Define the root logger to the system property "ratis.root.logger". +log4j.rootLogger=${ratis.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# Null Appender +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# Rolling File Appender - cap space usage at 5gb. +# +ratis.log.maxfilesize=256MB +ratis.log.maxbackupindex=20 +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${ratis.log.dir}/${ratis.log.file} + +log4j.appender.RFA.MaxFileSize=${ratis.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${ratis.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${ratis.log.dir}/${ratis.log.file} + +# Rollover at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
