http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/pom.xml ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-protocol/pom.xml b/distributedlog-proxy-protocol/pom.xml new file mode 100644 index 0000000..0f6a85f --- /dev/null +++ b/distributedlog-proxy-protocol/pom.xml @@ -0,0 +1,130 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog</artifactId> + <version>0.5.0-incubating-SNAPSHOT</version> + </parent> + <artifactId>distributedlog-proxy-protocol</artifactId> + <name>Apache DistributedLog :: Proxy Protocol</name> + <dependencies> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-protocol</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>libthrift</artifactId> + <version>${libthrift.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>scrooge-core_2.11</artifactId> + <version>${scrooge.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>finagle-core_2.11</artifactId> + <version>${finagle.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>finagle-thrift_2.11</artifactId> + <version>${finagle.version}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.twitter</groupId> + <artifactId>scrooge-maven-plugin</artifactId> + <version>${scrooge-maven-plugin.version}</version> + <configuration> + <language>java</language> + <thriftOpts> + <thriftOpt>--finagle</thriftOpt> + </thriftOpts> + </configuration> + <executions> + <execution> + <id>thrift-sources</id> + <phase>generate-sources</phase> + <goals> + <goal>compile</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>${maven-jar-plugin.version}</version> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>findbugs-maven-plugin</artifactId> + <configuration> + <excludeFilterFile>${basedir}/src/main/resources/findbugsExclude.xml</excludeFilterFile> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <version>${maven-checkstyle-plugin.version}</version> + <dependencies> + <dependency> + <groupId>com.puppycrawl.tools</groupId> + <artifactId>checkstyle</artifactId> + <version>${puppycrawl.checkstyle.version}</version> + </dependency> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-build-tools</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <configuration> + <configLocation>distributedlog/checkstyle.xml</configLocation> + <suppressionsLocation>distributedlog/suppressions.xml</suppressionsLocation> + <consoleOutput>true</consoleOutput> + <failOnViolation>true</failOnViolation> + <includeResources>false</includeResources> + <includeTestSourceDirectory>true</includeTestSourceDirectory> + </configuration> + <executions> + <execution> + <phase>test-compile</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project>
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/ProtocolUtils.java ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/ProtocolUtils.java b/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/ProtocolUtils.java new file mode 100644 index 0000000..1f91968 --- /dev/null +++ b/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/ProtocolUtils.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.distributedlog.protocol.util; + +import static com.google.common.base.Charsets.UTF_8; + +import org.apache.distributedlog.DLSN; +import java.util.zip.CRC32; +import org.apache.distributedlog.exceptions.DLException; +import org.apache.distributedlog.exceptions.OwnershipAcquireFailedException; +import org.apache.distributedlog.thrift.service.ResponseHeader; + +/** + * With CRC embedded in the application, we have to keep track of per api crc. Ideally this + * would be done by thrift. + */ +public class ProtocolUtils { + + // For request payload checksum + private static final ThreadLocal<CRC32> requestCRC = new ThreadLocal<CRC32>() { + @Override + protected CRC32 initialValue() { + return new CRC32(); + } + }; + + /** + * Generate crc32 for WriteOp. + */ + public static Long writeOpCRC32(String stream, byte[] payload) { + CRC32 crc = requestCRC.get(); + try { + crc.update(stream.getBytes(UTF_8)); + crc.update(payload); + return crc.getValue(); + } finally { + crc.reset(); + } + } + + /** + * Generate crc32 for TruncateOp. + */ + public static Long truncateOpCRC32(String stream, DLSN dlsn) { + CRC32 crc = requestCRC.get(); + try { + crc.update(stream.getBytes(UTF_8)); + crc.update(dlsn.serializeBytes()); + return crc.getValue(); + } finally { + crc.reset(); + } + } + + /** + * Generate crc32 for any op which only passes a stream name. + */ + public static Long streamOpCRC32(String stream) { + CRC32 crc = requestCRC.get(); + try { + crc.update(stream.getBytes(UTF_8)); + return crc.getValue(); + } finally { + crc.reset(); + } + } + + public static DLException exception(ResponseHeader response) { + String errMsg; + switch (response.getCode()) { + case FOUND: + if (response.isSetErrMsg()) { + errMsg = response.getErrMsg(); + } else { + errMsg = "Request is redirected to " + response.getLocation(); + } + return new OwnershipAcquireFailedException(errMsg, response.getLocation()); + case SUCCESS: + throw new IllegalArgumentException("Can't instantiate an exception for success response."); + default: + if (response.isSetErrMsg()) { + errMsg = response.getErrMsg(); + } else { + errMsg = response.getCode().name(); + } + return new DLException(response.getCode().getValue(), errMsg); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/package-info.java ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/package-info.java b/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/package-info.java new file mode 100644 index 0000000..311f6c1 --- /dev/null +++ b/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * defines the protocol related utilities. + */ +package org.apache.distributedlog.protocol.util; http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/src/main/resources/findbugsExclude.xml ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-protocol/src/main/resources/findbugsExclude.xml b/distributedlog-proxy-protocol/src/main/resources/findbugsExclude.xml new file mode 100644 index 0000000..05ee085 --- /dev/null +++ b/distributedlog-proxy-protocol/src/main/resources/findbugsExclude.xml @@ -0,0 +1,23 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +//--> +<FindBugsFilter> + <Match> + <!-- generated code, we can't be held responsible for findbugs in it //--> + <Class name="~org\.apache\.distributedlog\.thrift.*" /> + </Match> +</FindBugsFilter> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/src/main/thrift/service.thrift ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-protocol/src/main/thrift/service.thrift b/distributedlog-proxy-protocol/src/main/thrift/service.thrift new file mode 100644 index 0000000..45e1449 --- /dev/null +++ b/distributedlog-proxy-protocol/src/main/thrift/service.thrift @@ -0,0 +1,203 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +namespace java org.apache.distributedlog.thrift.service + +/* Response stats codes */ +enum StatusCode { + /* 2xx: action requested by the client was received, understood, accepted and processed successfully. */ + + /* standard response for successful requests. */ + SUCCESS = 200, + + /* 3xx: client must take additional action to complete the request. */ + + /* client closed. */ + CLIENT_CLOSED = 301, + /* found the stream in a different server, a redirection is required by client. */ + FOUND = 302, + + /* 4xx: client seems to have erred. */ + + /* request is denied for some reason */ + REQUEST_DENIED = 403, + /* request record too large */ + TOO_LARGE_RECORD = 413, + + /* 5xx: server failed to fulfill an apparently valid request. */ + + /* Generic error message, given when no more specific message is suitable. */ + INTERNAL_SERVER_ERROR = 500, + /* Not implemented */ + NOT_IMPLEMENTED = 501, + /* Already Closed Exception */ + ALREADY_CLOSED = 502, + /* Service is currently unavailable (because it is overloaded or down for maintenance). */ + SERVICE_UNAVAILABLE = 503, + /* Locking exception */ + LOCKING_EXCEPTION = 504, + /* ZooKeeper Errors */ + ZOOKEEPER_ERROR = 505, + /* Metadata exception */ + METADATA_EXCEPTION = 506, + /* BK Transmit Error */ + BK_TRANSMIT_ERROR = 507, + /* Flush timeout */ + FLUSH_TIMEOUT = 508, + /* Log empty */ + LOG_EMPTY = 509, + /* Log not found */ + LOG_NOT_FOUND = 510, + /* Truncated Transactions */ + TRUNCATED_TRANSACTION = 511, + /* End of Stream */ + END_OF_STREAM = 512, + /* Transaction Id Out of Order */ + TRANSACTION_OUT_OF_ORDER = 513, + /* Write exception */ + WRITE_EXCEPTION = 514, + /* Stream Unavailable */ + STREAM_UNAVAILABLE = 515, + /* Write cancelled exception */ + WRITE_CANCELLED_EXCEPTION = 516, + /* over-capacity/backpressure */ + OVER_CAPACITY = 517, + /** stream exists but is not ready (recovering etc.). + the difference between NOT_READY and UNAVAILABLE is that UNAVAILABLE + indicates the stream is no longer owned by the proxy and we should + redirect. NOT_READY indicates the stream exist at the proxy but isn't + eady for writes. */ + STREAM_NOT_READY = 518, + /* Region Unavailable */ + REGION_UNAVAILABLE = 519, + /* Invalid Enveloped Entry */ + INVALID_ENVELOPED_ENTRY = 520, + /* Unsupported metadata version */ + UNSUPPORTED_METADATA_VERSION = 521, + /* Log Already Exists */ + LOG_EXISTS = 522, + /* Checksum failed on the request */ + CHECKSUM_FAILED = 523, + /* Overcapacity: too many streams */ + TOO_MANY_STREAMS = 524, + /* Log Segment Not Found */ + LOG_SEGMENT_NOT_FOUND = 525, + /* End of Log Segment */ + END_OF_LOG_SEGMENT = 526, + /* Log Segment Is Truncated */ + LOG_SEGMENT_IS_TRUNCATED = 527, + + /* 6xx: unexpected */ + + UNEXPECTED = 600, + INTERRUPTED = 601, + INVALID_STREAM_NAME = 602, + ILLEGAL_STATE = 603, + + /* 10xx: reader exceptions */ + + RETRYABLE_READ = 1000, + LOG_READ_ERROR = 1001, + /* Read cancelled exception */ + READ_CANCELLED_EXCEPTION = 1002, +} + +/* Response Header */ +struct ResponseHeader { + 1: required StatusCode code; + 2: optional string errMsg; + 3: optional string location; +} + +/* Write Response */ +struct WriteResponse { + 1: required ResponseHeader header; + 2: optional string dlsn; +} + +/* Bulk write response */ +struct BulkWriteResponse { + 1: required ResponseHeader header; + 2: optional list<WriteResponse> writeResponses; +} + +/* Write Context */ +struct WriteContext { + 1: optional set<string> triedHosts; + 2: optional i64 crc32; + 3: optional bool isRecordSet; +} + +/* HeartBeat Options */ +struct HeartbeatOptions { + 1: optional bool sendHeartBeatToReader; +} + +/* Server Status */ +enum ServerStatus { + /* service is writing and accepting new streams */ + WRITE_AND_ACCEPT = 100, + /* service is only writing to old streams, not accepting new streams */ + WRITE_ONLY = 200, + /* service is shutting down, will not write */ + DOWN = 300, +} + +/* Server Info */ +struct ServerInfo { + 1: optional map<string, string> ownerships; + 2: optional ServerStatus serverStatus; +} + +/* Client Info */ +struct ClientInfo { + 1: optional string streamNameRegex; + 2: optional bool getOwnerships; +} + +service DistributedLogService { + + /* Deprecated */ + ServerInfo handshake(); + + ServerInfo handshakeWithClientInfo(ClientInfo clientInfo); + + /* Deprecated */ + WriteResponse heartbeat(string stream, WriteContext ctx); + + WriteResponse heartbeatWithOptions(string stream, WriteContext ctx, HeartbeatOptions options); + + /* Deprecated */ + WriteResponse write(string stream, binary data); + + WriteResponse writeWithContext(string stream, binary data, WriteContext ctx); + + BulkWriteResponse writeBulkWithContext(string stream, list<binary> data, WriteContext ctx); + + WriteResponse truncate(string stream, string dlsn, WriteContext ctx); + + WriteResponse release(string stream, WriteContext ctx); + + WriteResponse create(string stream, WriteContext ctx); + + WriteResponse delete(string stream, WriteContext ctx); + + WriteResponse getOwner(string stream, WriteContext ctx); + + /* Admin Methods */ + void setAcceptNewStream(bool enabled); +} http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/bundle ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/bin/bundle b/distributedlog-proxy-server/bin/bundle new file mode 100755 index 0000000..1b1904e --- /dev/null +++ b/distributedlog-proxy-server/bin/bundle @@ -0,0 +1,28 @@ +#!/bin/bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +set -e + +BASEDIR=$(dirname "$0") +DLOG_ROOT="${BASEDIR}/../.." + +cd "${DLOG_ROOT}" + +bash scripts/bundle service http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/common.sh ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/bin/common.sh b/distributedlog-proxy-server/bin/common.sh new file mode 100755 index 0000000..2b13157 --- /dev/null +++ b/distributedlog-proxy-server/bin/common.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +if [ $(uname) == "Linux" ]; then + # check if net.ipv6.bindv6only is set to 1 + bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) + if [ -n "${bindv6only}" ] && [ "${bindv6only}" -eq "1" ]; then + echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken" + echo "For more info (the following page also applies to DistributedLog): http://wiki.apache.org/hadoop/HadoopIPv6"; + exit 1 + fi +fi + +# See the following page for extensive details on setting +# up the JVM to accept JMX remote management: +# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# by default we allow local JMX connections +if [ -z "${JMXLOCALONLY}" ]; then + JMXLOCALONLY=false +fi + +if [ -z "${JMXDISABLE}" ]; then + echo "JMX enabled by default" >&2 + # for some reason these two options are necessary on jdk6 on Ubuntu + # accord to the docs they are not necessary, but otw jconsole cannot + # do a local attach + JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=${JMXLOCALONLY}" +else + echo "JMX disabled by user request" >&2 +fi + +echo "DLOG_HOME => ${DLOG_HOME}" +DEFAULT_LOG_CONF="${DLOG_HOME}/conf/log4j.properties" + +[ -f "${DLOG_HOME}/conf/dlogenv.sh" ] && source "${DLOG_HOME}/conf/dlogenv.sh" + +# exclude tests jar +RELEASE_JAR=$(ls ${DLOG_HOME}/distributedlog-*.jar 2> /dev/null | grep -v 'tests\|javadoc\|sources' | tail -1) +if [ $? == 0 ]; then + DLOG_JAR="${RELEASE_JAR}" +fi + +# exclude tests jar +BUILT_JAR=$(ls ${DLOG_HOME}/target/distributedlog-*.jar 2> /dev/null | grep -v 'tests\|javadoc\|sources' | tail -1) + +if [ -e "${BUILD_JAR}" ] && [ -e "${DLOG_JAR}" ]; then + echo "\nCouldn't find dlog jar."; + echo "Make sure you've run 'mvn package'\n"; + exit 1; +elif [ -e "${BUILT_JAR}" ]; then + DLOG_JAR="${BUILT_JAR}" +fi + +add_maven_deps_to_classpath() { + MVN="mvn" + if [ -n "${MAVEN_HOME}" ]; then + MVN="${MAVEN_HOME}/bin/mvn" + fi + + # Need to generate classpath from maven pom. This is costly so generate it + # and cache it. Save the file into our target dir so a mvn clean will get + # clean it up and force us create a new one. + f="${PWD}/${DLOG_HOME}/target/cached_classpath.txt" + if [ ! -f "${f}" ]; then + "${MVN}" -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + fi + DLOG_CLASSPATH="${CLASSPATH}":$(cat "${f}") +} + +if [ -d "${DLOG_HOME}/lib" ]; then + for i in ${DLOG_HOME}/lib/*.jar; do + DLOG_CLASSPATH="${DLOG_CLASSPATH}:${i}" + done +else + add_maven_deps_to_classpath +fi + +# if no args specified, exit +if [ $# = 0 ]; then + exit 1 +fi + +if [ -z "${DLOG_LOG_CONF}" ]; then + DLOG_LOG_CONF="${DEFAULT_LOG_CONF}" +fi + +DLOG_CLASSPATH="${DLOG_JAR}:${DLOG_CLASSPATH}:${DLOG_EXTRA_CLASSPATH}" +if [ -n "${DLOG_LOG_CONF}" ]; then + DLOG_CLASSPATH="$(dirname ${DLOG_LOG_CONF}):${DLOG_CLASSPATH}" + OPTS="${OPTS} -Dlog4j.configuration=$(basename ${DLOG_LOG_CONF})" +fi +OPTS="-cp ${DLOG_CLASSPATH} ${OPTS} ${DLOG_EXTRA_OPTS}" + +OPTS="${OPTS} ${DLOG_EXTRA_OPTS}" + +# Disable ipv6 as it can cause issues +OPTS="${OPTS} -Djava.net.preferIPv4Stack=true" + +# log directory & file +DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"} +DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"} +DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dlog.log"} + +#Configure log configuration system properties +OPTS="$OPTS -Ddlog.root.logger=${DLOG_ROOT_LOGGER}" +OPTS="$OPTS -Ddlog.log.dir=${DLOG_LOG_DIR}" +OPTS="$OPTS -Ddlog.log.file=${DLOG_LOG_FILE}" http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/dlog ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/bin/dlog b/distributedlog-proxy-server/bin/dlog new file mode 100755 index 0000000..99cad40 --- /dev/null +++ b/distributedlog-proxy-server/bin/dlog @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +set -e + +BASEDIR=$(dirname "$0") +DLOG_HOME="${BASEDIR}/.." + +usage() { + cat <<EOF +Usage: runner <command> +where command is one of: + local Run distributedlog sandbox + example Run distributedlog example + tool Run distributedlog tool + proxy_tool Run distributedlog proxy tool to interact with proxies + balancer Run distributedlog balancer + admin Run distributedlog admin tool + zkshell Run zookeeper shell + bkshell Run bookkeeper shell + help This help message + +or command is the full name of a class with a defined main() method. + +Environment variables: + DLOG_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) + DLOG_EXTRA_OPTS Extra options to be passed to the jvm + DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath + +These variable can also be set in conf/dlogenv.sh +EOF +} + +source "${DLOG_HOME}"/bin/common.sh + +# get arguments +COMMAND=$1 +shift + +case "${COMMAND}" in + local) + exec java $OPTS $JMX_ARGS org.apache.distributedlog.LocalDLMEmulator $@ + ;; + tool) + exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.tools.DistributedLogTool $@ + ;; + proxy_tool) + exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.service.tools.ProxyTool $@ + ;; + balancer) + exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.service.balancer.BalancerTool $@ + ;; + admin) + exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.admin.DistributedLogAdmin $@ + ;; + zkshell) + exec java $OPTS org.apache.zookeeper.ZooKeeperMain -server $@ + ;; + bkshell) + ENTRY_FORMATTER_ARG="-DentryFormatterClass=${ENTRY_FORMATTER_CLASS:-org.apache.bookkeeper.util.StringEntryFormatter}" + exec java $OPTS $ENTRY_FORMATTER_ARG org.apache.bookkeeper.bookie.BookieShell -conf $BOOKIE_CONF $@ + ;; + help) + usage + ;; + *) + exec java $OPTS $COMMAND $@ + ;; +esac + + http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/dlog-daemon.sh ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/bin/dlog-daemon.sh b/distributedlog-proxy-server/bin/dlog-daemon.sh new file mode 100755 index 0000000..fdd87df --- /dev/null +++ b/distributedlog-proxy-server/bin/dlog-daemon.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +usage() { + cat <<EOF +Usage: dlog-daemon.sh (start|stop) <service> <args...> +where service is one of: + zookeeper Run the zookeeper server + bookie Run the bookie server + bookie-rereplicator Run the bookie rereplicator + writeproxy Run the write proxy server + writeproxy-monitor Run the write proxy monitor + +where argument is one of: + -force (accepted only with stop service): Decides whether to stop the process forcefully if not stopped by normal shutdown +EOF +} + +BINDIR=$(dirname "$0") +DL_HOME=$(cd $BINDIR/..;pwd) + +if [ -f "${DL_HOME}/conf/dlogenv.sh" ]; then + . "${DL_HOME}/conf/dlogenv.sh" +fi + +SERVICE_PORT=${SERVICE_PORT:-"0"} +SERVICE_ARGS="" + +# DLOG logging configuration +DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DL_HOME/logs"} +DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-'INFO,R'} + +# Process Control Parameters +DLOG_STOP_TIMEOUT=${DLOG_STOP_TIMEOUT:-30} +DLOG_PID_DIR=${DLOG_PID_DIR:-$DL_HOME/pids} + +if [ $# -lt 2 ]; then + echo "Error: not enough arguments provided." + usage + exit 1 +fi + +command=$1 +shift + +service=$1 +shift + +service_class=$service +case $service in + (zookeeper) + service_class="org.apache.zookeeper.server.quorum.QuorumPeerMain" + DLOG_ROOT_LOGGER=${ZK_ROOT_LOGGER:-'INFO,R'} + ;; + (bookie) + service_class="org.apache.bookkeeper.proto.BookieServer" + DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'} + ;; + (bookie-rereplicator) + service_class="org.apache.bookkeeper.replication.AutoRecoveryMain" + DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'} + ;; + (writeproxy) + service_class="org.apache.distributedlog.service.DistributedLogServerApp" + DLOG_ROOT_LOGGER=${WP_ROOT_LOGGER:-'INFO,R'} + WP_CONF_FILE=${WP_CONF_FILE:-"$DL_HOME/conf/write_proxy.conf"} + WP_SERVICE_PORT=${WP_SERVICE_PORT:-'4181'} + WP_STATS_PORT=${WP_STATS_PORT:-'9000'} + WP_STATS_PROVIDER=${WP_STATS_PROVIDER:-'org.apache.bookkeeper.stats.CodahaleMetricsServletProvider'} + WP_SHARD_ID=${WP_SHARD_ID:-'0'} + WP_NAMESPACE=${WP_NAMESPACE:-'distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace'} + SERVICE_PORT=${WP_SERVICE_PORT} + SERVICE_ARGS="--conf ${WP_CONF_FILE} --uri ${WP_NAMESPACE} --shard-id ${WP_SHARD_ID} --port ${WP_SERVICE_PORT} --stats-port ${WP_STATS_PORT} --stats-provider ${WP_STATS_PROVIDER} --announce --thriftmux" + DLOG_EXTRA_OPTS="${DLOG_EXTRA_OPTS} -DcodahaleStatsHttpPort=${WP_STATS_PORT} -Dserver_port=${WP_SERVICE_PORT} -Dserver_shard=${WP_SHARD_ID}" + ;; + (writeproxy-monitor) + ;; + (*) + echo "Error: unknown service name $service" + usage + exit 1 + ;; +esac + +echo "doing $command $service ..." + +export DLOG_LOG_DIR=$DLOG_LOG_DIR +export DLOG_ROOT_LOGGER=$DLOG_ROOT_LOGGER +export DLOG_LOG_FILE=dlog-$service-$HOSTNAME-$SERVICE_PORT.log +export DLOG_EXTRA_OPTS=$DLOG_EXTRA_OPTS + +pid=$DLOG_PID_DIR/dlog-$service-$SERVICE_PORT.pid +out=$DLOG_LOG_DIR/dlog-$service-$HOSTNAME-$SERVICE_PORT.out +logfile=$DLOG_LOG_DIR/$DLOG_LOG_FILE + +rotate_out_log () { + log=$1; + num=5; + if [ -n "$2" ]; then + num=$2 + fi + if [ -f "$log" ]; then # rotate logs + while [ $num -gt 1 ]; do + prev=`expr $num - 1` + [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" + num=$prev + done + mv "$log" "$log.$num"; + fi +} + +mkdir -p "$DLOG_LOG_DIR" +mkdir -p "$DLOG_PID_DIR" + +case $command in + (start) + if [ -f $pid ]; then + if kill -0 $(cat $pid) > /dev/null 2>&1; then + echo $service running as process $(cat $pid). Stop it first. + exit 1 + fi + fi + + rotate_out_log $out + echo starting $service, logging to $logfile + dlog=$DL_HOME/bin/dlog + nohup $dlog $service_class ${SERVICE_ARGS} "$@" > "$out" 2>&1 < /dev/null & + echo $! > $pid + sleep 1; head $out + sleep 2; + if ! ps -p $! > /dev/null ; then + exit 1 + fi + ;; + + (stop) + if [ -f $pid ]; then + TARGET_PID=$(cat $pid) + if kill -0 $TARGET_PID > /dev/null 2>&1; then + echo stopping $service + kill $TARGET_PID + + count=0 + location=$DLOG_LOG_DIR + while ps -p $TARGET_PID > /dev/null; do + echo "Shutdown is in progress... Please wait..." + sleep 1 + count=$(expr $count + 1) + if [ "$count" = "$DLOG_STOP_TIMEOUT" ]; then + break + fi + done + if [ "$count" != "$DLOG_STOP_TIMEOUT" ]; then + echo "Shutdown completed." + fi + if kill -0 $TARGET_PID > /dev/null 2>&1; then + fileName=$location/$service.out + $JAVA_HOME/bin/jstack $TARGET_PID > $fileName + echo Thread dumps are taken for analysis at $fileName + if [ "$1" == "-force" ]; then + echo forcefully stopping $service + kill -9 $TARGET_PID >/dev/null 2>&1 + echo Successfully stopped the process + else + echo "WARNNING : $service is not stopped completely." + exit 1 + fi + fi + else + echo no $service to stop + fi + rm $pid + else + echo no $service to stop + fi + ;; + + (*) + usage + echo $supportedargs + exit 1 + ;; +esac http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/dlog-env.sh ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/bin/dlog-env.sh b/distributedlog-proxy-server/bin/dlog-env.sh new file mode 100644 index 0000000..50a1960 --- /dev/null +++ b/distributedlog-proxy-server/bin/dlog-env.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# we need the DLog URI to be set +if [[ -z "${DISTRIBUTEDLOG_URI}" ]]; then + echo "Environment variable DISTRIBUTEDLOG_URI is not set." + exit 1 +fi + +# add the jars from current dir to the class path (should be distributedlog-service) +for i in ./*.jar; do + CLASSPATH="$i:${CLASSPATH}" +done + +# add all the jar from lib/ to the class path +for i in ./lib/*.jar; do + CLASSPATH="$i:${CLASSPATH}" +done http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/dlog-start.sh ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/bin/dlog-start.sh b/distributedlog-proxy-server/bin/dlog-start.sh new file mode 100755 index 0000000..c9c2192 --- /dev/null +++ b/distributedlog-proxy-server/bin/dlog-start.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +DLOGBIN="${BASH_SOURCE-$0}" +DLOGBIN="$(dirname "${DLOGBIN}")" +DLOGBINDIR="$(cd "${DLOGBIN}"; pwd)" + +. "${DLOGBINDIR}"/dlog-env.sh + +java -cp "${CLASSPATH}" \ + -Dlog4j.configuration=conf/log4j.properties \ + -DstatsHttpPort=9000 -DstatsExport=true \ + -Dserver_shard=0 \ + org.apache.distributedlog.service.DistributedLogServerApp \ + --port 8000 \ + --uri "${DISTRIBUTEDLOG_URI}" \ + --conf conf/distributedlog.conf http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/zk-server-start.sh ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/bin/zk-server-start.sh b/distributedlog-proxy-server/bin/zk-server-start.sh new file mode 100644 index 0000000..f222756 --- /dev/null +++ b/distributedlog-proxy-server/bin/zk-server-start.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 [-daemon] zookeeper.properties" + exit 1 +fi +BASE_DIR=$(dirname $0) + +if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$BASE_DIR/../config/log4j.properties" +fi + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx512M -Xms512M" +fi + +EXTRA_ARGS="-name zookeeper -loggc" + +COMMAND=$1 +case $COMMAND in + -daemon) + EXTRA_ARGS="-daemon "$EXTRA_ARGS + shift + ;; + *) + ;; +esac + +exec $BASE_DIR/kafka-run-class.sh $EXTRA_ARGS org.apache.zookeeper.server.quorum.QuorumPeerMain "$@" http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/bookie.conf.template ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/conf/bookie.conf.template b/distributedlog-proxy-server/conf/bookie.conf.template new file mode 100644 index 0000000..5ca89d0 --- /dev/null +++ b/distributedlog-proxy-server/conf/bookie.conf.template @@ -0,0 +1,183 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +## Bookie settings + +# Port that bookie server listen on +bookiePort=3181 + +# TODO: change the journal directory +# Directory Bookkeeper outputs its write ahead log +journalDirectory=/tmp/data/bk/journal + +# TODO: change the ledgers directory +# Directory Bookkeeper outputs ledger snapshots +ledgerDirectories=/tmp/data/bk/ledgers + +# TODO: change the index directory +# Directory in which index files will be stored. +indexDirectories=/tmp/data/bk/ledgers + +# Ledger Manager Class +# What kind of ledger manager is used to manage how ledgers are stored, managed +# and garbage collected. Try to read 'BookKeeper Internals' for detail info. +ledgerManagerType=hierarchical + +# Root zookeeper path to store ledger metadata +# This parameter is used by zookeeper-based ledger manager as a root znode to +# store all ledgers. +zkLedgersRootPath=/messaging/bookkeeper/ledgers + +# Max file size of entry logger, in bytes +# A new entry log file will be created when the old one reaches the file size limitation +logSizeLimit=1073741823 + +# Max file size of journal file, in mega bytes +# A new journal file will be created when the old one reaches the file size limitation +# +journalMaxSizeMB=2048 + +# Max number of old journal file to kept +# Keep a number of old journal files would help data recovery in specia case +# +journalMaxBackups=5 + +# How long the interval to trigger next garbage collection, in milliseconds +# Since garbage collection is running in background, too frequent gc +# will heart performance. It is better to give a higher number of gc +# interval if there is enough disk capacity. +# gc per 1 hour (aligning with most DL rolling interval) +gcInitialWaitTime=600000 +gcWaitTime=3600000 +# do minor compaction per 2 hours +minorCompactionInterval=7200 +minorCompactionThreshold=0.2 +# disable major compaction +majorCompactionInterval=0 +# reduce major compaction threshold to a low value to prevent bad force compaction behavior +majorCompactionThreshold=0.3 +# Compaction Rate & Max Outstanding +compactionRate=10737418 +compactionMaxOutstandingRequests=10737418 + +# How long the interval to flush ledger index pages to disk, in milliseconds +# Flushing index files will introduce much random disk I/O. +# If separating journal dir and ledger dirs each on different devices, +# flushing would not affect performance. But if putting journal dir +# and ledger dirs on same device, performance degrade significantly +# on too frequent flushing. You can consider increment flush interval +# to get better performance, but you need to pay more time on bookie +# server restart after failure. +# +flushInterval=1000 + +# Interval to watch whether bookie is dead or not, in milliseconds +# +# bookieDeathWatchInterval=1000 + +## zookeeper client settings + +# A list of one of more servers on which zookeeper is running. +# The server list can be comma separated values, for example: +# zkServers=zk1:2181,zk2:2181,zk3:2181 +zkServers=localhost:2181 + +# ZooKeeper client session timeout in milliseconds +# Bookie server will exit if it received SESSION_EXPIRED because it +# was partitioned off from ZooKeeper for more than the session timeout +# JVM garbage collection, disk I/O will cause SESSION_EXPIRED. +# Increment this value could help avoiding this issue +zkTimeout=30000 + +## NIO Server settings + +# This settings is used to enabled/disabled Nagle's algorithm, which is a means of +# improving the efficiency of TCP/IP networks by reducing the number of packets +# that need to be sent over the network. +# If you are sending many small messages, such that more than one can fit in +# a single IP packet, setting server.tcpnodelay to false to enable Nagle algorithm +# can provide better performance. +# Default value is true. +# +serverTcpNoDelay=true + +## ledger cache settings + +# Max number of ledger index files could be opened in bookie server +# If number of ledger index files reaches this limitation, bookie +# server started to swap some ledgers from memory to disk. +# Too frequent swap will affect performance. You can tune this number +# to gain performance according your requirements. +openFileLimit=20000 + +# Size of a index page in ledger cache, in bytes +# A larger index page can improve performance writing page to disk, +# which is efficent when you have small number of ledgers and these +# ledgers have similar number of entries. +# If you have large number of ledgers and each ledger has fewer entries, +# smaller index page would improve memory usage. +pageSize=8192 + +# How many index pages provided in ledger cache +# If number of index pages reaches this limitation, bookie server +# starts to swap some ledgers from memory to disk. You can increment +# this value when you found swap became more frequent. But make sure +# pageLimit*pageSize should not more than JVM max memory limitation, +# otherwise you would got OutOfMemoryException. +# In general, incrementing pageLimit, using smaller index page would +# gain bettern performance in lager number of ledgers with fewer entries case +# If pageLimit is -1, bookie server will use 1/3 of JVM memory to compute +# the limitation of number of index pages. +pageLimit=131072 + +#If all ledger directories configured are full, then support only read requests for clients. +#If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted +#to read-only mode and serve only read requests. Otherwise the bookie will be shutdown. +readOnlyModeEnabled=true + +# Bookie Journal Settings +writeBufferSizeBytes=262144 +journalFlushWhenQueueEmpty=false +journalRemoveFromPageCache=true +journalAdaptiveGroupWrites=true +journalMaxGroupWaitMSec=4 +journalBufferedEntriesThreshold=180 +journalBufferedWritesThreshold=131072 +journalMaxGroupedEntriesToCommit=200 +journalPreAllocSizeMB=4 + +# Sorted Ledger Storage Settings +sortedLedgerStorageEnabled=true +skipListSizeLimit=67108864 +skipListArenaChunkSize=2097152 +skipListArenaMaxAllocSize=131072 +fileInfoCacheInitialCapacity=10000 +fileInfoMaxIdleTime=3600 + +# Bookie Threads Settings (NOTE: change this to align the cpu cores) +numAddWorkerThreads=4 +numJournalCallbackThreads=4 +numReadWorkerThreads=4 +numLongPollWorkerThreads=4 + +# stats +statsProviderClass=org.apache.bookkeeper.stats.CodahaleMetricsServletProvider +# Exporting codahale stats +codahaleStatsHttpPort=9001 +useHostNameAsBookieID=true +allowLoopback=true http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/distributedlog.conf ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/conf/distributedlog.conf b/distributedlog-proxy-server/conf/distributedlog.conf new file mode 100644 index 0000000..dac71ac --- /dev/null +++ b/distributedlog-proxy-server/conf/distributedlog.conf @@ -0,0 +1,125 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +######################## +# ZooKeeper Client Settings +######################## + +# zookeeper settings +zkSessionTimeoutSeconds=30 +zkNumRetries=0 +zkRetryStartBackoffMillis=100 +zkRetryMaxBackoffMillis=200 +# bkc zookeeper settings +bkcZKSessionTimeoutSeconds=60 +bkcZKNumRetries=20 +bkcZKRetryStartBackoffMillis=100 +bkcZKRetryMaxBackoffMillis=200 + +######################## +# BookKeeper Client Settings +######################## + +# bookkeeper client timeouts +bkcWriteTimeoutSeconds=10 +bkcReadTimeoutSeconds=1 +bkcNumWorkerThreads=16 +# bkcNumIOThreads=16 +bkc.numChannelsPerBookie=1 +bkc.enableTaskExecutionStats=true +bkc.connectTimeoutMillis=1000 +bkc.enablePerHostStats=true + +######################## +# DL Settings +######################## + +# lock timeout +lockTimeoutSeconds=0 +# dl worker threads +numWorkerThreads=16 + +### Recovery Related Settings + +# recover log segments in background +recoverLogSegmentsInBackground=true +# disable max id in proxy +maxIdSanityCheck=true +# use allocator pool for proxy +enableLedgerAllocatorPool=false +# ledger allocator pool size +ledgerAllocatorPoolCoreSize=20 +# check stream exists or not +createStreamIfNotExists=true +# encode dc id in version +encodeDCIDInVersion=true +# logSegmentNameVersion +logSegmentNameVersion=1 + +### Write Performance Related Settings + +# ensemble size +ensemble-size=3 +write-quorum-size=3 +ack-quorum-size=2 +bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy +bkc.delayEnsembleChange=true + +# sync settings +# buffer size is large because when we rewrite we perform a very large write to persist +# all queue state at once (up to max queue memory size, ex. 16MB). the write will be +# throttled if it takes too long, which can hurt performance, so important to optimize +# for this case. +output-buffer-size=512000 +enableImmediateFlush=false +periodicFlushFrequencyMilliSeconds=6 +logFlushTimeoutSeconds=120 + +### Ledger Rolling Related Settings + +# retention policy +retention-size=0 +# rolling ledgers (disable time rolling/enable size rolling) +rolling-interval=0 + +# max logsegment bytes=2GB +# much larger than max journal size, effectively never roll and let drpc do it +maxLogSegmentBytes=2147483648 + +# rolling concurrency +logSegmentRollingConcurrency=1 +# disable sanityCheckDelete +sanityCheckDelete=false +ledgerAllocatorPoolName=drpc-alloc-pool + +### Readahead settings + +enableReadAhead=true +ReadAheadBatchSize=10 +ReadAheadMaxEntries=100 +ReadAheadWaitTime=10 + +### Rate limit + +rpsSoftWriteLimit=1 +rpsHardWriteLimit=5 +rpsHardServiceLimit=15 + +### Config + +dynamicConfigReloadIntervalSec=5 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/dlogenv.sh ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/conf/dlogenv.sh b/distributedlog-proxy-server/conf/dlogenv.sh new file mode 100644 index 0000000..345e60f --- /dev/null +++ b/distributedlog-proxy-server/conf/dlogenv.sh @@ -0,0 +1,75 @@ +#!/bin/sh +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +################## +# General +################## + +# Log4j configuration file +# DLOG_LOG_CONF= + +# Extra options to be passed to the jvm +# DLOG_EXTRA_OPTS= + +# Add extra paths to the dlog classpath +# DLOG_EXTRA_CLASSPATH= + +# Configure the root logger +# DLOG_ROOT_LOGGER= + +# Configure the log dir +# DLOG_LOG_DIR= + +# Configure the log file +# DLOG_LOG_FILE= + +################# +# ZooKeeper +################# + +# Configure zookeeper root logger +# ZK_ROOT_LOGGER= + +################# +# Bookie +################# + +# Configure bookie root logger +# BK_ROOT_LOGGER= + +################# +# Write Proxy +################# + +# Configure write proxy root logger +# WP_ROOT_LOGGER= + +# write proxy configuration file +# WP_CONF_FILE=${DL_HOME}/conf/write_proxy.conf + +# port and stats port +# WP_SERVICE_PORT=4181 +# WP_STATS_PORT=9000 + +# shard id +# WP_SHARD_ID=0 + +# write proxy namespace +# WP_NAMESPACE=distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/conf/log4j.properties b/distributedlog-proxy-server/conf/log4j.properties new file mode 100644 index 0000000..73b4cfa --- /dev/null +++ b/distributedlog-proxy-server/conf/log4j.properties @@ -0,0 +1,60 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# +# DistributedLog Logging Configuration +# + +# Default values +dlog.root.logger=INFO, R +dlog.log.dir=logs +dlog.log.file=dlog.log + +log4j.rootLogger=${dlog.root.logger} +log4j.logger.org.apache.zookeeper=INFO +log4j.logger.org.apache.bookkeeper=INFO + +# redirect executor output to executors.log since slow op warnings can be quite verbose +log4j.logger.com.twitter.distributedlog.util.MonitoredFuturePool=INFO, Executors +log4j.logger.com.twitter.distributedlog.util.MonitoredScheduledThreadPoolExecutor=INFO, Executors +log4j.logger.org.apache.bookkeeper.util.SafeRunnable=INFO, Executors +log4j.additivity.com.twitter.distributedlog.util.MonitoredFuturePool=false +log4j.additivity.com.twitter.distributedlog.util.MonitoredScheduledThreadPoolExecutor=false +log4j.additivity.org.apache.bookkeeper.util.SafeRunnable=false + +log4j.appender.Executors=org.apache.log4j.RollingFileAppender +log4j.appender.Executors.Threshold=INFO +log4j.appender.Executors.File=${dlog.log.dir}/executors.log +log4j.appender.Executors.MaxFileSize=20MB +log4j.appender.Executors.MaxBackupIndex=5 +log4j.appender.Executors.layout=org.apache.log4j.PatternLayout +log4j.appender.Executors.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n + +log4j.appender.R=org.apache.log4j.RollingFileAppender +log4j.appender.R.Threshold=INFO +log4j.appender.R.File=${dlog.log.dir}/${dlog.log.file} +log4j.appender.R.MaxFileSize=20MB +log4j.appender.R.MaxBackupIndex=50 +log4j.appender.R.layout=org.apache.log4j.PatternLayout +log4j.appender.R.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.Threshold=INFO +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/write_proxy.conf ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/conf/write_proxy.conf b/distributedlog-proxy-server/conf/write_proxy.conf new file mode 100644 index 0000000..7f5351a --- /dev/null +++ b/distributedlog-proxy-server/conf/write_proxy.conf @@ -0,0 +1,143 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +######################## +# ZooKeeper Client Settings +######################## + +# zookeeper settings +zkSessionTimeoutSeconds=1 +zkNumRetries=0 +zkRetryStartBackoffMillis=100 +zkRetryMaxBackoffMillis=200 +# bkc zookeeper settings +bkcZKSessionTimeoutSeconds=60 +bkcZKNumRetries=20 +bkcZKRetryStartBackoffMillis=100 +bkcZKRetryMaxBackoffMillis=200 + +######################## +# BookKeeper Client Settings +######################## + +# bookkeeper client timeouts +bkcWriteTimeoutSeconds=2 +bkcReadTimeoutSeconds=2 +bkcNumWorkerThreads=32 +bkc.numChannelsPerBookie=1 +bkc.enableTaskExecutionStats=true +bkc.connectTimeoutMillis=200 +bkc.enableParallelRecoveryRead=true +bkc.recoveryReadBatchSize=5 +bkc.enablePerHostStats=true + +######################## +# DL Settings +######################## + +# Metadata Settings + +# ledger metadata version that supports sequence id +ledger-metadata-layout=5 + +# lock timeout +lockTimeoutSeconds=0 +# dl worker threads +numWorkerThreads=32 + +### Recovery Related Settings + +# recover log segments in background +recoverLogSegmentsInBackground=false +# disable max id in proxy +maxIdSanityCheck=false +# use allocator pool for proxy +enableLedgerAllocatorPool=true +# ledger allocator pool path +ledgerAllocatorPoolPath=.write_proxy_allocation_pool +# ledger allocator pool size +ledgerAllocatorPoolCoreSize=40 +# check stream exists or not +createStreamIfNotExists=true +# encode dc id in version +encodeDCIDInVersion=true +# logSegmentNameVersion +logSegmentNameVersion=1 + +### Write Performance Related Settings + +# ensemble size +ensemble-size=3 +write-quorum-size=3 +ack-quorum-size=2 +bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy +bkc.delayEnsembleChange=true +bkc.writeRequestToChannelAsync=true + +# enable immediate flush +enableImmediateFlush=true +# 0k output buffer +output-buffer-size=0 +# disable periodical flush +periodicFlushFrequencyMilliSeconds=0 +enableTaskExecutionStats=true +taskExecutionWarnTimeMicros=100000 + +### Ledger Rolling Related Settings + +# retention policy +retention-size=4 +# rolling ledgers (enable time rolling): 120 minutes = 2 hours +rolling-interval=120 +# max logsegment bytes : 2GB +maxLogSegmentBytes=2147483648 +# rolling concurrency +logSegmentRollingConcurrency=1 +# disable sanityCheckDelete +sanityCheckDelete=false +# compression codec +compressionType=lz4 + +### Per Stream Stats +enablePerStreamStat=true + +######################## +# DL Settings +######################## + +# proxy server settings +server_mode=DURABLE +serviceTimeoutMs=60000 +streamProbationTimeoutMs=120000 +server_threads=16 +server_dlsn_version=1 +server_enable_perstream_stat=true +server_graceful_shutdown_period_ms=20000 + +# write limits +perWriterOutstandingWriteLimit=-1 +globalOutstandingWriteLimit=-1 +outstandingWriteLimitDarkmode=false + +# bytes per second limit applied at the host level (50MBps on 1Gib machines) +bpsHardServiceLimit=52428800 +# bytes per second limit after which no new streams may be acquired (65MBps on 1Gib machines) +bpsStreamAcquireServiceLimit=47185920 + +# limit the maximum number of streams +maxAcquiredPartitionsPerProxy=-1 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/zookeeper.conf.template ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/conf/zookeeper.conf.template b/distributedlog-proxy-server/conf/zookeeper.conf.template new file mode 100644 index 0000000..3c0546e --- /dev/null +++ b/distributedlog-proxy-server/conf/zookeeper.conf.template @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The number of milliseconds of each tick +tickTime=2000 + +# the port at which the clients will connect +clientPort=2181 + +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 + +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=30 + +# the directory where the snapshot is stored. +dataDir=/tmp/data/zookeeper + +# where txlog are written +dataLogDir=/tmp/data/zookeeper/txlog + +# the port at which the admin will listen +adminPort=9990 +zookeeper.admin.enableServer=true + +# limit on queued clients - default: 1000 +globalOutstandingLimit=1000 + +# number of transactions before snapshots are taken - default: 100000 +snapCount=100000 + +# max # of clients - 0==unlimited +maxClientCnxns=25 + +# Election implementation to use. A value of "0" corresponds to the original +# UDP-based version, "1" corresponds to the non-authenticated UDP-based +# version of fast leader election, "2" corresponds to the authenticated +# UDP-based version of fast leader election, and "3" corresponds to TCP-based +# version of fast leader election. Currently, only 0 and 3 are supported, +# 3 being the default +electionAlg=3 + +# Leader accepts client connections. Default value is "yes". The leader +# machine coordinates updates. For higher update throughput at thes slight +# expense of read throughput the leader can be configured to not accept +# clients and focus on coordination. +leaderServes=yes + +# Skips ACL checks. This results in a boost in throughput, but opens up full +# access to the data tree to everyone. +skipACL=no + +# Purge txn logs every hour. Before 3.4.x this was done with an external cron +# job, now we can do it internally. +autopurge.purgeInterval=1 + +# Prior to version 3.4 ZooKeeper has always used NIO directly, however in +# versions 3.4 and later Netty is supported as an option to NIO (replaces). +# serverCnxnFactory=org.apache.zookeeper.server.NIOServerCnxnFactory + +standaloneEnabled=false +# ZooKeeper Dynamic Reconfiguration +# See: https://zookeeper.apache.org/doc/trunk/zookeeperReconfig.html +# +# standaloneEnabled=false +# dynamicConfigFile=/path/to/zoo.cfg.dynamic +# +server.1=127.0.0.1:2710:3710:participant;0.0.0.0:2181 http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/pom.xml ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/pom.xml b/distributedlog-proxy-server/pom.xml new file mode 100644 index 0000000..d7cbd56 --- /dev/null +++ b/distributedlog-proxy-server/pom.xml @@ -0,0 +1,275 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog</artifactId> + <version>0.5.0-incubating-SNAPSHOT</version> + </parent> + <artifactId>distributedlog-proxy-server</artifactId> + <name>Apache DistributedLog :: Proxy Service</name> + <dependencies> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-proxy-client</artifactId> + <version>${project.parent.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-proxy-protocol</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-core</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>finagle-ostrich4_2.11</artifactId> + <version>${finagle.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>finagle-thriftmux_2.11</artifactId> + <version>${finagle.version}</version> + </dependency> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-core</artifactId> + <version>${project.parent.version}</version> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junit.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>${mockito.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.bookkeeper.stats</groupId> + <artifactId>codahale-metrics-provider</artifactId> + <version>${bookkeeper.version}</version> + </dependency> + <dependency> + <groupId>com.codahale.metrics</groupId> + <artifactId>metrics-servlets</artifactId> + <version>${codahale.metrics.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>libthrift</artifactId> + <version>${libthrift.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>scrooge-core_2.11</artifactId> + <version>${scrooge.version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <version>${slf4j.version}</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <version>${curator.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-protocol</artifactId> + <version>${project.parent.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.twitter</groupId> + <artifactId>scrooge-maven-plugin</artifactId> + <version>${scrooge-maven-plugin.version}</version> + <configuration> + <language>java</language> + </configuration> + <executions> + <execution> + <id>thrift-sources</id> + <phase>generate-sources</phase> + <goals> + <goal>compile</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <version>${maven-assembly-plugin.version}</version> + <configuration> + <descriptors> + <descriptor>../src/assemble/bin.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>${maven-surefire-plugin.version}</version> + <configuration> + <redirectTestOutputToFile>true</redirectTestOutputToFile> + <argLine>-Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G</argLine> + <forkMode>always</forkMode> + <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds> + <properties> + <property> + <name>listener</name> + <value>org.apache.distributedlog.TimedOutTestsListener</value> + </property> + </properties> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>findbugs-maven-plugin</artifactId> + <configuration> + <excludeFilterFile>${basedir}/src/main/resources/findbugsExclude.xml</excludeFilterFile> + </configuration> + </plugin> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>${basedir}/lib</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <version>${maven-checkstyle-plugin.version}</version> + <dependencies> + <dependency> + <groupId>com.puppycrawl.tools</groupId> + <artifactId>checkstyle</artifactId> + <version>${puppycrawl.checkstyle.version}</version> + </dependency> + <dependency> + <groupId>org.apache.distributedlog</groupId> + <artifactId>distributedlog-build-tools</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <configuration> + <configLocation>distributedlog/checkstyle.xml</configLocation> + <suppressionsLocation>distributedlog/suppressions.xml</suppressionsLocation> + <consoleOutput>true</consoleOutput> + <failOnViolation>true</failOnViolation> + <includeResources>false</includeResources> + <includeTestSourceDirectory>true</includeTestSourceDirectory> + </configuration> + <executions> + <execution> + <phase>test-compile</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + <profiles> + <profile> + <id>twitter-ostrich-provider</id> + <dependencies> + <dependency> + <groupId>org.apache.bookkeeper.stats</groupId> + <artifactId>twitter-ostrich-provider</artifactId> + <version>${bookkeeper.version}</version> + <exclusions> + <exclusion> + <groupId>com.twitter</groupId> + <artifactId>ostrich_2.10</artifactId> + </exclusion> + <exclusion> + <groupId>com.twitter</groupId> + <artifactId>ostrich_2.9.2</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsServletProvider.java ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsServletProvider.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsServletProvider.java new file mode 100644 index 0000000..8db3e90 --- /dev/null +++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsServletProvider.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bookkeeper.stats; + +import com.codahale.metrics.health.HealthCheckRegistry; +import org.apache.commons.configuration.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Extend the codahale metrics provider to run servlets. + */ +public class CodahaleMetricsServletProvider extends CodahaleMetricsProvider { + + private static final Logger logger = LoggerFactory.getLogger(CodahaleMetricsServletProvider.class); + + ServletReporter servletReporter = null; + private final HealthCheckRegistry healthCheckRegistry = new HealthCheckRegistry(); + + @Override + public void start(Configuration conf) { + super.start(conf); + Integer httpPort = conf.getInteger("codahaleStatsHttpPort", null); + if (null != httpPort) { + servletReporter = new ServletReporter( + getMetrics(), + healthCheckRegistry, + httpPort); + try { + servletReporter.start(); + } catch (Exception e) { + logger.warn("Encountered error on starting the codahale metrics servlet", e); + } + } + } + + @Override + public void stop() { + if (null != servletReporter) { + try { + servletReporter.stop(); + } catch (Exception e) { + logger.error("Encountered error on stopping the codahale metrics servlet", e); + } + } + super.stop(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/HealthCheckServletContextListener.java ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/HealthCheckServletContextListener.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/HealthCheckServletContextListener.java new file mode 100644 index 0000000..348787a --- /dev/null +++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/HealthCheckServletContextListener.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bookkeeper.stats; + +import com.codahale.metrics.health.HealthCheckRegistry; +import com.codahale.metrics.servlets.HealthCheckServlet; + +/** + * Health Check Servlet Listener. + */ +public class HealthCheckServletContextListener extends HealthCheckServlet.ContextListener { + + private final HealthCheckRegistry healthCheckRegistry; + + public HealthCheckServletContextListener(HealthCheckRegistry healthCheckRegistry) { + this.healthCheckRegistry = healthCheckRegistry; + } + + @Override + protected HealthCheckRegistry getHealthCheckRegistry() { + return healthCheckRegistry; + } +} http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/MetricsServletContextListener.java ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/MetricsServletContextListener.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/MetricsServletContextListener.java new file mode 100644 index 0000000..15279fe --- /dev/null +++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/MetricsServletContextListener.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bookkeeper.stats; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.servlets.MetricsServlet; + +/** + * A servlet to report metrics. + */ +public class MetricsServletContextListener extends MetricsServlet.ContextListener { + + private final MetricRegistry metricRegistry; + + public MetricsServletContextListener(MetricRegistry metricRegistry) { + this.metricRegistry = metricRegistry; + } + + @Override + protected MetricRegistry getMetricRegistry() { + return metricRegistry; + } +} http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/ServletReporter.java ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/ServletReporter.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/ServletReporter.java new file mode 100644 index 0000000..267f75a --- /dev/null +++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/ServletReporter.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bookkeeper.stats; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.health.HealthCheckRegistry; +import com.codahale.metrics.servlets.AdminServlet; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +/** + * Starts a jetty server on a configurable port to export stats. + */ +public class ServletReporter { + + private final MetricRegistry metricRegistry; + private final HealthCheckRegistry healthCheckRegistry; + private final int port; + private final Server jettyServer; + + public ServletReporter(MetricRegistry metricRegistry, + HealthCheckRegistry healthCheckRegistry, + int port) { + this.metricRegistry = metricRegistry; + this.healthCheckRegistry = healthCheckRegistry; + this.port = port; + this.jettyServer = new Server(port); + } + + public void start() throws Exception { + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); + context.setContextPath("/"); + jettyServer.setHandler(context); + + context.addEventListener(new HealthCheckServletContextListener(healthCheckRegistry)); + context.addEventListener(new MetricsServletContextListener(metricRegistry)); + context.addServlet(new ServletHolder(new AdminServlet()), "/*"); + + jettyServer.start(); + } + + public void stop() throws Exception { + jettyServer.stop(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/package-info.java ---------------------------------------------------------------------- diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/package-info.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/package-info.java new file mode 100644 index 0000000..5bdb3ce --- /dev/null +++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Extension of {@link org.apache.bookkeeper.stats.CodahaleMetricsProvider}. + */ +package org.apache.bookkeeper.stats;
