This is an automated email from the ASF dual-hosted git repository. frankgh pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra-sidecar.git
The following commit(s) were added to refs/heads/trunk by this push: new 1d7b3f1 CASSANDRASC-98: Improve logging for traffic shaping / rate limiting configurations 1d7b3f1 is described below commit 1d7b3f10722b52482d2123a4784dda9c92949137 Author: Francisco Guerrero <fran...@apache.org> AuthorDate: Mon Jan 29 10:13:50 2024 -0800 CASSANDRASC-98: Improve logging for traffic shaping / rate limiting configurations Patch by Francisco Guerrero; Reviewed by Yifan Cai for CASSANDRASC-98 --- CHANGES.txt | 1 + .../cassandra/sidecar/common/utils/ByteUtils.java | 81 ++++++++++++++++++++++ .../sidecar/common/utils/ByteUtilsTest.java | 58 ++++++++++++++++ .../sidecar/server/HttpServerOptionsProvider.java | 31 +++++++-- .../cassandra/sidecar/server/MainModule.java | 19 +++-- .../cassandra/sidecar/utils/FileStreamer.java | 3 +- 6 files changed, 181 insertions(+), 12 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 27fba7e..1c3a6f8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,6 @@ 1.0.0 ----- + * Improve logging for traffic shaping / rate limiting configuration (CASSANDRASC-98) * Startup Validation Failures when Checking Sidecar Connectivity (CASSANDRASC-86) * Add support for additional digest validation during SSTable upload (CASSANDRASC-97) * Add sidecar client changes for restore from S3 (CASSANDRASC-95) diff --git a/common/src/main/java/org/apache/cassandra/sidecar/common/utils/ByteUtils.java b/common/src/main/java/org/apache/cassandra/sidecar/common/utils/ByteUtils.java new file mode 100644 index 0000000..8af203a --- /dev/null +++ b/common/src/main/java/org/apache/cassandra/sidecar/common/utils/ByteUtils.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.utils; + +/** + * A utility class to perform byte operations + */ +public final class ByteUtils +{ + /** + * {@code 1 Kibibyte} is equivalent to {@code 1,024 bytes} + */ + public static final long ONE_KIB = 1L << 10; + + /** + * {@code 1 Mebibyte} is equivalent to {@code 1,048,576 bytes} + */ + public static final long ONE_MIB = ONE_KIB << 10; + + /** + * {@code 1 Gibibyte} is equivalent to {@code 1,073,741,824 bytes} + */ + public static final long ONE_GIB = ONE_MIB << 10; + + /** + * {@code 1 Tebibyte} is equivalent to {@code 1,099,511,627,776 bytes} + */ + public static final long ONE_TIB = ONE_GIB << 10; + + /** + * {@code 1 Pebibyte} is equivalent to {@code 1,125,899,906,842,624 bytes} + */ + public static final long ONE_PIB = ONE_TIB << 10; + + /** + * {@code 1 Exbibyte} is equivalent to {@code 1,152,921,504,606,846,976 bytes} + */ + public static final long ONE_EIB = ONE_PIB << 10; + + /** + * Returns a human-readable representation of the number of {@code bytes} in the binary prefix format. A + * long input can only represent up to exbibytes units. + * + * @param bytes the non-negative number of bytes + * @return a human-readable representation of the number of {@code bytes} in the binary prefix format + * @throws IllegalArgumentException when {@code bytes} is a negative value + */ + public static String bytesToHumanReadableBinaryPrefix(long bytes) + { + Preconditions.checkArgument(bytes >= 0, "bytes cannot be negative"); + + if (bytes >= ONE_EIB) return formatHelper(bytes, ONE_EIB, "EiB"); + if (bytes >= ONE_PIB) return formatHelper(bytes, ONE_PIB, "PiB"); + if (bytes >= ONE_TIB) return formatHelper(bytes, ONE_TIB, "TiB"); + if (bytes >= ONE_GIB) return formatHelper(bytes, ONE_GIB, "GiB"); + if (bytes >= ONE_MIB) return formatHelper(bytes, ONE_MIB, "MiB"); + if (bytes >= ONE_KIB) return formatHelper(bytes, ONE_KIB, "KiB"); + return bytes + " B"; + } + + static String formatHelper(double bytes, long baseUnit, String unitName) + { + return String.format("%.2f %s", bytes / baseUnit, unitName); + } +} diff --git a/common/src/test/java/org/apache/cassandra/sidecar/common/utils/ByteUtilsTest.java b/common/src/test/java/org/apache/cassandra/sidecar/common/utils/ByteUtilsTest.java new file mode 100644 index 0000000..981729e --- /dev/null +++ b/common/src/test/java/org/apache/cassandra/sidecar/common/utils/ByteUtilsTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.utils; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +/** + * Unit tests for {@link ByteUtils} + */ +class ByteUtilsTest +{ + @ParameterizedTest + @ValueSource(longs = { -1L, -10L, -100L, -1024L, Long.MIN_VALUE }) + void failsWithNegativeInputs(long bytes) + { + assertThatIllegalArgumentException().isThrownBy(() -> ByteUtils.bytesToHumanReadableBinaryPrefix(bytes)) + .withMessage("bytes cannot be negative"); + } + + @Test + void testToHumanReadableBinaryPrefix() + { + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(0)).isEqualTo("0 B"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(1152921504606846976L)).isEqualTo("1.00 EiB"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(1152921504606846977L)).isEqualTo("1.00 EiB"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(Long.MAX_VALUE)).isEqualTo("8.00 EiB"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(64)).isEqualTo("64 B"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(3221225472L)).isEqualTo("3.00 GiB"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(4L * 1024 * 1024)).isEqualTo("4.00 MiB"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(4L * 1024 * 1024 * 1024)).isEqualTo("4.00 GiB"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(4L * 1024 * 1024 * 1024 * 1024)).isEqualTo("4.00 TiB"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(4L * 1024 * 1024 * 1024 * 1024 * 1024)) + .isEqualTo("4.00 PiB"); + assertThat(ByteUtils.bytesToHumanReadableBinaryPrefix(4L * 1024 * 1024 * 1024 * 1024 * 1024 * 1024)) + .isEqualTo("4.00 EiB"); + } +} diff --git a/src/main/java/org/apache/cassandra/sidecar/server/HttpServerOptionsProvider.java b/src/main/java/org/apache/cassandra/sidecar/server/HttpServerOptionsProvider.java index fad0b81..2de80dd 100644 --- a/src/main/java/org/apache/cassandra/sidecar/server/HttpServerOptionsProvider.java +++ b/src/main/java/org/apache/cassandra/sidecar/server/HttpServerOptionsProvider.java @@ -39,6 +39,7 @@ import org.apache.cassandra.sidecar.utils.SslUtils; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.cassandra.sidecar.common.utils.ByteUtils.bytesToHumanReadableBinaryPrefix; /** * A provider that takes the {@link SidecarConfiguration} and builds {@link HttpServerOptions} from the configured @@ -143,18 +144,34 @@ public class HttpServerOptionsProvider implements Function<SidecarConfiguration, /** * Returns the built {@link TrafficShapingOptions} that are going to be applied to the server. * - * @param trafficShapingConfig the configuration for the traffic shaping options. + * @param config the configuration for the traffic shaping options. * @return the built {@link TrafficShapingOptions} from the {@link TrafficShapingConfiguration} */ - protected TrafficShapingOptions buildTrafficShapingOptions(TrafficShapingConfiguration trafficShapingConfig) + protected TrafficShapingOptions buildTrafficShapingOptions(TrafficShapingConfiguration config) { + long inboundGlobalBandwidthBytesPerSecond = config.inboundGlobalBandwidthBytesPerSecond(); + long outboundGlobalBandwidthBytesPerSecond = config.outboundGlobalBandwidthBytesPerSecond(); + long peakOutboundGlobalBandwidthBytesPerSecond = config.peakOutboundGlobalBandwidthBytesPerSecond(); + LOGGER.info("Configured traffic shaping options. InboundGlobalBandwidth={}/s " + + "rawInboundGlobalBandwidth={} B/s OutboundGlobalBandwidth={}/s rawOutboundGlobalBandwidth={} B/s " + + "PeakOutboundGlobalBandwidth={}/s rawPeakOutboundGlobalBandwidth={} B/s IntervalForStats={}ms " + + "MaxDelayToWait={}ms", + bytesToHumanReadableBinaryPrefix(inboundGlobalBandwidthBytesPerSecond), + inboundGlobalBandwidthBytesPerSecond, + bytesToHumanReadableBinaryPrefix(outboundGlobalBandwidthBytesPerSecond), + outboundGlobalBandwidthBytesPerSecond, + bytesToHumanReadableBinaryPrefix(peakOutboundGlobalBandwidthBytesPerSecond), + peakOutboundGlobalBandwidthBytesPerSecond, + config.checkIntervalForStatsMillis(), + config.maxDelayToWaitMillis() + ); return new TrafficShapingOptions() - .setInboundGlobalBandwidth(trafficShapingConfig.inboundGlobalBandwidthBytesPerSecond()) - .setOutboundGlobalBandwidth(trafficShapingConfig.outboundGlobalBandwidthBytesPerSecond()) - .setPeakOutboundGlobalBandwidth(trafficShapingConfig.peakOutboundGlobalBandwidthBytesPerSecond()) - .setCheckIntervalForStats(trafficShapingConfig.checkIntervalForStatsMillis()) + .setInboundGlobalBandwidth(inboundGlobalBandwidthBytesPerSecond) + .setOutboundGlobalBandwidth(outboundGlobalBandwidthBytesPerSecond) + .setPeakOutboundGlobalBandwidth(peakOutboundGlobalBandwidthBytesPerSecond) + .setCheckIntervalForStats(config.checkIntervalForStatsMillis()) .setCheckIntervalForStatsTimeUnit(MILLISECONDS) - .setMaxDelayToWait(trafficShapingConfig.maxDelayToWaitMillis()) + .setMaxDelayToWait(config.maxDelayToWaitMillis()) .setMaxDelayToWaitUnit(MILLISECONDS); } } diff --git a/src/main/java/org/apache/cassandra/sidecar/server/MainModule.java b/src/main/java/org/apache/cassandra/sidecar/server/MainModule.java index 31ab7b1..008fadc 100644 --- a/src/main/java/org/apache/cassandra/sidecar/server/MainModule.java +++ b/src/main/java/org/apache/cassandra/sidecar/server/MainModule.java @@ -26,6 +26,8 @@ import java.util.Map; import java.util.stream.Collectors; import com.google.common.util.concurrent.SidecarRateLimiter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.datastax.driver.core.NettyOptions; import com.google.inject.AbstractModule; @@ -98,6 +100,7 @@ import org.apache.cassandra.sidecar.stats.SidecarStats; import org.apache.cassandra.sidecar.utils.CassandraVersionProvider; import org.apache.cassandra.sidecar.utils.TimeProvider; +import static org.apache.cassandra.sidecar.common.utils.ByteUtils.bytesToHumanReadableBinaryPrefix; import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_STOP; /** @@ -105,6 +108,7 @@ import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_ */ public class MainModule extends AbstractModule { + private static final Logger LOGGER = LoggerFactory.getLogger(MainModule.class); public static final Map<String, String> OK_STATUS = Collections.singletonMap("status", "OK"); public static final Map<String, String> NOT_OK_STATUS = Collections.singletonMap("status", "NOT_OK"); @@ -383,10 +387,13 @@ public class MainModule extends AbstractModule @Provides @Singleton + @Named("StreamRequestRateLimiter") public SidecarRateLimiter streamRequestRateLimiter(ServiceConfiguration config) { - return SidecarRateLimiter.create(config.throttleConfiguration() - .rateLimitStreamRequestsPerSecond()); + long permitsPerSecond = config.throttleConfiguration().rateLimitStreamRequestsPerSecond(); + LOGGER.info("Configuring streamRequestRateLimiter. rateLimitStreamRequestsPerSecond={}", + permitsPerSecond); + return SidecarRateLimiter.create(permitsPerSecond); } @Provides @@ -394,8 +401,12 @@ public class MainModule extends AbstractModule @Named("IngressFileRateLimiter") public SidecarRateLimiter ingressFileRateLimiter(ServiceConfiguration config) { - return SidecarRateLimiter.create(config.trafficShapingConfiguration() - .inboundGlobalFileBandwidthBytesPerSecond()); + long bytesPerSecond = config.trafficShapingConfiguration() + .inboundGlobalFileBandwidthBytesPerSecond(); + LOGGER.info("Configuring ingressFileRateLimiter. inboundGlobalFileBandwidth={}/s " + + "rawInboundGlobalFileBandwidth={} B/s", bytesToHumanReadableBinaryPrefix(bytesPerSecond), + bytesPerSecond); + return SidecarRateLimiter.create(bytesPerSecond); } @Provides diff --git a/src/main/java/org/apache/cassandra/sidecar/utils/FileStreamer.java b/src/main/java/org/apache/cassandra/sidecar/utils/FileStreamer.java index 77cd16f..2f133af 100644 --- a/src/main/java/org/apache/cassandra/sidecar/utils/FileStreamer.java +++ b/src/main/java/org/apache/cassandra/sidecar/utils/FileStreamer.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.google.inject.name.Named; import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.ext.web.handler.HttpException; @@ -60,7 +61,7 @@ public class FileStreamer @Inject public FileStreamer(ExecutorPools executorPools, ServiceConfiguration config, - SidecarRateLimiter rateLimiter, + @Named("StreamRequestRateLimiter") SidecarRateLimiter rateLimiter, SidecarStats stats) { this.executorPools = executorPools; --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org