This is an automated email from the ASF dual-hosted git repository. echauchot pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/flink-connector-cassandra.git
commit c8dcb55bbe7e7ead865aad9e2ce03768b2a6cc75 Author: Etienne Chauchot <[email protected]> AuthorDate: Mon Jul 28 14:51:21 2025 +0200 [FLINK-37937] upgrade to latest cassandra 4.x. reformat --- .../cassandra/CassandraTestEnvironment.java | 59 +- .../resources/{cassandra.yml => cassandra.yaml} | 956 +++++++++++++++------ 2 files changed, 738 insertions(+), 277 deletions(-) diff --git a/flink-connector-cassandra/src/test/java/org/apache/flink/connector/cassandra/CassandraTestEnvironment.java b/flink-connector-cassandra/src/test/java/org/apache/flink/connector/cassandra/CassandraTestEnvironment.java index 190b669..dc783e9 100644 --- a/flink-connector-cassandra/src/test/java/org/apache/flink/connector/cassandra/CassandraTestEnvironment.java +++ b/flink-connector-cassandra/src/test/java/org/apache/flink/connector/cassandra/CassandraTestEnvironment.java @@ -39,6 +39,7 @@ import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; import java.net.InetSocketAddress; import java.time.Duration; @@ -50,7 +51,7 @@ import java.time.Duration; @Testcontainers public class CassandraTestEnvironment implements TestResource { private static final Logger LOG = LoggerFactory.getLogger(CassandraTestEnvironment.class); - private static final String DOCKER_CASSANDRA_IMAGE = "cassandra:4.0.8"; + private static final String DOCKER_CASSANDRA_IMAGE = "cassandra:4.1.9"; private static final int CQL_PORT = 9042; private static final int READ_TIMEOUT_MILLIS = 36000; @@ -85,7 +86,7 @@ public class CassandraTestEnvironment implements TestResource { + " (col1, col2, col3, col4)" + " VALUES (%d, %d, %d, %d)"; private static final int NB_SPLITS_RECORDS = 1000; - private static final long FLSUH_MEMTABLES_DELAY = 30_000L; + private static final long FLUSH_MEMTABLES_DELAY = 30_000L; @Container private final CassandraContainer cassandraContainer1; @Container private final CassandraContainer cassandraContainer2; @@ -101,25 +102,29 @@ public class CassandraTestEnvironment implements TestResource { this.insertTestDataForSplitSizeTests = insertTestDataForSplitSizeTests; Network network = Network.newNetwork(); - cassandraContainer1 = (CassandraContainer) new CassandraContainer(DOCKER_CASSANDRA_IMAGE) - .withNetwork(network) - .withEnv("CASSANDRA_CLUSTER_NAME", "testcontainers") - .withEnv("CASSANDRA_SEEDS", "cassandra") - .withEnv("JVM_OPTS", "") - .withNetworkAliases("cassandra") - .withCopyFileToContainer( - MountableFile.forClasspathResource("cassandra.yaml"), - "/etc/cassandra/cassandra.yaml" //for timeouts - ); - cassandraContainer2 = (CassandraContainer) new CassandraContainer(DOCKER_CASSANDRA_IMAGE) - .withNetwork(network) - .withEnv("CASSANDRA_CLUSTER_NAME", "testcontainers") - .withEnv("JVM_OPTS", "") - .withEnv("CASSANDRA_SEEDS", "cassandra") - .withCopyFileToContainer( - MountableFile.forClasspathResource("cassandra.yaml"), - "/etc/cassandra/cassandra.yaml" //for timeouts - ); + cassandraContainer1 = + (CassandraContainer) + new CassandraContainer(DOCKER_CASSANDRA_IMAGE) + .withNetwork(network) + .withEnv("CASSANDRA_CLUSTER_NAME", "testcontainers") + .withEnv("CASSANDRA_SEEDS", "cassandra") + .withEnv("JVM_OPTS", "") + .withNetworkAliases("cassandra") + .withCopyFileToContainer( + MountableFile.forClasspathResource("cassandra.yaml"), + "/etc/cassandra/cassandra.yaml" // for timeouts + ); + cassandraContainer2 = + (CassandraContainer) + new CassandraContainer(DOCKER_CASSANDRA_IMAGE) + .withNetwork(network) + .withEnv("CASSANDRA_CLUSTER_NAME", "testcontainers") + .withEnv("JVM_OPTS", "") + .withEnv("CASSANDRA_SEEDS", "cassandra") + .withCopyFileToContainer( + MountableFile.forClasspathResource("cassandra.yaml"), + "/etc/cassandra/cassandra.yaml" // for timeouts + ); } @Override @@ -224,17 +229,15 @@ public class CassandraTestEnvironment implements TestResource { * Force the refresh of system.size_estimates table. It is needed for the tests because we just * inserted records. It is done on a single node as the size estimation for split generation is * evaluated based on the ring fraction the connect node represents in the cluster. We first - * flush the memTables to SSTables because the size estimates are only on SSTables. Then we refresh - * the size estimates. + * flush the MemTables to SSTables because the size estimates are only on SSTables. Then we + * refresh the size estimates. */ void refreshSizeEstimates(String table) throws Exception { final ExecResult execResult1 = - cassandraContainer1.execInContainer( - "nodetool", "flush", KEYSPACE, table); - Thread.sleep(FLSUH_MEMTABLES_DELAY); + cassandraContainer1.execInContainer("nodetool", "flush", KEYSPACE, table); + Thread.sleep(FLUSH_MEMTABLES_DELAY); final ExecResult execResult2 = - cassandraContainer1.execInContainer( - "nodetool", "refreshsizeestimates"); + cassandraContainer1.execInContainer("nodetool", "refreshsizeestimates"); if (execResult1.getExitCode() != 0 || execResult2.getExitCode() != 0) { throw new RuntimeException( "Failed to refresh system.size_estimates on the Cassandra cluster"); diff --git a/flink-connector-cassandra/src/test/resources/cassandra.yml b/flink-connector-cassandra/src/test/resources/cassandra.yaml similarity index 61% rename from flink-connector-cassandra/src/test/resources/cassandra.yml rename to flink-connector-cassandra/src/test/resources/cassandra.yaml index cf4918e..71566fe 100644 --- a/flink-connector-cassandra/src/test/resources/cassandra.yml +++ b/flink-connector-cassandra/src/test/resources/cassandra.yaml @@ -61,14 +61,16 @@ hinted_handoff_enabled: true # this defines the maximum amount of time a dead host will have hints # generated. After it has been dead this long, new hints for it will not be # created until it has been seen alive and gone down again. -max_hint_window_in_ms: 10800000 # 3 hours +# Min unit: ms +max_hint_window: 3h -# Maximum throttle in KBs per second, per delivery thread. This will be +# Maximum throttle in KiBs per second, per delivery thread. This will be # reduced proportionally to the number of nodes in the cluster. (If there # are two nodes in the cluster, each delivery thread will use the maximum # rate; if there are three, each will throttle to half of the maximum, # since we expect two nodes to be delivering hints simultaneously.) -hinted_handoff_throttle_in_kb: 1024 +# Min unit: KiB +hinted_handoff_throttle: 1024KiB # Number of threads with which to deliver hints; # Consider increasing this number when you have multi-dc deployments, since @@ -81,10 +83,26 @@ max_hints_delivery_threads: 2 # How often hints should be flushed from the internal buffers to disk. # Will *not* trigger fsync. -hints_flush_period_in_ms: 10000 +# Min unit: ms +hints_flush_period: 10000ms -# Maximum size for a single hints file, in megabytes. -max_hints_file_size_in_mb: 128 +# Maximum size for a single hints file, in mebibytes. +# Min unit: MiB +max_hints_file_size: 128MiB + +# The file size limit to store hints for an unreachable host, in mebibytes. +# Once the local hints files have reached the limit, no more new hints will be created. +# Set a non-positive value will disable the size limit. +# max_hints_size_per_host: 0MiB + +# Enable / disable automatic cleanup for the expired and orphaned hints file. +# Disable the option in order to preserve those hints on the disk. +auto_hints_cleanup_enabled: false + +# Enable/disable transfering hints to a peer during decommission. Even when enabled, this does not guarantee +# consistency for logged batches, and it may delay decommission when coupled with a strict hinted_handoff_throttle. +# Default: true +# transfer_hints_on_decommission: true # Compression to apply to the hint files. If omitted, hints files # will be written uncompressed. LZ4, Snappy, and Deflate compressors @@ -94,9 +112,50 @@ max_hints_file_size_in_mb: 128 # parameters: # - -# Maximum throttle in KBs per second, total. This will be +# Enable / disable persistent hint windows. +# +# If set to false, a hint will be stored only in case a respective node +# that hint is for is down less than or equal to max_hint_window. +# +# If set to true, a hint will be stored in case there is not any +# hint which was stored earlier than max_hint_window. This is for cases +# when a node keeps to restart and hints are not delivered yet, we would be saving +# hints for that node indefinitely. +# +# Defaults to true. +# +# hint_window_persistent_enabled: true + +# Maximum throttle in KiBs per second, total. This will be # reduced proportionally to the number of nodes in the cluster. -batchlog_replay_throttle_in_kb: 1024 +# Min unit: KiB +batchlog_replay_throttle: 1024KiB + +# Strategy to choose the batchlog storage endpoints. +# +# Available options: +# +# - random_remote +# Default, purely random, prevents the local rack, if possible. +# +# - prefer_local +# Similar to random_remote. Random, except that one of the replications will go to the local rack, +# which mean it offers lower availability guarantee than random_remote or dynamic_remote. +# +# - dynamic_remote +# Using DynamicEndpointSnitch to select batchlog storage endpoints, prevents the +# local rack, if possible. This strategy offers the same availability guarantees +# as random_remote but selects the fastest endpoints according to the DynamicEndpointSnitch. +# (DynamicEndpointSnitch currently only tracks reads and not writes - i.e. write-only +# (or mostly-write) workloads might not benefit from this strategy.) +# Note: this strategy will fall back to random_remote, if dynamic_snitch is not enabled. +# +# - dynamic +# Mostly the same as dynamic_remote, except that local rack is not excluded, which mean it offers lower +# availability guarantee than random_remote or dynamic_remote. +# Note: this strategy will fall back to random_remote, if dynamic_snitch is not enabled. +# +# batchlog_endpoint_strategy: random_remote # Authentication backend, implementing IAuthenticator; used to identify users # Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator, @@ -139,35 +198,68 @@ role_manager: CassandraRoleManager # increase system_auth keyspace replication factor if you use this authorizer. network_authorizer: AllowAllNetworkAuthorizer +# Depending on the auth strategy of the cluster, it can be beneficial to iterate +# from root to table (root -> ks -> table) instead of table to root (table -> ks -> root). +# As the auth entries are whitelisting, once a permission is found you know it to be +# valid. We default to false as the legacy behavior is to query at the table level then +# move back up to the root. See CASSANDRA-17016 for details. +# traverse_auth_from_root: false + # Validity period for roles cache (fetching granted roles can be an expensive # operation depending on the role manager, CassandraRoleManager is one example) # Granted roles are cached for authenticated sessions in AuthenticatedUser and # after the period specified here, become eligible for (async) reload. # Defaults to 2000, set to 0 to disable caching entirely. # Will be disabled automatically for AllowAllAuthenticator. -roles_validity_in_ms: 2000 +# For a long-running cache using roles_cache_active_update, consider +# setting to something longer such as a daily validation: 86400000 +# Min unit: ms +roles_validity: 2000ms # Refresh interval for roles cache (if enabled). # After this interval, cache entries become eligible for refresh. Upon next # access, an async reload is scheduled and the old value returned until it -# completes. If roles_validity_in_ms is non-zero, then this must be +# completes. If roles_validity is non-zero, then this must be # also. -# Defaults to the same value as roles_validity_in_ms. -# roles_update_interval_in_ms: 2000 +# This setting is also used to inform the interval of auto-updating if +# using roles_cache_active_update. +# Defaults to the same value as roles_validity. +# For a long-running cache, consider setting this to 60000 (1 hour) etc. +# Min unit: ms +# roles_update_interval: 2000ms + +# If true, cache contents are actively updated by a background task at the +# interval set by roles_update_interval. If false, cache entries +# become eligible for refresh after their update interval. Upon next access, +# an async reload is scheduled and the old value returned until it completes. +# roles_cache_active_update: false # Validity period for permissions cache (fetching permissions can be an # expensive operation depending on the authorizer, CassandraAuthorizer is # one example). Defaults to 2000, set to 0 to disable. # Will be disabled automatically for AllowAllAuthorizer. -permissions_validity_in_ms: 2000 +# For a long-running cache using permissions_cache_active_update, consider +# setting to something longer such as a daily validation: 86400000ms +# Min unit: ms +permissions_validity: 2000ms # Refresh interval for permissions cache (if enabled). # After this interval, cache entries become eligible for refresh. Upon next # access, an async reload is scheduled and the old value returned until it -# completes. If permissions_validity_in_ms is non-zero, then this must be +# completes. If permissions_validity is non-zero, then this must be # also. -# Defaults to the same value as permissions_validity_in_ms. -# permissions_update_interval_in_ms: 2000 +# This setting is also used to inform the interval of auto-updating if +# using permissions_cache_active_update. +# Defaults to the same value as permissions_validity. +# For a longer-running permissions cache, consider setting to update hourly (60000) +# Min unit: ms +# permissions_update_interval: 2000ms + +# If true, cache contents are actively updated by a background task at the +# interval set by permissions_update_interval. If false, cache entries +# become eligible for refresh after their update interval. Upon next access, +# an async reload is scheduled and the old value returned until it completes. +# permissions_cache_active_update: false # Validity period for credentials cache. This cache is tightly coupled to # the provided PasswordAuthenticator implementation of IAuthenticator. If @@ -178,15 +270,28 @@ permissions_validity_in_ms: 2000 # underlying table, it may not bring a significant reduction in the # latency of individual authentication attempts. # Defaults to 2000, set to 0 to disable credentials caching. -credentials_validity_in_ms: 2000 +# For a long-running cache using credentials_cache_active_update, consider +# setting to something longer such as a daily validation: 86400000 +# Min unit: ms +credentials_validity: 2000ms # Refresh interval for credentials cache (if enabled). # After this interval, cache entries become eligible for refresh. Upon next # access, an async reload is scheduled and the old value returned until it -# completes. If credentials_validity_in_ms is non-zero, then this must be +# completes. If credentials_validity is non-zero, then this must be # also. -# Defaults to the same value as credentials_validity_in_ms. -# credentials_update_interval_in_ms: 2000 +# This setting is also used to inform the interval of auto-updating if +# using credentials_cache_active_update. +# Defaults to the same value as credentials_validity. +# For a longer-running permissions cache, consider setting to update hourly (60000) +# Min unit: ms +# credentials_update_interval: 2000ms + +# If true, cache contents are actively updated by a background task at the +# interval set by credentials_update_interval. If false (default), cache entries +# become eligible for refresh after their update interval. Upon next access, +# an async reload is scheduled and the old value returned until it completes. +# credentials_cache_active_update: false # The partitioner is responsible for distributing groups of rows (by # partition key) across nodes in the cluster. The partitioner can NOT be @@ -286,8 +391,9 @@ commit_failure_policy: stop # fit in the cache. In most cases it is not neccessary to change this value. # Constantly re-preparing statements is a performance penalty. # -# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater -prepared_statements_cache_size_mb: +# Default value ("auto") is 1/256th of the heap or 10MiB, whichever is greater +# Min unit: MiB +prepared_statements_cache_size: # Maximum size of the key cache in memory. # @@ -300,8 +406,9 @@ prepared_statements_cache_size_mb: # # NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. # -# Default value is empty to make it "auto" (min(5% of Heap (in MB), 100MB)). Set to 0 to disable key cache. -key_cache_size_in_mb: +# Default value is empty to make it "auto" (min(5% of Heap (in MiB), 100MiB)). Set to 0 to disable key cache. +# Min unit: MiB +key_cache_size: # Duration in seconds after which Cassandra should # save the key cache. Caches are saved to saved_caches_directory as @@ -312,7 +419,8 @@ key_cache_size_in_mb: # has limited use. # # Default is 14400 or 4 hours. -key_cache_save_period: 14400 +# Min unit: s +key_cache_save_period: 4h # Number of keys from the key cache to save # Disabled by default, meaning all keys are going to be saved @@ -336,7 +444,8 @@ key_cache_save_period: 14400 # headroom for OS block level cache. Do never allow your system to swap. # # Default value is 0, to disable row caching. -row_cache_size_in_mb: 0 +# Min unit: MiB +row_cache_size: 0MiB # Duration in seconds after which Cassandra should save the row cache. # Caches are saved to saved_caches_directory as specified in this configuration file. @@ -346,7 +455,8 @@ row_cache_size_in_mb: 0 # has limited use. # # Default is 0 to disable saving the row cache. -row_cache_save_period: 0 +# Min unit: s +row_cache_save_period: 0s # Number of keys from the row cache to save. # Specify 0 (which is the default), meaning all keys are going to be saved @@ -363,16 +473,18 @@ row_cache_save_period: 0 # # NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. # -# Default value is empty to make it "auto" (min(2.5% of Heap (in MB), 50MB)). Set to 0 to disable counter cache. +# Default value is empty to make it "auto" (min(2.5% of Heap (in MiB), 50MiB)). Set to 0 to disable counter cache. # NOTE: if you perform counter deletes and rely on low gcgs, you should disable the counter cache. -counter_cache_size_in_mb: +# Min unit: MiB +counter_cache_size: # Duration in seconds after which Cassandra should # save the counter cache (keys only). Caches are saved to saved_caches_directory as # specified in this configuration file. # # Default is 7200 or 2 hours. -counter_cache_save_period: 7200 +# Min unit: s +counter_cache_save_period: 7200s # Number of keys from the counter cache to save # Disabled by default, meaning all keys are going to be saved @@ -383,9 +495,10 @@ counter_cache_save_period: 7200 # saved_caches_directory: /var/lib/cassandra/saved_caches # Number of seconds the server will wait for each cache (row, key, etc ...) to load while starting -# the Cassandra process. Setting this to a negative value is equivalent to disabling all cache loading on startup +# the Cassandra process. Setting this to zero is equivalent to disabling all cache loading on startup # while still having the cache during runtime. -# cache_load_timeout_seconds: 30 +# Min unit: s +# cache_load_timeout: 30s # commitlog_sync may be either "periodic", "group", or "batch." # @@ -398,19 +511,22 @@ counter_cache_save_period: 7200 # # group mode is similar to batch mode, where Cassandra will not ack writes # until the commit log has been flushed to disk. The difference is group -# mode will wait up to commitlog_sync_group_window_in_ms between flushes. +# mode will wait up to commitlog_sync_group_window between flushes. # -# commitlog_sync_group_window_in_ms: 1000 +# Min unit: ms +# commitlog_sync_group_window: 1000ms # # the default option is "periodic" where writes may be acked immediately -# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# and the CommitLog is simply synced every commitlog_sync_period # milliseconds. commitlog_sync: periodic -commitlog_sync_period_in_ms: 10000 +# Min unit: ms +commitlog_sync_period: 10000ms # When in periodic commitlog mode, the number of milliseconds to block writes # while waiting for a slow disk flush to complete. -# periodic_commitlog_sync_lag_block_in_ms: +# Min unit: ms +# periodic_commitlog_sync_lag_block: # The size of the individual commitlog file segments. A commitlog # segment may be archived, deleted, or recycled once all the data @@ -421,14 +537,15 @@ commitlog_sync_period_in_ms: 10000 # archiving commitlog segments (see commitlog_archiving.properties), # then you probably want a finer granularity of archiving; 8 or 16 MB # is reasonable. -# Max mutation size is also configurable via max_mutation_size_in_kb setting in -# cassandra.yaml. The default is half the size commitlog_segment_size_in_mb * 1024. +# Max mutation size is also configurable via max_mutation_size setting in +# cassandra.yaml. The default is half the size commitlog_segment_size in bytes. # This should be positive and less than 2048. # -# NOTE: If max_mutation_size_in_kb is set explicitly then commitlog_segment_size_in_mb must -# be set to at least twice the size of max_mutation_size_in_kb / 1024 +# NOTE: If max_mutation_size is set explicitly then commitlog_segment_size must +# be set to at least twice the size of max_mutation_size # -commitlog_segment_size_in_mb: 32 +# Min unit: MiB +commitlog_segment_size: 32MiB # Compression to apply to the commit log. If omitted, the commit log # will be written uncompressed. LZ4, Snappy, and Deflate compressors @@ -456,15 +573,15 @@ commitlog_segment_size_in_mb: 32 # any class that implements the SeedProvider interface and has a # constructor that takes a Map<String, String> of parameters will do. seed_provider: - # Addresses of hosts that are deemed contact points. - # Cassandra nodes use this list of hosts to find each other and learn - # the topology of the ring. You must change this if you are running - # multiple nodes! - - class_name: org.apache.cassandra.locator.SimpleSeedProvider - parameters: - # seeds is actually a comma-delimited list of addresses. - # Ex: "<ip1>,<ip2>,<ip3>" - - seeds: "cassandra" + # Addresses of hosts that are deemed contact points. + # Cassandra nodes use this list of hosts to find each other and learn + # the topology of the ring. You must change this if you are running + # multiple nodes! + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # seeds is actually a comma-delimited list of addresses. + # Ex: "<ip1>,<ip2>,<ip3>" + - seeds: "cassandra" # For workloads with more data than can fit in memory, Cassandra's # bottleneck will be reads that need to fetch data from @@ -492,7 +609,8 @@ concurrent_materialized_view_writes: 32 # overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size # if the default 64k chunk size is used). # Memory is only allocated when needed. -# networking_cache_size_in_mb: 128 +# Min unit: MiB +# networking_cache_size: 128MiB # Enable the sstable chunk cache. The chunk cache will store recently accessed # sections of the sstable in-memory as uncompressed buffers. @@ -506,11 +624,12 @@ concurrent_materialized_view_writes: 32 # overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size # if the default 64k chunk size is used). # Memory is only allocated when needed. -# file_cache_size_in_mb: 512 +# Min unit: MiB +# file_cache_size: 512MiB # Flag indicating whether to allocate on or off heap when the sstable buffer # pool is exhausted, that is when it has exceeded the maximum memory -# file_cache_size_in_mb, beyond which it will not cache buffers but allocate on request. +# file_cache_size, beyond which it will not cache buffers but allocate on request. # buffer_pool_use_heap_if_exhausted: true @@ -524,8 +643,10 @@ concurrent_materialized_view_writes: 32 # accepting writes when the limit is exceeded until a flush completes, # and will trigger a flush based on memtable_cleanup_threshold # If omitted, Cassandra will set both to 1/4 the size of the heap. -# memtable_heap_space_in_mb: 2048 -# memtable_offheap_space_in_mb: 2048 +# Min unit: MiB +# memtable_heap_space: 2048MiB +# Min unit: MiB +# memtable_offheap_space: 2048MiB # memtable_cleanup_threshold is deprecated. The default calculation # is the only reasonable choice. See the comments on memtable_flush_writers @@ -553,16 +674,41 @@ concurrent_materialized_view_writes: 32 # off heap objects memtable_allocation_type: heap_buffers -# Limit memory usage for Merkle tree calculations during repairs. The default -# is 1/16th of the available heap. The main tradeoff is that smaller trees -# have less resolution, which can lead to over-streaming data. If you see heap -# pressure during repairs, consider lowering this, but you cannot go below -# one megabyte. If you see lots of over-streaming, consider raising -# this or using subrange repair. +# Limit memory usage for Merkle tree calculations during repairs of a certain +# table and common token range. Repair commands targetting multiple tables or +# virtual nodes can exceed this limit depending on concurrent_merkle_tree_requests. +# +# The default is 1/16th of the available heap. The main tradeoff is that +# smaller trees have less resolution, which can lead to over-streaming data. +# If you see heap pressure during repairs, consider lowering this, but you +# cannot go below one mebibyte. If you see lots of over-streaming, consider +# raising this or using subrange repair. # # For more details see https://issues.apache.org/jira/browse/CASSANDRA-14096. # -# repair_session_space_in_mb: +# Min unit: MiB +# repair_session_space: + +# The number of simultaneous Merkle tree requests during repairs that can +# be performed by a repair command. The size of each validation request is +# limited by the repair_session_space property, so setting this to 1 will make +# sure that a repair command doesn't exceed that limit, even if the repair +# command is repairing multiple tables or multiple virtual nodes. +# +# There isn't a limit by default for backwards compatibility, but this can +# produce OOM for commands repairing multiple tables or multiple virtual nodes. +# A limit of just 1 simultaneous Merkle tree request is generally recommended +# with no virtual nodes so repair_session_space, and thereof the Merkle tree +# resolution, can be high. For virtual nodes a value of 1 with the default +# repair_session_space value will produce higher resolution Merkle trees +# at the expense of speed. Alternatively, when working with virtual nodes it +# can make sense to reduce the repair_session_space and increase the value of +# concurrent_merkle_tree_requests because each range will contain fewer data. +# +# For more details see https://issues.apache.org/jira/browse/CASSANDRA-19336. +# +# A zero value means no limit. +# concurrent_merkle_tree_requests: 0 # Total space to use for commit logs on disk. # @@ -573,7 +719,7 @@ memtable_allocation_type: heap_buffers # The default value is the smaller of 8192, and 1/4 of the total space # of the commitlog volume. # -# commitlog_total_space_in_mb: 8192 +# commitlog_total_space: 8192MiB # This sets the number of memtable flush writer threads per disk # as well as the total number of memtables that can be flushed concurrently. @@ -602,7 +748,7 @@ memtable_allocation_type: heap_buffers # and flush size and frequency. More is not better you just need enough flush writers # to never stall waiting for flushing to free memory. # -#memtable_flush_writers: 2 +# memtable_flush_writers: 2 # Total space to use for change-data-capture logs on disk. # @@ -610,14 +756,16 @@ memtable_allocation_type: heap_buffers # on Mutations including tables with CDC enabled. A CDCCompactor is responsible # for parsing the raw CDC logs and deleting them when parsing is completed. # -# The default value is the min of 4096 mb and 1/8th of the total space +# The default value is the min of 4096 MiB and 1/8th of the total space # of the drive where cdc_raw_directory resides. -# cdc_total_space_in_mb: 4096 +# Min unit: MiB +# cdc_total_space: 4096MiB # When we hit our cdc_raw limit and the CDCCompactor is either running behind # or experiencing backpressure, we check at the following interval to see if any # new space for cdc-tracked tables has been made available. Default to 250ms -# cdc_free_space_check_interval_ms: 250 +# Min unit: ms +# cdc_free_space_check_interval: 250ms # A fixed memory pool size in MB for for SSTable index summaries. If left # empty, this will default to 5% of the heap size. If the memory usage of @@ -625,13 +773,15 @@ memtable_allocation_type: heap_buffers # shrink their index summaries in order to meet this limit. However, this # is a best-effort process. In extreme conditions Cassandra may need to use # more than this amount of memory. -index_summary_capacity_in_mb: +# Min unit: KiB +index_summary_capacity: # How frequently index summaries should be resampled. This is done # periodically to redistribute memory from the fixed-size pool to sstables -# proportional their recent read rates. Setting to -1 will disable this +# proportional their recent read rates. Setting to null value will disable this # process, leaving existing index summaries at their current sampling level. -index_summary_resize_interval_in_minutes: 60 +# Min unit: m +index_summary_resize_interval: 60m # Whether to, when doing sequential writing, fsync() at intervals in # order to force the operating system to flush the dirty @@ -639,7 +789,8 @@ index_summary_resize_interval_in_minutes: 60 # impacting read latencies. Almost always a good idea on SSDs; not # necessarily on platters. trickle_fsync: false -trickle_fsync_interval_in_kb: 10240 +# Min unit: KiB +trickle_fsync_interval: 10240KiB # TCP port, for commands and data # For security reasons, you should not expose this port to the internet. Firewall it if needed. @@ -664,7 +815,7 @@ ssl_storage_port: 7001 # # Setting listen_address to 0.0.0.0 is always wrong. # -listen_address: 172.20.0.3 +listen_address: 172.19.0.3 # Set listen_address OR listen_interface, not both. Interfaces must correspond # to a single address, IP aliasing is not supported. @@ -678,7 +829,7 @@ listen_address: 172.20.0.3 # Address to broadcast to other Cassandra nodes # Leaving this blank will set it to the same value as listen_address -broadcast_address: 172.20.0.3 +broadcast_address: 172.19.0.3 # When using multiple physical network interfaces, set this # to true to listen on broadcast_address in addition to @@ -709,11 +860,15 @@ native_transport_port: 9042 # The maximum threads for handling requests (note that idle threads are stopped # after 30 seconds so there is not corresponding minimum setting). # native_transport_max_threads: 128 +# The maximum threads for handling auth requests in a separate executor from main request executor. +# When set to 0, main executor for requests is used. +# native_transport_max_auth_threads: 0 # # The maximum size of allowed frame. Frame (requests) larger than this will -# be rejected as invalid. The default is 256MB. If you're changing this parameter, -# you may want to adjust max_value_size_in_mb accordingly. This should be positive and less than 2048. -# native_transport_max_frame_size_in_mb: 256 +# be rejected as invalid. The default is 16MiB. If you're changing this parameter, +# you may want to adjust max_value_size accordingly. This should be positive and less than 2048. +# Min unit: MiB +# native_transport_max_frame_size: 16MiB # The maximum number of concurrent client connections. # The default is -1, which means unlimited. @@ -735,7 +890,18 @@ native_transport_allow_older_protocols: true # values for heartbeat intervals have to be set on the client side. # # Idle connection timeouts are disabled by default. -# native_transport_idle_timeout_in_ms: 60000 +# Min unit: ms +# native_transport_idle_timeout: 60000ms + +# When enabled, limits the number of native transport requests dispatched for processing per second. +# Behavior once the limit has been breached depends on the value of THROW_ON_OVERLOAD specified in +# the STARTUP message sent by the client during connection establishment. (See section "4.1.1. STARTUP" +# in "CQL BINARY PROTOCOL v5".) With the THROW_ON_OVERLOAD flag enabled, messages that breach the limit +# are dropped, and an OverloadedException is thrown for the client to handle. When the flag is not +# enabled, the server will stop consuming messages from the channel/socket, putting backpressure on +# the client while already dispatched messages are processed. +# native_transport_rate_limiting_enabled: false +# native_transport_max_requests_per_second: 1000000 # The address or interface to bind the native transport server to. # @@ -764,7 +930,7 @@ rpc_address: 0.0.0.0 # be set to 0.0.0.0. If left blank, this will be set to the value of # rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must # be set. -broadcast_rpc_address: 172.20.0.3 +broadcast_rpc_address: 172.19.0.3 # enable or disable keepalive on rpc/native connections rpc_keepalive: true @@ -778,12 +944,14 @@ rpc_keepalive: true # /proc/sys/net/ipv4/tcp_wmem # /proc/sys/net/ipv4/tcp_wmem # and 'man tcp' -# internode_socket_send_buffer_size_in_bytes: +# Min unit: B +# internode_socket_send_buffer_size: # Uncomment to set socket buffer size for internode communication # Note that when setting this, the buffer size is limited by net.core.wmem_max # and when not setting it it is defined by net.ipv4.tcp_wmem -# internode_socket_receive_buffer_size_in_bytes: +# Min unit: B +# internode_socket_receive_buffer_size: # Set to true to have Cassandra create a hard link to each sstable # flushed or streamed locally in a backups/ subdirectory of the @@ -803,6 +971,14 @@ snapshot_before_compaction: false # lose data on truncation or drop. auto_snapshot: true +# Adds a time-to-live (TTL) to auto snapshots generated by table +# truncation or drop (when enabled). +# After the TTL is elapsed, the snapshot is automatically cleared. +# By default, auto snapshots *do not* have TTL, uncomment the property below +# to enable TTL on auto snapshots. +# Accepted units: d (days), h (hours) or m (minutes) +# auto_snapshot_ttl: 30d + # The act of creating or clearing a snapshot involves creating or removing # potentially tens of thousands of links, which can cause significant performance # impact, especially on consumer grade SSDs. A non-zero value here can @@ -820,7 +996,8 @@ snapshot_links_per_second: 0 # - but, Cassandra will keep the collation index in memory for hot # rows (as part of the key cache), so a larger granularity means # you can cache more hot rows -column_index_size_in_kb: 64 +# Min unit: KiB +column_index_size: 64KiB # Per sstable indexed key cache entries (the collation index in memory # mentioned above) exceeding this size will not be held on heap. @@ -829,7 +1006,8 @@ column_index_size_in_kb: 64 # # Note that this size refers to the size of the # serialized index information and not the size of the partition. -column_index_cache_size_in_kb: 2 +# Min unit: KiB +column_index_cache_size: 2KiB # Number of simultaneous compactions to allow, NOT including # validation "compactions" for anti-entropy repair. Simultaneous @@ -838,14 +1016,14 @@ column_index_cache_size_in_kb: 2 # during a single long running compactions. The default is usually # fine and if you experience problems with compaction running too # slowly or too fast, you should look at -# compaction_throughput_mb_per_sec first. +# compaction_throughput first. # # concurrent_compactors defaults to the smaller of (number of disks, # number of cores), with a minimum of 2 and a maximum of 8. # # If your data directories are backed by SSD, you should increase this # to the number of cores. -#concurrent_compactors: 1 +# concurrent_compactors: 1 # Number of simultaneous repair validations to allow. If not set or set to # a value less than 1, it defaults to the value of concurrent_compactors. @@ -866,37 +1044,60 @@ concurrent_materialized_view_builders: 1 # Setting this to 0 disables throttling. Note that this accounts for all types # of compaction, including validation compaction (building Merkle trees # for repairs). -compaction_throughput_mb_per_sec: 64 +compaction_throughput: 64MiB/s # When compacting, the replacement sstable(s) can be opened before they # are completely written, and used in place of the prior sstables for # any range that has been written. This helps to smoothly transfer reads # between the sstables, reducing page cache churn and keeping hot rows hot -sstable_preemptive_open_interval_in_mb: 50 +# Set sstable_preemptive_open_interval to null for disabled which is equivalent to +# sstable_preemptive_open_interval_in_mb being negative +# Min unit: MiB +sstable_preemptive_open_interval: 50MiB + +# Starting from 4.1 sstables support UUID based generation identifiers. They are disabled by default +# because once enabled, there is no easy way to downgrade. When the node is restarted with this option +# set to true, each newly created sstable will have a UUID based generation identifier and such files are +# not readable by previous Cassandra versions. At some point, this option will become true by default +# and eventually get removed from the configuration. +uuid_sstable_identifiers_enabled: false # When enabled, permits Cassandra to zero-copy stream entire eligible # SSTables between nodes, including every component. # This speeds up the network transfer significantly subject to -# throttling specified by stream_throughput_outbound_megabits_per_sec. +# throttling specified by entire_sstable_stream_throughput_outbound, +# and entire_sstable_inter_dc_stream_throughput_outbound +# for inter-DC transfers. # Enabling this will reduce the GC pressure on sending and receiving node. # When unset, the default is enabled. While this feature tries to keep the # disks balanced, it cannot guarantee it. This feature will be automatically # disabled if internode encryption is enabled. # stream_entire_sstables: true +# Throttles entire SSTable outbound streaming file transfers on +# this node to the given total throughput in Mbps. +# Setting this value to 0 it disables throttling. +# When unset, the default is 200 Mbps or 24 MiB/s. +# entire_sstable_stream_throughput_outbound: 24MiB/s + +# Throttles entire SSTable file streaming between datacenters. +# Setting this value to 0 disables throttling for entire SSTable inter-DC file streaming. +# When unset, the default is 200 Mbps or 24 MiB/s. +# entire_sstable_inter_dc_stream_throughput_outbound: 24MiB/s + # Throttles all outbound streaming file transfers on this node to the # given total throughput in Mbps. This is necessary because Cassandra does # mostly sequential IO when streaming data during bootstrap or repair, which # can lead to saturating the network connection and degrading rpc performance. -# When unset, the default is 200 Mbps or 25 MB/s. -# stream_throughput_outbound_megabits_per_sec: 200 +# When unset, the default is 200 Mbps or 24 MiB/s. +# stream_throughput_outbound: 24MiB/s # Throttles all streaming file transfer between the datacenters, # this setting allows users to throttle inter dc stream throughput in addition # to throttling all network stream traffic as configured with # stream_throughput_outbound_megabits_per_sec -# When unset, the default is 200 Mbps or 25 MB/s -# inter_dc_stream_throughput_outbound_megabits_per_sec: 200 +# When unset, the default is 200 Mbps or 24 MiB/s. +# inter_dc_stream_throughput_outbound: 24MiB/s # Server side timeouts for requests. The server will return a timeout exception # to the client if it can't complete an operation within the corresponding @@ -905,52 +1106,62 @@ sstable_preemptive_open_interval_in_mb: 50 # failures. # 2) operations that use too much CPU/read too much data (leading to memory build # up) by putting a limit to how long an operation will execute. -# For this reason, you should avoid putting these settings too high. In other words, -# if you are timing out requests because of underlying resource constraints then -# increasing the timeout will just cause more problems. Of course putting them too -# low is equally ill-advised since clients could get timeouts even for successful +# For this reason, you should avoid putting these settings too high. In other words, +# if you are timing out requests because of underlying resource constraints then +# increasing the timeout will just cause more problems. Of course putting them too +# low is equally ill-advised since clients could get timeouts even for successful # operations just because the timeout setting is too tight. # How long the coordinator should wait for read operations to complete. # Lowest acceptable value is 10 ms. -read_request_timeout_in_ms: 15000 +# Min unit: ms +read_request_timeout: 15000ms # How long the coordinator should wait for seq or index scans to complete. # Lowest acceptable value is 10 ms. -range_request_timeout_in_ms: 10000 +# Min unit: ms +range_request_timeout: 10000ms # How long the coordinator should wait for writes to complete. # Lowest acceptable value is 10 ms. -write_request_timeout_in_ms: 6000 +# Min unit: ms +write_request_timeout: 6000ms # How long the coordinator should wait for counter writes to complete. # Lowest acceptable value is 10 ms. -counter_write_request_timeout_in_ms: 5000 +# Min unit: ms +counter_write_request_timeout: 5000ms # How long a coordinator should continue to retry a CAS operation # that contends with other proposals for the same row. # Lowest acceptable value is 10 ms. -cas_contention_timeout_in_ms: 1000 +# Min unit: ms +cas_contention_timeout: 1000ms # How long the coordinator should wait for truncates to complete # (This can be much longer, because unless auto_snapshot is disabled # we need to flush first so we can snapshot before removing the data.) # Lowest acceptable value is 10 ms. -truncate_request_timeout_in_ms: 60000 +# Min unit: ms +truncate_request_timeout: 60000ms # The default timeout for other, miscellaneous operations. # Lowest acceptable value is 10 ms. -request_timeout_in_ms: 30000 +# Min unit: ms +request_timeout: 30000ms # Defensive settings for protecting Cassandra from true network partitions. # See (CASSANDRA-14358) for details. # # The amount of time to wait for internode tcp connections to establish. -# internode_tcp_connect_timeout_in_ms: 2000 +# Min unit: ms +# internode_tcp_connect_timeout: 2000ms # # The amount of time unacknowledged data is allowed on a connection before we throw out the connection # Note this is only supported on Linux + epoll, and it appears to behave oddly above a setting of 30000 # (it takes much longer than 30s) as of Linux 4.12. If you want something that high set this to 0 # which picks up the OS default and configure the net.ipv4.tcp_retries2 sysctl to be ~8. -# internode_tcp_user_timeout_in_ms: 30000 +# Min unit: ms +# internode_tcp_user_timeout: 30000ms # The amount of time unacknowledged data is allowed on a streaming connection. # The default is 5 minutes. Increase it or set it to 0 in order to increase the timeout. -# internode_streaming_tcp_user_timeout_in_ms: 300000 +# Min unit: ms +# internode_streaming_tcp_user_timeout: 300000ms # Global, per-endpoint and per-connection limits imposed on messages queued for delivery to other nodes # and waiting to be processed on arrival from other nodes in the cluster. These limits are applied to the on-wire @@ -958,7 +1169,7 @@ request_timeout_in_ms: 30000 # # The basic per-link limit is consumed in isolation before any endpoint or global limit is imposed. # Each node-pair has three links: urgent, small and large. So any given node may have a maximum of -# N*3*(internode_application_send_queue_capacity_in_bytes+internode_application_receive_queue_capacity_in_bytes) +# N*3*(internode_application_send_queue_capacity+internode_application_receive_queue_capacity) # messages queued without any coordination between them although in practice, with token-aware routing, only RF*tokens # nodes should need to communicate with significant bandwidth. # @@ -967,18 +1178,20 @@ request_timeout_in_ms: 30000 # The global limit is imposed on all messages exceeding the per-link limit, simultaneously with the per-endpoint limit, # on all links to or from any node in the cluster. # -# internode_application_send_queue_capacity_in_bytes: 4194304 #4MiB -# internode_application_send_queue_reserve_endpoint_capacity_in_bytes: 134217728 #128MiB -# internode_application_send_queue_reserve_global_capacity_in_bytes: 536870912 #512MiB -# internode_application_receive_queue_capacity_in_bytes: 4194304 #4MiB -# internode_application_receive_queue_reserve_endpoint_capacity_in_bytes: 134217728 #128MiB -# internode_application_receive_queue_reserve_global_capacity_in_bytes: 536870912 #512MiB +# Min unit: B +# internode_application_send_queue_capacity: 4MiB +# internode_application_send_queue_reserve_endpoint_capacity: 128MiB +# internode_application_send_queue_reserve_global_capacity: 512MiB +# internode_application_receive_queue_capacity: 4MiB +# internode_application_receive_queue_reserve_endpoint_capacity: 128MiB +# internode_application_receive_queue_reserve_global_capacity: 512MiB # How long before a node logs slow queries. Select queries that take longer than # this timeout to execute, will generate an aggregated log message, so that slow queries # can be identified. Set this value to zero to disable slow query logging. -slow_query_log_timeout_in_ms: 500 +# Min unit: ms +slow_query_log_timeout: 500ms # Enable operation timeout information exchange between nodes to accurately # measure request timeouts. If disabled, replicas will assume that requests @@ -988,21 +1201,66 @@ slow_query_log_timeout_in_ms: 500 # # Warning: It is generally assumed that users have setup NTP on their clusters, and that clocks are modestly in sync, # since this is a requirement for general correctness of last write wins. -#cross_node_timeout: true - -# Set keep-alive period for streaming -# This node will send a keep-alive message periodically with this period. -# If the node does not receive a keep-alive message from the peer for -# 2 keep-alive cycles the stream session times out and fail -# Default value is 300s (5 minutes), which means stalled stream -# times out in 10 minutes by default -# streaming_keep_alive_period_in_secs: 300 +# internode_timeout: true + +# Set period for idle state control messages for earlier detection of failed streams +# This node will send a keep-alive message periodically on the streaming's control channel. +# This ensures that any eventual SocketTimeoutException will occur within 2 keep-alive cycles +# If the node cannot send, or timeouts sending, the keep-alive message on the netty control channel +# the stream session is closed. +# Default value is 300s (5 minutes), which means stalled streams +# are detected within 10 minutes +# Specify 0 to disable. +# Min unit: s +# streaming_keep_alive_period: 300s # Limit number of connections per host for streaming # Increase this when you notice that joins are CPU-bound rather that network # bound (for example a few nodes with big files). # streaming_connections_per_host: 1 +# Settings for stream stats tracking; used by system_views.streaming table +# How long before a stream is evicted from tracking; this impacts both historic and currently running +# streams. +# streaming_state_expires: 3d +# How much memory may be used for tracking before evicting session from tracking; once crossed +# historic and currently running streams maybe impacted. +# streaming_state_size: 40MiB +# Enable/Disable tracking of streaming stats +# streaming_stats_enabled: true + +# Allows denying configurable access (rw/rr) to operations on configured ks, table, and partitions, intended for use by +# operators to manage cluster health vs application access. See CASSANDRA-12106 and CEP-13 for more details. +# partition_denylist_enabled: false + +# denylist_writes_enabled: true +# denylist_reads_enabled: true +# denylist_range_reads_enabled: true + +# The interval at which keys in the cache for denylisting will "expire" and async refresh from the backing DB. +# Note: this serves only as a fail-safe, as the usage pattern is expected to be "mutate state, refresh cache" on any +# changes to the underlying denylist entries. See documentation for details. +# Min unit: s +# denylist_refresh: 600s + +# In the event of errors on attempting to load the denylist cache, retry on this interval. +# Min unit: s +# denylist_initial_load_retry: 5s + +# We cap the number of denylisted keys allowed per table to keep things from growing unbounded. Nodes will warn above +# this limit while allowing new denylisted keys to be inserted. Denied keys are loaded in natural query / clustering +# ordering by partition key in case of overflow. +# denylist_max_keys_per_table: 1000 + +# We cap the total number of denylisted keys allowed in the cluster to keep things from growing unbounded. +# Nodes will warn on initial cache load that there are too many keys and be direct the operator to trim down excess +# entries to within the configured limits. +# denylist_max_keys_total: 10000 + +# Since the denylist in many ways serves to protect the health of the cluster from partitions operators have identified +# as being in a bad state, we usually want more robustness than just CL.ONE on operations to/from these tables to +# ensure that these safeguards are in place. That said, we allow users to configure this if they're so inclined. +# denylist_consistency_level: QUORUM # phi value that must be reached for a host to be marked down. # most users should never need to adjust this. @@ -1075,10 +1333,12 @@ endpoint_snitch: GossipingPropertyFileSnitch # controls how often to perform the more expensive part of host score # calculation -dynamic_snitch_update_interval_in_ms: 100 +# Min unit: ms +dynamic_snitch_update_interval: 100ms # controls how often to reset all host scores, allowing a bad host to # possibly recover -dynamic_snitch_reset_interval_in_ms: 600000 +# Min unit: ms +dynamic_snitch_reset_interval: 600000ms # if set greater than zero, this will allow # 'pinning' of replicas to hosts in order to increase cache capacity. # The badness threshold will control how much worse the pinned host has to be @@ -1111,39 +1371,44 @@ dynamic_snitch_badness_threshold: 1.0 # Step 2: Set optional=false (or remove it) and if you generated truststores and want to use mutual # auth set require_client_auth=true. Restart all nodes server_encryption_options: - # On outbound connections, determine which type of peers to securely connect to. - # The available options are : - # none : Do not encrypt outgoing connections - # dc : Encrypt connections to peers in other datacenters but not within datacenters - # rack : Encrypt connections to peers in other racks but not within racks - # all : Always use encrypted connections - internode_encryption: none - # When set to true, encrypted and unencrypted connections are allowed on the storage_port - # This should _only be true_ while in unencrypted or transitional operation - # optional defaults to true if internode_encryption is none - # optional: true - # If enabled, will open up an encrypted listening socket on ssl_storage_port. Should only be used - # during upgrade to 4.0; otherwise, set to false. - enable_legacy_ssl_storage_port: false - # Set to a valid keystore if internode_encryption is dc, rack or all - keystore: conf/.keystore - keystore_password: cassandra - # Verify peer server certificates - require_client_auth: false - # Set to a valid trustore if require_client_auth is true - truststore: conf/.truststore - truststore_password: cassandra - # Verify that the host name in the certificate matches the connected host - require_endpoint_verification: false - # More advanced defaults: - # protocol: TLS - # store_type: JKS - # cipher_suites: [ - # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - # TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, - # TLS_RSA_WITH_AES_256_CBC_SHA - # ] + # On outbound connections, determine which type of peers to securely connect to. + # The available options are : + # none : Do not encrypt outgoing connections + # dc : Encrypt connections to peers in other datacenters but not within datacenters + # rack : Encrypt connections to peers in other racks but not within racks + # all : Always use encrypted connections + internode_encryption: none + # When set to true, encrypted and unencrypted connections are allowed on the storage_port + # This should _only be true_ while in unencrypted or transitional operation + # optional defaults to true if internode_encryption is none + # optional: true + # If enabled, will open up an encrypted listening socket on ssl_storage_port. Should only be used + # during upgrade to 4.0; otherwise, set to false. + legacy_ssl_storage_port_enabled: false + # Set to a valid keystore if internode_encryption is dc, rack or all + keystore: conf/.keystore + keystore_password: cassandra + # Configure the way Cassandra creates SSL contexts. + # To use PEM-based key material, see org.apache.cassandra.security.PEMBasedSslContextFactory + # ssl_context_factory: + # # Must be an instance of org.apache.cassandra.security.ISslContextFactory + # class_name: org.apache.cassandra.security.DefaultSslContextFactory + # Verify peer server certificates + require_client_auth: false + # Set to a valid trustore if require_client_auth is true + truststore: conf/.truststore + truststore_password: cassandra + # Verify that the host name in the certificate matches the connected host + require_endpoint_verification: false + # More advanced defaults: + # protocol: TLS + # store_type: JKS + # cipher_suites: [ + # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + # TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, + # TLS_RSA_WITH_AES_256_CBC_SHA + # ] # Configure client-to-server encryption. # @@ -1158,29 +1423,34 @@ server_encryption_options: # Step 2: Set optional=false (or remove it) and if you generated truststores and want to use mutual # auth set require_client_auth=true. Restart all nodes client_encryption_options: - # Enable client-to-server encryption - enabled: false - # When set to true, encrypted and unencrypted connections are allowed on the native_transport_port - # This should _only be true_ while in unencrypted or transitional operation - # optional defaults to true when enabled is false, and false when enabled is true. - # optional: true - # Set keystore and keystore_password to valid keystores if enabled is true - keystore: conf/.keystore - keystore_password: cassandra - # Verify client certificates - require_client_auth: false - # Set trustore and truststore_password if require_client_auth is true - # truststore: conf/.truststore - # truststore_password: cassandra - # More advanced defaults: - # protocol: TLS - # store_type: JKS - # cipher_suites: [ - # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - # TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, - # TLS_RSA_WITH_AES_256_CBC_SHA - # ] + # Enable client-to-server encryption + enabled: false + # When set to true, encrypted and unencrypted connections are allowed on the native_transport_port + # This should _only be true_ while in unencrypted or transitional operation + # optional defaults to true when enabled is false, and false when enabled is true. + # optional: true + # Set keystore and keystore_password to valid keystores if enabled is true + keystore: conf/.keystore + keystore_password: cassandra + # Configure the way Cassandra creates SSL contexts. + # To use PEM-based key material, see org.apache.cassandra.security.PEMBasedSslContextFactory + # ssl_context_factory: + # # Must be an instance of org.apache.cassandra.security.ISslContextFactory + # class_name: org.apache.cassandra.security.DefaultSslContextFactory + # Verify client certificates + require_client_auth: false + # Set trustore and truststore_password if require_client_auth is true + # truststore: conf/.truststore + # truststore_password: cassandra + # More advanced defaults: + # protocol: TLS + # store_type: JKS + # cipher_suites: [ + # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + # TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, + # TLS_RSA_WITH_AES_256_CBC_SHA + # ] # internode_compression controls whether traffic between nodes is # compressed. @@ -1203,28 +1473,22 @@ internode_compression: dc inter_dc_tcp_nodelay: false # TTL for different trace types used during logging of the repair process. -tracetype_query_ttl: 86400 -tracetype_repair_ttl: 604800 +# Min unit: s +trace_type_query_ttl: 1d +# Min unit: s +trace_type_repair_ttl: 7d -# If unset, all GC Pauses greater than gc_log_threshold_in_ms will log at +# If unset, all GC Pauses greater than gc_log_threshold will log at # INFO level # UDFs (user defined functions) are disabled by default. # As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code. -enable_user_defined_functions: false +user_defined_functions_enabled: false # Enables scripted UDFs (JavaScript UDFs). -# Java UDFs are always enabled, if enable_user_defined_functions is true. +# Java UDFs are always enabled, if user_defined_functions_enabled is true. # Enable this option to be able to use UDFs with "language javascript" or any custom JSR-223 provider. -# This option has no effect, if enable_user_defined_functions is false. -enable_scripted_user_defined_functions: false - -# The default Windows kernel timer and scheduling resolution is 15.6ms for power conservation. -# Lowering this value on Windows can provide much tighter latency and better throughput, however -# some virtualized environments may see a negative performance impact from changing this setting -# below their system default. The sysinternals 'clockres' tool can confirm your system's default -# setting. -windows_timer_interval: 1 - +# This option has no effect, if user_defined_functions_enabled is false. +scripted_user_defined_functions_enabled: false # Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from # a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by @@ -1239,19 +1503,19 @@ windows_timer_interval: 1 # Currently, only the following file types are supported for transparent data encryption, although # more are coming in future cassandra releases: commitlog, hints transparent_data_encryption_options: - enabled: false - chunk_length_kb: 64 - cipher: AES/CBC/PKCS5Padding - key_alias: testing:1 - # CBC IV length for AES needs to be 16 bytes (which is also the default size) - # iv_length: 16 - key_provider: - - class_name: org.apache.cassandra.security.JKSKeyProvider - parameters: - - keystore: conf/.keystore - keystore_password: cassandra - store_type: JCEKS - key_password: cassandra + enabled: false + chunk_length_kb: 64 + cipher: AES/CBC/PKCS5Padding + key_alias: testing:1 + # CBC IV length for AES needs to be 16 bytes (which is also the default size) + # iv_length: 16 + key_provider: + - class_name: org.apache.cassandra.security.JKSKeyProvider + parameters: + - keystore: conf/.keystore + keystore_password: cassandra + store_type: JCEKS + key_password: cassandra ##################### @@ -1290,32 +1554,54 @@ replica_filtering_protection: cached_rows_warn_threshold: 2000 cached_rows_fail_threshold: 32000 -# Log WARN on any multiple-partition batch size exceeding this value. 5kb per batch by default. +# Log WARN on any multiple-partition batch size exceeding this value. 5KiB per batch by default. # Caution should be taken on increasing the size of this threshold as it can lead to node instability. -batch_size_warn_threshold_in_kb: 5 +# Min unit: KiB +batch_size_warn_threshold: 5KiB -# Fail any multiple-partition batch exceeding this value. 50kb (10x warn threshold) by default. -batch_size_fail_threshold_in_kb: 50 +# Fail any multiple-partition batch exceeding this value. 50KiB (10x warn threshold) by default. +# Min unit: KiB +batch_size_fail_threshold: 50KiB # Log WARN on any batches not of type LOGGED than span across more partitions than this limit unlogged_batch_across_partitions_warn_threshold: 10 # Log a warning when compacting partitions larger than this value -compaction_large_partition_warning_threshold_mb: 100 +compaction_large_partition_warning_threshold: 100MiB + +# Log a warning when writing more tombstones than this value to a partition +compaction_tombstone_warning_threshold: 100000 # GC Pauses greater than 200 ms will be logged at INFO level # This threshold can be adjusted to minimize logging if necessary -# gc_log_threshold_in_ms: 200 +# Min unit: ms +# gc_log_threshold: 200ms -# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level +# GC Pauses greater than gc_warn_threshold will be logged at WARN level # Adjust the threshold based on your application throughput requirement. Setting to 0 # will deactivate the feature. -# gc_warn_threshold_in_ms: 1000 +# Min unit: ms +# gc_warn_threshold: 1000ms # Maximum size of any value in SSTables. Safety measure to detect SSTable corruption # early. Any value size larger than this threshold will result into marking an SSTable -# as corrupted. This should be positive and less than 2048. -# max_value_size_in_mb: 256 +# as corrupted. This should be positive and less than 2GiB. +# Min unit: MiB +# max_value_size: 256MiB + +# ** Impact on keyspace creation ** +# If replication factor is not mentioned as part of keyspace creation, default_keyspace_rf would apply. +# Changing this configuration would only take effect for keyspaces created after the change, but does not impact +# existing keyspaces created prior to the change. +# ** Impact on keyspace alter ** +# When altering a keyspace from NetworkTopologyStrategy to SimpleStrategy, default_keyspace_rf is applied if rf is not +# explicitly mentioned. +# ** Impact on system keyspaces ** +# This would also apply for any system keyspaces that need replication factor. +# A further note about system keyspaces - system_traces and system_distributed keyspaces take RF of 2 or default, +# whichever is higher, and system_auth keyspace takes RF of 1 or default, whichever is higher. +# Suggested value for use in production: 3 +# default_keyspace_rf: 1 # Track a metric per keyspace indicating whether replication achieved the ideal consistency # level for writes without timing out. This is different from the consistency level requested by @@ -1331,36 +1617,38 @@ compaction_large_partition_warning_threshold_mb: 100 # Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs # on audit_logging for full details about the various configuration options. audit_logging_options: - enabled: false - logger: - - class_name: BinAuditLogger - # audit_logs_dir: - # included_keyspaces: - # excluded_keyspaces: system, system_schema, system_virtual_schema - # included_categories: - # excluded_categories: - # included_users: - # excluded_users: - # roll_cycle: HOURLY - # block: true - # max_queue_weight: 268435456 # 256 MiB - # max_log_size: 17179869184 # 16 GiB - ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled: - # archive_command: - # max_archive_retries: 10 + enabled: false + logger: + - class_name: BinAuditLogger + # audit_logs_dir: + # included_keyspaces: + # excluded_keyspaces: system, system_schema, system_virtual_schema + # included_categories: + # excluded_categories: + # included_users: + # excluded_users: + # roll_cycle: HOURLY + # block: true + # max_queue_weight: 268435456 # 256 MiB + # max_log_size: 17179869184 # 16 GiB + ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled: + # archive_command: + # max_archive_retries: 10 # default options for full query logging - these can be overridden from command line when executing # nodetool enablefullquerylog -#full_query_logging_options: - # log_dir: - # roll_cycle: HOURLY - # block: true - # max_queue_weight: 268435456 # 256 MiB - # max_log_size: 17179869184 # 16 GiB - ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled: - # archive_command: - # max_archive_retries: 10 +# full_query_logging_options: + # log_dir: + # roll_cycle: HOURLY + # block: true + # max_queue_weight: 268435456 # 256 MiB + # max_log_size: 17179869184 # 16 GiB + ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled: + # archive_command: + ## note that enabling this allows anyone with JMX/nodetool access to run local shell commands as the user running cassandra + # allow_nodetool_archive_command: false + # max_archive_retries: 10 # validate tombstones on reads and compaction # can be either "disabled", "warn" or "exception" @@ -1395,25 +1683,195 @@ report_unconfirmed_repaired_data_mismatches: false # Having many tables and/or keyspaces negatively affects performance of many operations in the # cluster. When the number of tables/keyspaces in the cluster exceeds the following thresholds # a client warning will be sent back to the user when creating a table or keyspace. +# As of cassandra 4.1, these properties are deprecated in favor of keyspaces_warn_threshold and tables_warn_threshold # table_count_warn_threshold: 150 # keyspace_count_warn_threshold: 40 +# configure the read and write consistency levels for modifications to auth tables +# auth_read_consistency_level: LOCAL_QUORUM +# auth_write_consistency_level: EACH_QUORUM + +# Delays on auth resolution can lead to a thundering herd problem on reconnects; this option will enable +# warming of auth caches prior to node completing startup. See CASSANDRA-16958 +# auth_cache_warming_enabled: false + ######################### # EXPERIMENTAL FEATURES # ######################### # Enables materialized view creation on this node. # Materialized views are considered experimental and are not recommended for production use. -enable_materialized_views: false +materialized_views_enabled: false # Enables SASI index creation on this node. # SASI indexes are considered experimental and are not recommended for production use. -enable_sasi_indexes: false +sasi_indexes_enabled: false # Enables creation of transiently replicated keyspaces on this node. # Transient replication is experimental and is not recommended for production use. -enable_transient_replication: false +transient_replication_enabled: false # Enables the used of 'ALTER ... DROP COMPACT STORAGE' statements on this node. # 'ALTER ... DROP COMPACT STORAGE' is considered experimental and is not recommended for production use. -enable_drop_compact_storage: false +drop_compact_storage_enabled: false + +# Whether or not USE <keyspace> is allowed. This is enabled by default to avoid failure on upgrade. +#use_statements_enabled: true + +# When the client triggers a protocol exception or unknown issue (Cassandra bug) we increment +# a client metric showing this; this logic will exclude specific subnets from updating these +# metrics +#client_error_reporting_exclusions: +# subnets: +# - 127.0.0.1 +# - 127.0.0.0/31 + +# Enables read thresholds (warn/fail) across all replicas for reporting back to the client. +# See: CASSANDRA-16850 +# read_thresholds_enabled: false # scheduled to be set true in 4.2 +# When read_thresholds_enabled: true, this tracks the materialized size of a query on the +# coordinator. If coordinator_read_size_warn_threshold is defined, this will emit a warning +# to clients with details on what query triggered this as well as the size of the result set; if +# coordinator_read_size_fail_threshold is defined, this will fail the query after it +# has exceeded this threshold, returning a read error to the user. +# coordinator_read_size_warn_threshold: +# coordinator_read_size_fail_threshold: +# When read_thresholds_enabled: true, this tracks the size of the local read (as defined by +# heap size), and will warn/fail based off these thresholds; undefined disables these checks. +# local_read_size_warn_threshold: +# local_read_size_fail_threshold: +# When read_thresholds_enabled: true, this tracks the expected memory size of the RowIndexEntry +# and will warn/fail based off these thresholds; undefined disables these checks +# row_index_read_size_warn_threshold: +# row_index_read_size_fail_threshold: + +# Guardrail to warn or fail when creating more user keyspaces than threshold. +# The two thresholds default to -1 to disable. +# keyspaces_warn_threshold: -1 +# keyspaces_fail_threshold: -1 +# Guardrail to warn or fail when creating more user tables than threshold. +# The two thresholds default to -1 to disable. +# tables_warn_threshold: -1 +# tables_fail_threshold: -1 +# Guardrail to enable or disable the ability to create uncompressed tables +# uncompressed_tables_enabled: true +# Guardrail to warn or fail when creating/altering a table with more columns per table than threshold. +# The two thresholds default to -1 to disable. +# columns_per_table_warn_threshold: -1 +# columns_per_table_fail_threshold: -1 +# Guardrail to warn or fail when creating more secondary indexes per table than threshold. +# The two thresholds default to -1 to disable. +# secondary_indexes_per_table_warn_threshold: -1 +# secondary_indexes_per_table_fail_threshold: -1 +# Guardrail to enable or disable the creation of secondary indexes +# secondary_indexes_enabled: true +# Guardrail to warn or fail when creating more materialized views per table than threshold. +# The two thresholds default to -1 to disable. +# materialized_views_per_table_warn_threshold: -1 +# materialized_views_per_table_fail_threshold: -1 +# Guardrail to warn about, ignore or reject properties when creating tables. By default all properties are allowed. +# table_properties_warned: [] +# table_properties_ignored: [] +# table_properties_disallowed: [] +# Guardrail to allow/disallow user-provided timestamps. Defaults to true. +# user_timestamps_enabled: true +# Guardrail to allow/disallow GROUP BY functionality. +# group_by_enabled: true +# Guardrail to allow/disallow TRUNCATE and DROP TABLE statements +# drop_truncate_table_enabled: true +# Guardrail to warn or fail when using a page size greater than threshold. +# The two thresholds default to -1 to disable. +# page_size_warn_threshold: -1 +# page_size_fail_threshold: -1 +# Guardrail to allow/disallow list operations that require read before write, i.e. setting list element by index and +# removing list elements by either index or value. Defaults to true. +# read_before_write_list_operations_enabled: true +# Guardrail to warn or fail when querying with an IN restriction selecting more partition keys than threshold. +# The two thresholds default to -1 to disable. +# partition_keys_in_select_warn_threshold: -1 +# partition_keys_in_select_fail_threshold: -1 +# Guardrail to warn or fail when an IN query creates a cartesian product with a size exceeding threshold, +# eg. "a in (1,2,...10) and b in (1,2...10)" results in cartesian product of 100. +# The two thresholds default to -1 to disable. +# in_select_cartesian_product_warn_threshold: -1 +# in_select_cartesian_product_fail_threshold: -1 +# Guardrail to warn about or reject read consistency levels. By default, all consistency levels are allowed. +# read_consistency_levels_warned: [] +# read_consistency_levels_disallowed: [] +# Guardrail to warn about or reject write consistency levels. By default, all consistency levels are allowed. +# write_consistency_levels_warned: [] +# write_consistency_levels_disallowed: [] +# Guardrail to warn or fail when encountering larger size of collection data than threshold. +# At query time this guardrail is applied only to the collection fragment that is being writen, even though in the case +# of non-frozen collections there could be unaccounted parts of the collection on the sstables. This is done this way to +# prevent read-before-write. The guardrail is also checked at sstable write time to detect large non-frozen collections, +# although in that case exceeding the fail threshold will only log an error message, without interrupting the operation. +# The two thresholds default to null to disable. +# Min unit: B +# collection_size_warn_threshold: +# Min unit: B +# collection_size_fail_threshold: +# Guardrail to warn or fail when encountering more elements in collection than threshold. +# At query time this guardrail is applied only to the collection fragment that is being writen, even though in the case +# of non-frozen collections there could be unaccounted parts of the collection on the sstables. This is done this way to +# prevent read-before-write. The guardrail is also checked at sstable write time to detect large non-frozen collections, +# although in that case exceeding the fail threshold will only log an error message, without interrupting the operation. +# The two thresholds default to -1 to disable. +# items_per_collection_warn_threshold: -1 +# items_per_collection_fail_threshold: -1 +# Guardrail to allow/disallow querying with ALLOW FILTERING. Defaults to true. +# allow_filtering_enabled: true +# Guardrail to warn or fail when creating a user-defined-type with more fields in than threshold. +# Default -1 to disable. +# fields_per_udt_warn_threshold: -1 +# fields_per_udt_fail_threshold: -1 +# Guardrail to warn or fail when local data disk usage percentage exceeds threshold. Valid values are in [1, 100]. +# This is only used for the disks storing data directories, so it won't count any separate disks used for storing +# the commitlog, hints nor saved caches. The disk usage is the ratio between the amount of space used by the data +# directories and the addition of that same space and the remaining free space on disk. The main purpose of this +# guardrail is rejecting user writes when the disks are over the defined usage percentage, so the writes done by +# background processes such as compaction and streaming don't fail due to a full disk. The limits should be defined +# accordingly to the expected data growth due to those background processes, so for example a compaction strategy +# doubling the size of the data would require to keep the disk usage under 50%. +# The two thresholds default to -1 to disable. +# data_disk_usage_percentage_warn_threshold: -1 +# data_disk_usage_percentage_fail_threshold: -1 +# Allows defining the max disk size of the data directories when calculating thresholds for +# disk_usage_percentage_warn_threshold and disk_usage_percentage_fail_threshold, so if this is greater than zero they +# become percentages of a fixed size on disk instead of percentages of the physically available disk size. This should +# be useful when we have a large disk and we only want to use a part of it for Cassandra's data directories. +# Valid values are in [1, max available disk size of all data directories]. +# Defaults to null to disable and use the physically available disk size of data directories during calculations. +# Min unit: B +# data_disk_usage_max_disk_size: +# Guardrail to warn or fail when the minimum replication factor is lesser than threshold. +# This would also apply to system keyspaces. +# Suggested value for use in production: 2 or higher +# minimum_replication_factor_warn_threshold: -1 +# minimum_replication_factor_fail_threshold: -1 + +# Startup Checks are executed as part of Cassandra startup process, not all of them +# are configurable (so you can disable them) but these which are enumerated bellow. +# Uncomment the startup checks and configure them appropriately to cover your needs. +# +#startup_checks: +# Verifies correct ownership of attached locations on disk at startup. See CASSANDRA-16879 for more details. +# check_filesystem_ownership: +# enabled: false +# ownership_token: "sometoken" # (overriden by "CassandraOwnershipToken" system property) +# ownership_filename: ".cassandra_fs_ownership" # (overriden by "cassandra.fs_ownership_filename") +# Prevents a node from starting if snitch's data center differs from previous data center. +# check_dc: +# enabled: true # (overriden by cassandra.ignore_dc system property) +# Prevents a node from starting if snitch's rack differs from previous rack. +# check_rack: +# enabled: true # (overriden by cassandra.ignore_rack system property) +# Enable this property to fail startup if the node is down for longer than gc_grace_seconds, to potentially +# prevent data resurrection on tables with deletes. By default, this will run against all keyspaces and tables +# except the ones specified on excluded_keyspaces and excluded_tables. +# check_data_resurrection: +# enabled: false +# file where Cassandra periodically writes the last time it was known to run +# heartbeat_file: /var/lib/cassandra/data/cassandra-heartbeat +# excluded_keyspaces: # comma separated list of keyspaces to exclude from the check +# excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check
