Updated Branches: refs/heads/trunk 567928f5f -> a8e848241
Make SerializingCache Memory Pluggable Patch by vijay, reviewed by jbellis for CASSANDRA-3997 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a8e84824 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a8e84824 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a8e84824 Branch: refs/heads/trunk Commit: a8e848241a5eab8947cb521dd631da91deaa1d5d Parents: 567928f Author: Vijay Parthasarathy <vijay2...@gmail.com> Authored: Sun Nov 18 07:59:18 2012 -0800 Committer: Vijay Parthasarathy <vijay2...@gmail.com> Committed: Sun Nov 18 07:59:18 2012 -0800 ---------------------------------------------------------------------- conf/cassandra-env.sh | 5 ++ conf/cassandra.yaml | 11 +++ src/java/org/apache/cassandra/config/Config.java | 2 + .../cassandra/config/DatabaseDescriptor.java | 9 ++- .../org/apache/cassandra/io/util/IAllocator.java | 24 +++++++ .../cassandra/io/util/JEMallocAllocator.java | 43 ++++++++++++ src/java/org/apache/cassandra/io/util/Memory.java | 23 ++----- .../apache/cassandra/io/util/NativeAllocator.java | 51 +++++++++++++++ .../org/apache/cassandra/utils/FBUtilities.java | 8 ++ 9 files changed, 157 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8e84824/conf/cassandra-env.sh ---------------------------------------------------------------------- diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh index 95760dd..8df11ea 100644 --- a/conf/cassandra-env.sh +++ b/conf/cassandra-env.sh @@ -208,6 +208,11 @@ JVM_OPTS="$JVM_OPTS -XX:+UseCMSInitiatingOccupancyOnly" # JVM_OPTS="$JVM_OPTS -XX:PrintFLSStatistics=1" # JVM_OPTS="$JVM_OPTS -Xloggc:/var/log/cassandra/gc-`date +%s`.log" +# Configure the following for JEMallocAllocator and if jemalloc is not available in the system +# library path (Example: /usr/local/lib/). Usually "make install" will do the right thing. +# export LD_LIBRARY_PATH=<JEMALLOC_HOME>/lib/ +# JVM_OPTS="-Djava.library.path=<JEMALLOC_HOME>/lib/" + # uncomment to have Cassandra JVM listen for remote debuggers/profilers on port 1414 # JVM_OPTS="$JVM_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=1414" http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8e84824/conf/cassandra.yaml ---------------------------------------------------------------------- diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index a79e150..9c74475 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -164,6 +164,17 @@ row_cache_save_period: 0 # Defaults to SerializingCacheProvider row_cache_provider: SerializingCacheProvider +# The pluggable Memory allocation for Off heap row cache, Experiments show that JEMAlloc +# saves some memory than the native GCC allocator. +# +# Supported values are: NativeAllocator, JEMallocAllocator +# +# If you intend to use JEMallocAllocator you have to install JEMalloc as library and +# modify cassandra-env.sh as directed in the file. +# +# Defaults to NativeAllocator +# memory_allocator: NativeAllocator + # saved caches saved_caches_directory: /var/lib/cassandra/saved_caches http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8e84824/src/java/org/apache/cassandra/config/Config.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index c605a3a..d2f268a 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -18,6 +18,7 @@ package org.apache.cassandra.config; import org.apache.cassandra.cache.SerializingCacheProvider; +import org.apache.cassandra.io.util.NativeAllocator; /** * A class that contains configuration properties for the cassandra node it runs within. @@ -155,6 +156,7 @@ public class Config public volatile int row_cache_save_period = 0; public int row_cache_keys_to_save = Integer.MAX_VALUE; public String row_cache_provider = SerializingCacheProvider.class.getSimpleName(); + public String memory_allocator = NativeAllocator.class.getSimpleName(); public boolean populate_io_cache_on_flush = false; private static boolean loadYaml = true; http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8e84824/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index e615887..e14e589 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -40,6 +40,7 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.IAllocator; import org.apache.cassandra.locator.DynamicEndpointSnitch; import org.apache.cassandra.locator.EndpointSnitchInfo; import org.apache.cassandra.locator.IEndpointSnitch; @@ -49,7 +50,6 @@ import org.apache.cassandra.scheduler.IRequestScheduler; import org.apache.cassandra.scheduler.NoScheduler; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.MigrationManager; -import org.apache.cassandra.thrift.ThriftServer; import org.apache.cassandra.utils.FBUtilities; import org.yaml.snakeyaml.Loader; import org.yaml.snakeyaml.TypeDescription; @@ -86,6 +86,7 @@ public class DatabaseDescriptor private static long keyCacheSizeInMB; private static IRowCacheProvider rowCacheProvider; + private static IAllocator memoryAllocator; private static String localDC; @@ -441,6 +442,7 @@ public class DatabaseDescriptor } rowCacheProvider = FBUtilities.newCacheProvider(conf.row_cache_provider); + memoryAllocator = FBUtilities.newOffHeapAllocator(conf.memory_allocator); if(conf.encryption_options != null) { @@ -1228,6 +1230,11 @@ public class DatabaseDescriptor return rowCacheProvider; } + public static IAllocator getoffHeapMemoryAllocator() + { + return memoryAllocator; + } + public static int getStreamingSocketTimeout() { return conf.streaming_socket_timeout_in_ms; http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8e84824/src/java/org/apache/cassandra/io/util/IAllocator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/IAllocator.java b/src/java/org/apache/cassandra/io/util/IAllocator.java new file mode 100644 index 0000000..d633617 --- /dev/null +++ b/src/java/org/apache/cassandra/io/util/IAllocator.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.util; + +public interface IAllocator +{ + long allocate(long size); + void free(long peer); +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8e84824/src/java/org/apache/cassandra/io/util/JEMallocAllocator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/JEMallocAllocator.java b/src/java/org/apache/cassandra/io/util/JEMallocAllocator.java new file mode 100644 index 0000000..45057f2 --- /dev/null +++ b/src/java/org/apache/cassandra/io/util/JEMallocAllocator.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.util; + +import com.sun.jna.Library; +import com.sun.jna.Native; + +public class JEMallocAllocator implements IAllocator +{ + public interface JEMLibrary extends Library + { + long malloc(long size); + + void free(long pointer); + } + + private static final JEMLibrary instance = (JEMLibrary) Native.loadLibrary("jemalloc", JEMLibrary.class); + + public long allocate(long size) + { + return instance.malloc(size); + } + + public void free(long peer) + { + instance.free(peer); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8e84824/src/java/org/apache/cassandra/io/util/Memory.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/Memory.java b/src/java/org/apache/cassandra/io/util/Memory.java index 76c65e0..9ed6981 100644 --- a/src/java/org/apache/cassandra/io/util/Memory.java +++ b/src/java/org/apache/cassandra/io/util/Memory.java @@ -19,28 +19,15 @@ package org.apache.cassandra.io.util; import sun.misc.Unsafe; -import java.lang.reflect.Field; +import org.apache.cassandra.config.DatabaseDescriptor; /** * An off-heap region of memory that must be manually free'd when no longer needed. */ public class Memory { - private static final Unsafe unsafe; - - static - { - try - { - Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); - field.setAccessible(true); - unsafe = (sun.misc.Unsafe) field.get(null); - } - catch (Exception e) - { - throw new AssertionError(e); - } - } + private static final Unsafe unsafe = NativeAllocator.unsafe; + private static final IAllocator allocator = DatabaseDescriptor.getoffHeapMemoryAllocator(); protected long peer; // size of the memory region @@ -49,7 +36,7 @@ public class Memory protected Memory(long bytes) { size = bytes; - peer = unsafe.allocateMemory(size); + peer = allocator.allocate(size); } public static Memory allocate(long bytes) @@ -150,7 +137,7 @@ public class Memory public void free() { assert peer != 0; - unsafe.freeMemory(peer); + allocator.free(peer); peer = 0; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8e84824/src/java/org/apache/cassandra/io/util/NativeAllocator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/NativeAllocator.java b/src/java/org/apache/cassandra/io/util/NativeAllocator.java new file mode 100644 index 0000000..da0b362 --- /dev/null +++ b/src/java/org/apache/cassandra/io/util/NativeAllocator.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.util; + +import java.lang.reflect.Field; + +import sun.misc.Unsafe; + +public class NativeAllocator implements IAllocator +{ + static final Unsafe unsafe; + static + { + try + { + Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + unsafe = (sun.misc.Unsafe) field.get(null); + } + catch (Exception e) + { + throw new AssertionError(e); + } + } + + public long allocate(long size) + { + return unsafe.allocateMemory(size); + } + + public void free(long peer) + { + unsafe.freeMemory(peer); + } + +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/a8e84824/src/java/org/apache/cassandra/utils/FBUtilities.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/utils/FBUtilities.java b/src/java/org/apache/cassandra/utils/FBUtilities.java index 2f59ca1..bc910bd 100644 --- a/src/java/org/apache/cassandra/utils/FBUtilities.java +++ b/src/java/org/apache/cassandra/utils/FBUtilities.java @@ -49,6 +49,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.IAllocator; import org.apache.cassandra.net.IAsyncResult; import org.apache.thrift.TBase; import org.apache.thrift.TDeserializer; @@ -392,6 +393,13 @@ public class FBUtilities return FBUtilities.construct(partitionerClassName, "partitioner"); } + public static IAllocator newOffHeapAllocator(String offheap_allocator) throws ConfigurationException + { + if (!offheap_allocator.contains(".")) + offheap_allocator = "org.apache.cassandra.io.util." + offheap_allocator; + return FBUtilities.construct(offheap_allocator, "off-heap allocator"); + } + /** * @return The Class for the given name. * @param classname Fully qualified classname.